Fix optional dependencies problem.

kobalt/src/Build.kt

@@ -26,7 +26,7 @@ object Versions {
     val okio = "1.6.0"
     val retrofit = "2.1.0"
     val gson = "2.6.2"
-    val aether = "1.1.0"
+    val aether = "1.0.0.v20140518"
     val sonatypeAether = "1.13.1"
     val maven = "3.3.9"
 }
@@ -102,6 +102,7 @@ val kobaltPluginApi = project {
 
                 "org.slf4j:slf4j-nop:1.6.0",
                 "org.eclipse.aether:aether-spi:${Versions.aether}",
+                "org.eclipse.aether:aether-util:${Versions.aether}",
                 "org.eclipse.aether:aether-impl:${Versions.aether}",
                 "org.eclipse.aether:aether-connector-basic:${Versions.aether}",
                 "org.eclipse.aether:aether-transport-file:${Versions.aether}",
@@ -158,6 +159,7 @@ val kobaltApp = project(kobaltPluginApi, wrapper) {
                 "com.squareup.retrofit2:converter-gson:${Versions.retrofit}",
                 "com.squareup.okhttp3:okhttp-ws:${Versions.okhttp}",
                 "biz.aQute.bnd:bndlib:2.4.0",
+                "org.sonatype.aether:aether-api:${Versions.sonatypeAether}",
 
                 "com.squareup.okhttp3:logging-interceptor:3.2.0",
 
@@ -176,7 +178,10 @@ val kobaltApp = project(kobaltPluginApi, wrapper) {
 
     dependenciesTest {
         compile("org.testng:testng:6.9.11",
-                "org.assertj:assertj-core:3.4.1")
+                "org.assertj:assertj-core:3.4.1",
+                "org.eclipse.aether:aether-spi:${Versions.aether}",
+                "org.eclipse.aether:aether-util:${Versions.aether}"
+                )
     }
 
     assemble {

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/JarGenerator.kt

@@ -104,7 +104,7 @@ class JarGenerator @Inject constructor(val dependencyManager: DependencyManager)
                 context.variant.productFlavor.compileDependencies +
                 context.variant.productFlavor.compileRuntimeDependencies
             val transitiveDependencies = dependencyManager.calculateDependencies(project, context,
-                    listOf(Scope.COMPILE), allDependencies)
+                    scopes = listOf(Scope.COMPILE), passedDependencies = allDependencies)
             transitiveDependencies.map {
                 it.jarFile.get()
             }.forEach { file : File ->

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/IDependencyManager.kt

@@ -1,6 +1,8 @@
 package com.beust.kobalt.api
 
+import com.beust.kobalt.maven.aether.Filters
 import com.beust.kobalt.maven.aether.Scope
+import org.eclipse.aether.graph.DependencyFilter
 
 /**
  * Manage the creation of dependencies and also provide dependencies for projects.
@@ -36,6 +38,7 @@ interface IDependencyManager {
      * allDependencies is typically either compileDependencies or testDependencies
      */
     fun calculateDependencies(project: Project?, context: KobaltContext,
-            scopeFilters: Collection<Scope> = listOf(Scope.COMPILE),
+            dependencyFilter: DependencyFilter = Filters.EXCLUDE_OPTIONAL_FILTER,
+            scopes: List<Scope> = listOf(Scope.COMPILE),
             vararg passedDependencies: List<IClasspathDependency>): List<IClasspathDependency>
 }

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/DependencyManager.kt

@@ -2,12 +2,15 @@ package com.beust.kobalt.maven
 
 import com.beust.kobalt.KobaltException
 import com.beust.kobalt.api.*
+import com.beust.kobalt.maven.aether.Filters
 import com.beust.kobalt.maven.aether.KobaltAether
 import com.beust.kobalt.maven.aether.Scope
 import com.beust.kobalt.maven.dependency.FileDependency
 import com.beust.kobalt.misc.KFiles
 import com.beust.kobalt.misc.KobaltExecutors
 import com.google.common.collect.ArrayListMultimap
+import org.eclipse.aether.graph.DependencyFilter
+import org.eclipse.aether.util.filter.OrDependencyFilter
 import java.io.File
 import java.util.*
 import javax.inject.Inject
@@ -92,7 +95,8 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val
      * are passed, they are calculated from the scope filters.
      */
     override fun calculateDependencies(project: Project?, context: KobaltContext,
-            scopeFilters: Collection<Scope>,
+            dependencyFilter: DependencyFilter,
+            scopes: List<Scope>,
             vararg passedDependencies: List<IClasspathDependency>): List<IClasspathDependency> {
         val result = arrayListOf<IClasspathDependency>()
 
@@ -100,7 +104,7 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val
          * Extract the correct dependencies from the project based on the scope filters.
          */
         fun filtersToDependencies(project: Project, scopes: Collection<Scope>): List<IClasspathDependency> {
-            return arrayListOf<IClasspathDependency>().apply {
+            val result = arrayListOf<IClasspathDependency>().apply {
                 if (scopes.contains(Scope.COMPILE)) {
                     addAll(project.compileDependencies)
                 }
@@ -111,17 +115,18 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val
                     addAll(project.testDependencies)
                 }
             }
+            return result
         }
 
         val allDependencies : Array<out List<IClasspathDependency>> =
             if (project == null || passedDependencies.any()) passedDependencies
-            else arrayOf(filtersToDependencies(project, scopeFilters))
+            else arrayOf(filtersToDependencies(project, scopes))
 
         allDependencies.forEach { dependencies ->
-            result.addAll(transitiveClosure(dependencies, scopeFilters, project?.name))
+            result.addAll(transitiveClosure(dependencies, dependencyFilter, project?.name))
         }
         result.addAll(runClasspathContributors(project, context))
-        result.addAll(dependentProjectDependencies(project, context, scopeFilters))
+        result.addAll(dependentProjectDependencies(project, context, dependencyFilter, scopes.contains(Scope.TEST)))
 
         // Dependencies get reordered by transitiveClosure() but since we just added a bunch of new ones,
         // we need to reorder them again in case we're adding dependencies that are already present
@@ -144,13 +149,13 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val
      * TODO: This should be private, everyone should be calling calculateDependencies().
      */
     fun transitiveClosure(dependencies : List<IClasspathDependency>,
-            scopeFilter: Collection<Scope> = emptyList(),
+            dependencyFilter: DependencyFilter? = null,
             requiredBy: String? = null): List<IClasspathDependency> {
         val result = arrayListOf<IClasspathDependency>()
         dependencies.forEach {
             result.add(it)
             if (it.isMaven) {
-                val resolved = aether.resolveAll(it.id, null, scopeFilter).map { it.toString() }
+                val resolved = aether.resolveAll(it.id, null, dependencyFilter)
                 result.addAll(resolved.map { create(it) })
             }
         }
@@ -183,7 +188,7 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val
      * their own dependencies
      */
     private fun dependentProjectDependencies(project: Project?, context: KobaltContext,
-            scopeFilters: Collection<Scope>): List<IClasspathDependency> {
+            dependencyFilter: DependencyFilter, isTest: Boolean): List<IClasspathDependency> {
         if (project == null) {
             return emptyList()
         } else {
@@ -199,8 +204,8 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val
 
             project.dependsOn.forEach { p ->
                 maybeAddClassDir(KFiles.joinDir(p.directory, p.classesDir(context)))
-                if (scopeFilters.contains(Scope.TEST)) maybeAddClassDir(KFiles.makeOutputTestDir(project).path)
+                if (isTest) maybeAddClassDir(KFiles.makeOutputTestDir(project).path)
-                val otherDependencies = calculateDependencies(p, context, scopeFilters)
+                val otherDependencies = calculateDependencies(p, context, dependencyFilter, Scope.toScopes(isTest))
                 result.addAll(otherDependencies)
 
             }
@@ -225,8 +230,13 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val
                     deps.add(testProvidedDependencies)
                     scopeFilters.add(Scope.TEST)
                 }
+                val filter =
+                    if (isTest) OrDependencyFilter(Filters.COMPILE_FILTER, Filters.TEST_FILTER)
+                    else Filters.COMPILE_FILTER
                 deps.filter { it.any() }.forEach {
-                    transitive.addAll(calculateDependencies(project, context, scopeFilters, it))
+                    transitive.addAll(calculateDependencies(project, context, filter,
+                            scopes = Scope.toScopes(isTest),
+                            passedDependencies = it))
                 }
             }
         }

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/DependencyManager2.kt

@@ -9,6 +9,7 @@ import com.beust.kobalt.maven.dependency.FileDependency
 import com.beust.kobalt.misc.KFiles
 import com.google.common.collect.ArrayListMultimap
 import com.google.inject.Inject
+import org.eclipse.aether.graph.DependencyFilter
 import java.io.File
 import java.util.*
 
@@ -59,18 +60,18 @@ class DependencyManager2 @Inject constructor(val aether: KobaltAether) {
      * Resolve the dependencies for the give project based on the scope filters.
      */
     fun resolve(project: Project, context: KobaltContext, isTest: Boolean,
-            passedScopeFilters : List<Scope> = emptyList(),
+            passedScopes : List<Scope> = emptyList(),
             passedIds: List<IClasspathDependency> = emptyList()): List<IClasspathDependency> {
         val result = hashSetOf<IClasspathDependency>()
         val nonMavenDependencies = hashSetOf<IClasspathDependency>()
 
-        val scopeFilters =
+        val actualScopes =
-            if (passedScopeFilters.isEmpty())
+            if (passedScopes.isEmpty())
-                if (isTest) listOf(Scope.TEST)
+                if (isTest) listOf(Scope.TEST, Scope.COMPILE)
                 else listOf(Scope.COMPILE)
-            else passedScopeFilters
+            else passedScopes
 
-        val toDependencies = Scope.toDependencyLambda(scopeFilters)
+        val toDependencies = Scope.toDependencyLambda(actualScopes)
 
         // Make sure that classes/ and test-classes/ are always at the top of this classpath,
         // so that older versions of that project on the classpath don't shadow them
@@ -82,7 +83,7 @@ class DependencyManager2 @Inject constructor(val aether: KobaltAether) {
         // Passed and direct ids
         val ids = hashSetOf<IClasspathDependency>().apply {
             addAll(passedIds)
-            addAll(toDependencies(project))
+            addAll(toDependencies(project).filter { ! it.optional })
         }
 
         // Contributed id's
@@ -105,8 +106,9 @@ class DependencyManager2 @Inject constructor(val aether: KobaltAether) {
         var i = 0
         ids.forEach {
             if (it.isMaven) {
-                val resolved = aether.resolveAll(it.id, filterScopes = scopeFilters)
+                result.add(it)
-                        .map { create(it.toString(), false, project.directory) }
+                val resolved = aether.resolveAll(it.id, dependencyFilter = scopesToDependencyFilter(actualScopes))
+                        .map { create(it, false, project.directory) }
                 i++
                 result.addAll(resolved)
             } else {
@@ -119,6 +121,18 @@ class DependencyManager2 @Inject constructor(val aether: KobaltAether) {
         return reorderDependencies(result)
     }
 
+    private fun scopesToDependencyFilter(scopes: List<Scope>, includeOptional: Boolean = false): DependencyFilter {
+        return DependencyFilter { p0, p1 ->
+            if (p0.dependency.optional && ! includeOptional) return@DependencyFilter false
+
+            val result = scopes.any {
+                p0.dependency.scope == "" && scopes.contains(Scope.COMPILE) ||
+                        p0.dependency.scope == it.scope
+            }
+            result
+        }
+    }
+
     /**
      * Reorder dependencies so that if an artifact appears several times, only the one with the higest version
      * is included.

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/aether/Aether.kt

@@ -5,7 +5,6 @@ import com.beust.kobalt.api.IClasspathDependency
 import com.beust.kobalt.api.Kobalt
 import com.beust.kobalt.api.Project
 import com.beust.kobalt.internal.KobaltSettings
-import com.beust.kobalt.internal.KobaltSettingsXml
 import com.beust.kobalt.internal.getProxy
 import com.beust.kobalt.maven.CompletedFuture
 import com.beust.kobalt.maven.LocalDep
@@ -23,7 +22,6 @@ import org.eclipse.aether.collection.CollectRequest
 import org.eclipse.aether.collection.CollectResult
 import org.eclipse.aether.graph.Dependency
 import org.eclipse.aether.graph.DependencyFilter
-import org.eclipse.aether.graph.DependencyNode
 import org.eclipse.aether.repository.ArtifactRepository
 import org.eclipse.aether.repository.RemoteRepository
 import org.eclipse.aether.resolution.DependencyRequest
@@ -32,10 +30,7 @@ import org.eclipse.aether.resolution.VersionRangeRequest
 import org.eclipse.aether.resolution.VersionRangeResult
 import org.eclipse.aether.transfer.ArtifactNotFoundException
 import org.eclipse.aether.util.artifact.JavaScopes
-import org.eclipse.aether.util.filter.AndDependencyFilter
-import org.eclipse.aether.util.filter.DependencyFilterUtils
 import java.io.File
-import java.util.*
 import java.util.concurrent.Future
 
 enum class Scope(val scope: String, val dependencyLambda: (Project) -> List<IClasspathDependency>) {
@@ -47,26 +42,22 @@ enum class Scope(val scope: String, val dependencyLambda: (Project) -> List<ICla
     ;
 
     companion object {
-        /**
+        fun toScopes(isTest: Boolean) = if (isTest) listOf(Scope.TEST, Scope.COMPILE) else listOf(Scope.COMPILE)
-         * @return a filter that excludes optional dependencies and allows all the scopes passed in parameter.
-         */
-        fun toFilter(scopes: Collection<Scope>): DependencyFilter {
-            val javaScopes = scopes.map { DependencyFilterUtils.classpathFilter(it.scope) }.toTypedArray()
-            return AndDependencyFilter(KobaltAether.ExcludeOptionalDependencyFilter(), *javaScopes)
-        }
 
         /**
          * @return a lambda that extracts the correct dependencies from a project based on the scope
-         * filters passed.
+         * filters passed (excludes optional dependencies).
          */
         fun toDependencyLambda(scopes: Collection<Scope>) : (Project) -> List<IClasspathDependency> {
             val result = { project : Project ->
-                scopes.fold(arrayListOf<IClasspathDependency>(),
+                val deps = scopes.fold(arrayListOf<IClasspathDependency>(),
                     { list: ArrayList<IClasspathDependency>, scope: Scope ->
-                        list.addAll(scope.dependencyLambda(project))
+                        list.addAll(scope.dependencyLambda(project).filter { ! it.optional })
                         list
                     })
+                deps
             }
+
             return result
         }
     }
@@ -94,16 +85,16 @@ class KobaltAether @Inject constructor (val settings: KobaltSettings, val aether
         DependencyResult(AetherDependency(it.artifact), it.repository.toString())
     }
 
-    fun resolveAll(id: String, artifactScope: Scope? = null, filterScopes: Collection<Scope> = emptyList())
+    fun resolveAll(id: String, artifactScope: Scope? = null, dependencyFilter: DependencyFilter?)
             : List<String> {
-        val results = aether.resolve(DefaultArtifact(id), artifactScope, filterScopes)
+        val results = aether.resolve(DefaultArtifact(id), artifactScope, dependencyFilter)
         return results.map { it.artifact.toString() }
     }
 
-    fun resolve(id: String, artifactScope: Scope? = null, filterScopes: Collection<Scope> = emptyList())
+    fun resolve(id: String, artifactScope: Scope? = null, dependencyFilter: DependencyFilter = Filters.COMPILE_FILTER)
             : DependencyResult {
         kobaltLog(ConsoleRepositoryListener.LOG_LEVEL, "Resolving $id")
-        val result = resolveToArtifact(id, artifactScope, filterScopes)
+        val result = resolveToArtifact(id, artifactScope, dependencyFilter)
         if (result != null) {
             return DependencyResult(AetherDependency(result.artifact), result.repository.toString())
         } else {
@@ -111,34 +102,23 @@ class KobaltAether @Inject constructor (val settings: KobaltSettings, val aether
         }
     }
 
-    fun resolveToArtifact(id: String, artifactScope: Scope? = null, filterScopes: Collection<Scope> = emptyList())
+    fun resolveToArtifact(id: String, artifactScope: Scope? = null,
+            dependencyFilter: DependencyFilter? = null)
             : AetherResult? {
         kobaltLog(ConsoleRepositoryListener.LOG_LEVEL, "Resolving $id")
-        val results = aether.resolve(DefaultArtifact(MavenId.toKobaltId(id)), artifactScope, filterScopes)
+        val results = aether.resolve(DefaultArtifact(MavenId.toKobaltId(id)), artifactScope, dependencyFilter)
         if (results.size > 0) {
             return results[0]
         } else {
             return null
         }
     }
-
-    class ExcludeOptionalDependencyFilter : DependencyFilter {
-        override fun accept(node: DependencyNode?, p1: MutableList<DependencyNode>?): Boolean {
-//        val result = node != null && ! node.dependency.isOptional
-            val accept1 = node == null || node.artifact.artifactId != "srczip"
-            val accept2 = node != null && !node.dependency.isOptional
-            val result = accept1 && accept2
-            return result
-        }
-    }
 }
 
 @Singleton
 class Aether(localRepo: File, val settings: KobaltSettings, eventBus: EventBus) {
     private val system = Booter.newRepositorySystem()
     private val session = Booter.newRepositorySystemSession(system, localRepo, settings, eventBus)
-//    private val classpathFilter = Scopes.toFilter(Scopes.COMPILE, Scopes.TEST)
-//    private val testClasspathFilter = Scopes.toFilter(Scopes.TEST)
 
     private val kobaltRepositories: List<RemoteRepository>
         get() = Kobalt.repos.map {
@@ -170,7 +150,7 @@ class Aether(localRepo: File, val settings: KobaltSettings, eventBus: EventBus)
         if (resolved != null) {
             val newArtifact = DefaultArtifact(artifact.groupId, artifact.artifactId, artifact.extension,
                     resolved.highestVersion.toString())
-            val artifactResult = resolve(newArtifact, null, emptyList())
+            val artifactResult = resolve(newArtifact, null)
             if (artifactResult.any()) {
                 return artifactResult[0]
             } else {
@@ -187,7 +167,9 @@ class Aether(localRepo: File, val settings: KobaltSettings, eventBus: EventBus)
         return result
     }
 
-    fun resolve(artifact: Artifact, artifactScope: Scope?, filterScopes: Collection<Scope>): List<AetherResult> {
+    fun resolve(artifact: Artifact, artifactScope: Scope?,
+            dependencyFilter: DependencyFilter? = null)
+            : List<AetherResult> {
         fun manageException(ex: Exception, artifact: Artifact): List<AetherResult> {
             if (artifact.extension == "pom") {
                 // Only display a warning for .pom files. Not resolving a .jar or other artifact
@@ -198,7 +180,6 @@ class Aether(localRepo: File, val settings: KobaltSettings, eventBus: EventBus)
         }
 
         try {
-            val scopeFilter = Scope.toFilter(filterScopes)
             val result =
                 if (KobaltAether.isRangeVersion(artifact.version)) {
                     val request = rangeRequest(artifact)
@@ -212,7 +193,8 @@ class Aether(localRepo: File, val settings: KobaltSettings, eventBus: EventBus)
                         throw KobaltException("Couldn't resolve range artifact " + artifact)
                     }
                 } else {
-                    val dependencyRequest = DependencyRequest(collectRequest(artifact, artifactScope), scopeFilter)
+                    val dependencyRequest = DependencyRequest(collectRequest(artifact, artifactScope), dependencyFilter)
+
                     try {
                         system.resolveDependencies(session, dependencyRequest).artifactResults.map {
                             AetherResult(it.artifact, it.repository)
@@ -256,7 +238,7 @@ class AetherDependency(val artifact: Artifact, override val optional: Boolean =
             if (file.exists()) {
                 CompletedFuture(file)
             } else {
-                val td = aether.resolve(artifact, null, emptyList())
+                val td = aether.resolve(artifact, null)
                 if (td.any()) {
                     val newFile = td[0].artifact.file
                     if (newFile != null) {
@@ -318,21 +300,38 @@ class AetherDependency(val artifact: Artifact, override val optional: Boolean =
     override fun toString() = id
 }
 
-fun main(argv: Array<String>) {
+fun f(argv: Array<String>) {
-    val request = CollectRequest().apply {
+    val collectRequest = CollectRequest().apply {
-        root = Dependency(DefaultArtifact("org.testng:testng:6.9.11"), JavaScopes.COMPILE)
+        root = Dependency(DefaultArtifact("com.squareup.retrofit2:converter-jackson:jar:2.1.0"), JavaScopes.COMPILE)
         repositories = listOf(
-                RemoteRepository.Builder("Maven", "default", "http://repo1.maven.org/maven2/").build(),
+//                RemoteRepository.Builder("Maven", "default", "http://repo1.maven.org/maven2/").build()
-                RemoteRepository.Builder("JCenter", "default", "https://jcenter.bintray.com").build())
+                RemoteRepository.Builder("JCenter", "default", "https://jcenter.bintray.com").build()
-    }
+        )
-    val dependencyRequest = DependencyRequest().apply {
-        collectRequest = request
     }
+//    val dependencyRequest = DependencyRequest().apply {
+//        collectRequest = request
+//        filter = object: DependencyFilter {
+//            override fun accept(p0: DependencyNode, p1: MutableList<DependencyNode>?): Boolean {
+//                if (p0.artifact.artifactId.contains("android")) {
+//                    println("ANDROID")
+//                }
+//                return p0.dependency.scope == JavaScopes.COMPILE
+//            }
+//
+//        }
+//    }
+    val dr2 = DependencyRequest(collectRequest, null).apply {}
+
+
+//    val system = ManualRepositorySystemFactory.newRepositorySystem()
+//    val session = DefaultRepositorySystemSession()
+//    val localRepo = LocalRepository(File("/Users/cedricbeust/t/localAether").absolutePath)
+//    session.localRepositoryManager = system.newLocalRepositoryManager(session, localRepo)
+
     val system = Booter.newRepositorySystem()
-    val session = Booter.newRepositorySystemSession(system, File("/tmp"), KobaltSettings(KobaltSettingsXml()),
+    val session = Booter.newRepositorySystemSession(system)
-            EventBus())
+
-//    val session = MavenRepositorySystemUtils.newSession(KobaltSettings(KobaltSettingsXml()))
+    val result = system.resolveDependencies(session, dr2).artifactResults
-    val result = system.resolveDependencies(session, dependencyRequest).artifactResults
     println("RESULT: " + result)
 
 //    KobaltLogger.LOG_LEVEL = 1
@@ -346,4 +345,36 @@ fun main(argv: Array<String>) {
 //    println("Artifact: " + d)
 }
 
+fun f2() {
+    val system = Booter.newRepositorySystem()
+
+    val session = Booter.newRepositorySystemSession(system)
+
+    val artifact = DefaultArtifact("com.squareup.retrofit2:converter-jackson:jar:2.1.0")
+
+//        DependencyFilter classpathFlter = DependencyFilterUtils.classpathFilter( JavaScopes.COMPILE );
+    val f2 = DependencyFilter { dependencyNode, list ->
+        println("ACCEPTING " + dependencyNode)
+        true
+    }
+
+    val collectRequest = CollectRequest()
+    collectRequest.root = Dependency(artifact, JavaScopes.COMPILE)
+    collectRequest.repositories = listOf(
+        RemoteRepository.Builder("Maven", "default", "http://repo1.maven.org/maven2/").build()
+    )
+
+    val dependencyRequest = DependencyRequest(collectRequest, null)
+
+    val artifactResults = system.resolveDependencies(session, dependencyRequest).artifactResults
+
+    for (artifactResult in artifactResults) {
+        println(artifactResult.artifact.toString() + " resolved to " + artifactResult.artifact.file)
+    }
+}
+
+
+fun main(args: Array<String>) {
+    f2()
+}
 

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/aether/Booter.kt

@@ -16,6 +16,21 @@ object Booter {
         // return org.eclipse.aether.examples.plexus.PlexusRepositorySystemFactory.newRepositorySystem();
     }
 
+    fun newRepositorySystemSession(system: RepositorySystem): DefaultRepositorySystemSession {
+        val session = org.apache.maven.repository.internal.MavenRepositorySystemUtils.newSession()
+
+        val localRepo = LocalRepository("target/local-repo")
+        session.localRepositoryManager = system.newLocalRepositoryManager(session, localRepo)
+
+        session.transferListener = ConsoleTransferListener()
+        session.repositoryListener = ConsoleRepositoryListener(System.out, EventBus())
+
+        // uncomment to generate dirty trees
+        // session.setDependencyGraphTransformer( null );
+
+        return session
+    }
+
     fun newRepositorySystemSession(system: RepositorySystem, repo: File, settings: KobaltSettings,
             eventBus: EventBus): DefaultRepositorySystemSession {
         val session = MavenRepositorySystemUtils.newSession(settings)

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/aether/Filters.kt

@@ -0,0 +1,15 @@
+package com.beust.kobalt.maven.aether
+
+import org.eclipse.aether.graph.DependencyFilter
+import org.eclipse.aether.util.artifact.JavaScopes
+
+object Filters {
+    val COMPILE_FILTER = DependencyFilter { p0, p1 ->
+        p0.dependency.scope == "" || p0.dependency.scope == JavaScopes.COMPILE
+    }
+    val TEST_FILTER = DependencyFilter { p0, p1 -> p0.dependency.scope == JavaScopes.TEST }
+
+    val EXCLUDE_OPTIONAL_FILTER = DependencyFilter { p0, p1 ->
+        ! p0.dependency.optional
+    }
+}

src/main/kotlin/com/beust/kobalt/app/BuildFileCompiler.kt

@@ -46,7 +46,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil
 
     private val SCRIPT_JAR = "buildScript.jar"
 
-    fun compileBuildFiles(args: Args): FindProjectResult {
+    fun compileBuildFiles(args: Args, forceRecompile: Boolean = false): FindProjectResult {
         //
         // Create the KobaltContext
         // Note: can't use apply{} here or each field will refer to itself instead of the constructor field
@@ -66,7 +66,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil
         //
         // Find all the projects in the build file, possibly compiling them
         //
-        val projectResult = findProjects(context)
+        val projectResult = findProjects(context, forceRecompile)
 
         return projectResult
     }
@@ -76,7 +76,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil
     class FindProjectResult(val context: KobaltContext, val projects: List<Project>, val pluginUrls: List<URL>,
             val taskResult: TaskResult)
 
-    private fun findProjects(context: KobaltContext): FindProjectResult {
+    private fun findProjects(context: KobaltContext, forceRecompile: Boolean): FindProjectResult {
         var errorTaskResult: TaskResult? = null
         val projects = arrayListOf<Project>()
         buildFiles.forEach { buildFile ->
@@ -105,7 +105,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil
             KFiles.saveFile(modifiedBuildFile, parsedBuildFile.buildScriptCode)
             val taskResult = maybeCompileBuildFile(context, BuildFile(Paths.get(modifiedBuildFile.path),
                     "Modified ${Constants.BUILD_FILE_NAME}", buildFile.realPath),
-                    buildScriptJarFile, pluginUrls)
+                    buildScriptJarFile, pluginUrls, forceRecompile)
             if (taskResult.success) {
                 projects.addAll(buildScriptUtil.runBuildScriptJarFile(buildScriptJarFile, pluginUrls, context))
             } else {
@@ -124,7 +124,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil
     }
 
     private fun maybeCompileBuildFile(context: KobaltContext, buildFile: BuildFile, buildScriptJarFile: File,
-            pluginUrls: List<URL>) : TaskResult {
+            pluginUrls: List<URL>, forceRecompile: Boolean) : TaskResult {
         kobaltLog(2, "Running build file ${buildFile.name} jar: $buildScriptJarFile")
 
         // If the user specifed --profiles, always recompile the build file since we don't know if
@@ -135,8 +135,8 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil
         // compiled with.
         val bs = BuildScriptJarFile(buildScriptJarFile)
         val same = bs.sameProfiles(args.profiles)
-        if (same && buildScriptUtil.isUpToDate(buildFile, buildScriptJarFile)) {
+        if (same && ! forceRecompile && buildScriptUtil.isUpToDate(buildFile, buildScriptJarFile)) {
-            kobaltLog(2, "  Build file is up to date")
+            kobaltLog(2, "  Build file $buildScriptJarFile is up to date")
             return TaskResult()
         } else {
             kobaltLog(2, "  Need to recompile ${buildFile.name}")

src/main/kotlin/com/beust/kobalt/app/BuildScriptUtil.kt

@@ -13,6 +13,7 @@ import com.beust.kobalt.internal.build.BuildFile
 import com.beust.kobalt.misc.KFiles
 import com.beust.kobalt.misc.Topological
 import com.beust.kobalt.misc.kobaltLog
+import com.beust.kobalt.misc.warn
 import com.beust.kobalt.plugin.KobaltPlugin
 import com.google.inject.Inject
 import java.io.File
@@ -63,12 +64,16 @@ class BuildScriptUtil @Inject constructor(val plugins: Plugins, val files: KFile
                 val name = entry.name;
                 if (name.endsWith(".class")) {
                     val className = name.substring(0, name.length - 6).replace("/", ".")
+                    try {
                         val cl: Class<*>? = classLoader.loadClass(className)
                         if (cl != null) {
                             classes.add(cl)
                         } else {
                             throw KobaltException("Couldn't instantiate $className")
                         }
+                    } catch(ex: ClassNotFoundException) {
+                        warn("Couldn't find class $className")
+                    }
                 }
                 entry = stream.nextJarEntry;
             }

src/main/kotlin/com/beust/kobalt/app/ParsedBuildFile.kt

@@ -162,8 +162,8 @@ class ParsedBuildFile(val buildFile: BuildFile, val context: KobaltContext, val
         //
         // Compile the jar file
         //
-        val kotlintDeps = dependencyManager.calculateDependencies(null, context)
+        val kotlinDeps = dependencyManager.calculateDependencies(null, context)
-        val deps: List<String> = kotlintDeps.map { it.jarFile.get().absolutePath }
+        val deps: List<String> = kotlinDeps.map { it.jarFile.get().absolutePath }
         val outputJar = File(buildScriptJarFile.absolutePath)
         val result = kotlinCompilePrivate {
             classpath(files.kobaltJar)

src/main/kotlin/com/beust/kobalt/plugin/application/ApplicationPlugin.kt

@@ -112,7 +112,7 @@ class ApplicationPlugin @Inject constructor(val configActor: ConfigActor<Applica
             val allDependencies = project.compileDependencies + project.compileRuntimeDependencies
             val allTheDependencies =
                     dependencyManager.calculateDependencies(project, context,
-                            listOf(Scope.COMPILE, Scope.RUNTIME),
+                            scopes = listOf(Scope.COMPILE, Scope.RUNTIME),
                             passedDependencies = allDependencies)
                             .map { it.jarFile.get().path }
             allDeps.addAll(allTheDependencies)

src/test/kotlin/com/beust/kobalt/maven/DependencyManagerTest.kt

@@ -6,9 +6,11 @@ import com.beust.kobalt.TestModule
 import com.beust.kobalt.api.IClasspathDependency
 import com.beust.kobalt.api.Kobalt
 import com.beust.kobalt.app.BuildFileCompiler
+import com.beust.kobalt.maven.aether.Filters
 import com.beust.kobalt.maven.aether.Scope
 import com.google.inject.Inject
 import org.assertj.core.api.Assertions.assertThat
+import org.eclipse.aether.util.filter.AndDependencyFilter
 import org.testng.annotations.Guice
 import org.testng.annotations.Test
 
@@ -19,7 +21,17 @@ class DependencyManagerTest @Inject constructor(val dependencyManager: Dependenc
 
     private fun assertContains(dependencies: List<IClasspathDependency>, vararg ids: String) {
         ids.forEach { id ->
-            assertThat(dependencies.any { it.id.contains(id) }).isTrue()
+            if (! dependencies.any { it.id.contains(id) }) {
+                throw AssertionError("Couldn't find $id in $dependencies")
+            }
+        }
+    }
+
+    private fun assertDoesNotContain(dependencies: List<IClasspathDependency>, vararg ids: String) {
+        ids.forEach { id ->
+            if (dependencies.any { it.id.contains(id) }) {
+                throw AssertionError("$id should not be found in $dependencies")
+            }
         }
     }
 
@@ -27,87 +39,118 @@ class DependencyManagerTest @Inject constructor(val dependencyManager: Dependenc
     fun testScopeDependenciesShouldBeDownloaded() {
         val testDeps = listOf(dependencyManager.create("org.testng:testng:6.9.11"))
 
+        val filter = AndDependencyFilter(Filters.EXCLUDE_OPTIONAL_FILTER, Filters.COMPILE_FILTER)
+
         // Should only resolve to TestNG
-        dependencyManager.transitiveClosure(testDeps, listOf(Scope.COMPILE)).let { dependencies ->
+        dependencyManager.transitiveClosure(testDeps, filter).let { dependencies ->
             assertThat(dependencies.any { it.id.contains(":jcommander:") }).isFalse()
             assertContains(dependencies, ":testng:")
         }
 
         // Should resolve to TestNG and its dependencies
-        dependencyManager.transitiveClosure(testDeps, listOf(Scope.TEST)).let { dependencies ->
+        dependencyManager.transitiveClosure(testDeps).let { dependencies ->
             assertContains(dependencies, ":jcommander:")
             assertContains(dependencies, ":bsh:")
             assertContains(dependencies, ":ant:")
             assertContains(dependencies, ":ant-launcher:")
             assertContains(dependencies, ":testng:")
         }
-
     }
 
     @Test
     fun honorRuntimeDependenciesBetweenProjects() {
+        Kobalt.context = null
         val buildFileString = """
             import com.beust.kobalt.*
 
-            val lib = project {
+            val lib1 = project {
-                name = "lib"
+                name = "lib1"
                 dependencies {
-                    compile("org.testng:testng:6.9.11")
+                    compile("com.beust:klaxon:0.26",
-                    runtime("com.beust:jcommander:1.48")
+                        "com.beust:jcommander:1.48")
                 }
             }
 
-            val p = project(lib) {
+            val p = project(lib1) {
-                name = "transitive"
+                name = "transitive1"
             }
         """
 
-        val compileResult = compileBuildFile(buildFileString, Args(), compilerFactory)
+        val compileResult = compileBuildFile(sharedBuildFile, Args(), compilerFactory)
         val project2 = compileResult.projects[1]
-        val dependencies = dependencyManager.calculateDependencies(project2, Kobalt.context!!,
+        val dependencies = dependencyManager.calculateDependencies(project2, Kobalt.context!!, Filters.COMPILE_FILTER)
-                listOf(Scope.COMPILE, Scope.RUNTIME))
+        assertContains(dependencies, ":klaxon:")
-        assertContains(dependencies, ":testng:")
+        assertContains(dependencies, ":guice:")
-        assertContains(dependencies, ":jcommander:")
+        assertDoesNotContain(dependencies, ":guave:")
     }
 
+    val sharedBuildFile = """
+            import com.beust.kobalt.*
+
+            val lib2 = project {
+                name = "lib2"
+                dependencies {
+                    // pick dependencies that don't have dependencies themselves, to avoid interferences
+                    compile("com.beust:klaxon:0.27",
+                        "com.google.inject:guice:4.0")
+                    runtime("com.beust:jcommander:1.48")
+                    compileOptional("junit:junit:4.12")
+                }
+            }
+
+            val p = project(lib2) {
+                name = "transitive2"
+            }
+        """
+
     @Test
     fun honorRuntimeDependenciesBetweenProjects2() {
         val buildFileString = """
             import com.beust.kobalt.*
 
-            val lib = project {
+            val lib2 = project {
-                name = "lib"
+                name = "lib2"
                 dependencies {
-                    compile("org.testng:testng:6.9.11")
+                    // pick dependencies that don't have dependencies themselves, to avoid interferences
+                    compile("com.beust:klaxon:0.27",
+                        "com.google.inject:guice:4.0)
                     runtime("com.beust:jcommander:1.48")
                 }
             }
 
-            val p = project(lib) {
+            val p = project(lib2) {
-                name = "transitive"
+                name = "transitive2"
             }
         """
 
-        val compileResult = compileBuildFile(buildFileString, Args(), compilerFactory)
+        val compileResult = compileBuildFile(sharedBuildFile, Args(), compilerFactory)
         val project2 = compileResult.projects[1]
 
-        dependencyManager2.resolve(project2, Kobalt.context!!, isTest = false,
+        Kobalt.context!!.let { context ->
-                passedScopeFilters = listOf(Scope.COMPILE, Scope.RUNTIME)).let { dependencies ->
+            dependencyManager2.resolve(project2, context, isTest = false,
-            assertThat(dependencies.size).isEqualTo(4)
+                    passedScopes = listOf(Scope.COMPILE)).let { dependencies ->
-            assertContains(dependencies, ":testng:")
+                assertContains(dependencies, ":klaxon:jar:0.27")
+                assertContains(dependencies, ":guice:")
+                assertDoesNotContain(dependencies, ":jcommander:")
+                assertDoesNotContain(dependencies, ":junit:")
+            }
+
+            dependencyManager2.resolve(project2, context, isTest = false,
+                    passedScopes = listOf(Scope.RUNTIME)).let { dependencies ->
                 assertContains(dependencies, ":jcommander:")
+                assertDoesNotContain(dependencies, ":klaxon:jar:0.27")
+                assertDoesNotContain(dependencies, ":guice:")
+                assertDoesNotContain(dependencies, ":junit:")
             }
 
-        dependencyManager2.resolve(project2, Kobalt.context!!, isTest = false,
+            dependencyManager2.resolve(project2, context, isTest = false,
-                passedScopeFilters = listOf(Scope.COMPILE)).let { dependencies ->
+                    passedScopes = listOf(Scope.COMPILE, Scope.RUNTIME)).let { dependencies ->
-            assertThat(dependencies.size).isEqualTo(3)
+                assertContains(dependencies, ":klaxon:")
-            assertContains(dependencies, ":testng:")
-        }
-
-        dependencyManager2.resolve(project2, Kobalt.context!!, isTest = false,
-                passedScopeFilters = listOf(Scope.RUNTIME)).let { dependencies ->
-            assertThat(dependencies.size).isEqualTo(3)
                 assertContains(dependencies, ":jcommander:")
+                assertContains(dependencies, ":guice:")
+                assertDoesNotContain(dependencies, ":junit:")
+            }
+
         }
 
     }

src/test/kotlin/com/beust/kobalt/maven/DownloadTest.kt

@@ -147,7 +147,7 @@ class DownloadTest @Inject constructor(
 
     @Test
     fun parentPomTest() {
-        // Resolve com.squareup.retrofit2:converter-moshi:2.0.0
+        // Resolve com.squareup.retrofit2:converter-moshi:1.1.0
         // This id has a parent pom which defines moshi version to be 1.1.0. Make sure that this
         // version is being fetched instead of moshi:1.2.0-SNAPSHOT (which gets discarded anyway
         // since snapshots are not allowed to be returned when looking up a versionless id)

src/test/kotlin/com/beust/kobalt/misc/AetherTest.kt

@@ -0,0 +1,55 @@
+package com.beust.kobalt.misc
+
+import com.beust.kobalt.TestModule
+import com.beust.kobalt.maven.DependencyManager
+import com.beust.kobalt.maven.aether.Booter
+import com.beust.kobalt.maven.aether.KobaltAether
+import com.google.inject.Inject
+import org.assertj.core.api.Assertions.assertThat
+import org.eclipse.aether.artifact.DefaultArtifact
+import org.eclipse.aether.collection.CollectRequest
+import org.eclipse.aether.graph.Dependency
+import org.eclipse.aether.repository.RemoteRepository
+import org.eclipse.aether.resolution.DependencyRequest
+import org.eclipse.aether.util.artifact.JavaScopes
+import org.testng.annotations.Guice
+import org.testng.annotations.Test
+
+@Guice(modules = arrayOf(TestModule::class))
+class AetherTest {
+    @Inject
+    lateinit var kobaltAether: KobaltAether
+
+    @Inject
+    lateinit var dependencyManager: DependencyManager
+
+    @Test
+    fun aetherShouldNotIncludeOptionalDependencies() {
+        val system = Booter.newRepositorySystem()
+        val session = Booter.newRepositorySystemSession(system)
+        val artifact = DefaultArtifact("com.squareup.retrofit2:converter-jackson:jar:2.1.0")
+
+        val collectRequest = CollectRequest().apply {
+            root = Dependency(artifact, JavaScopes.COMPILE)
+            repositories = listOf(
+                    RemoteRepository.Builder("Maven", "default", "http://repo1.maven.org/maven2/").build()
+            )
+        }
+
+        val dependencyRequest = DependencyRequest(collectRequest, null)
+
+        val artifactResults = system.resolveDependencies(session, dependencyRequest).artifactResults
+
+        // Make sure that com.google.android is not included (it's an optional dependency of retrofit2)
+        assertThat(artifactResults.none { it.toString().contains("android") })
+    }
+
+    @Test
+    fun kobaltAetherShouldNotIncludeOptionalDependencies() {
+        val dep = kobaltAether.create("com.squareup.retrofit2:converter-jackson:jar:2.1.0", optional = false)
+        val closure = dependencyManager.transitiveClosure(listOf(dep))
+
+        // Make sure that com.google.android is not included (it's an optional dependency of retrofit2)
+        assertThat(closure.none { it.toString().contains("android") })
+    }
+}