Better classpath calculations.

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/CompilerUtils.kt

@@ -3,6 +3,8 @@ package com.beust.kobalt.internal
 import com.beust.kobalt.TaskResult
 import com.beust.kobalt.api.*
 import com.beust.kobalt.maven.DependencyManager
+import com.beust.kobalt.maven.DependencyManager2
+import com.beust.kobalt.maven.aether.Scope
 import com.beust.kobalt.maven.dependency.FileDependency
 import com.beust.kobalt.misc.KFiles
 import com.beust.kobalt.misc.log
@@ -15,7 +17,8 @@ import java.util.*
  * Central place to compile files, used by plug-ins and non plug-ins.
  */
 class CompilerUtils @Inject constructor(val files: KFiles,
-        val dependencyManager: DependencyManager) {
+        val dependencyManager: DependencyManager,
+        val dependencyManager2: DependencyManager2) {
 
     class CompilerResult(val successResults: List<TaskResult>, val failedResult: TaskResult?)
 
@@ -68,8 +71,17 @@ class CompilerUtils @Inject constructor(val files: KFiles,
             isTest: Boolean, sourceDirectories: List<File>, sourceSuffixes: List<String>): CompilerActionInfo {
         copyResources(project, context, SourceSet.of(isTest))
 
-        val fullClasspath = if (isTest) dependencyManager.testDependencies(project, context)
+        val fullClasspath = dependencyManager2.resolve(project, context, isTest, listOf(Scope.COMPILE, Scope.TEST))
-            else dependencyManager.dependencies(project, context)
+
+//        if (project.name == "ktor-core" && isTest) {
+//            val contains = fullClasspath.filter{it.id.contains("json")}
+//            val fc1 =
+//                if (isTest) dependencyManager.testDependencies(project, context)
+//                else dependencyManager.dependencies(project, context)
+//
+//            println("DONOTCOMMIT")
+//            val d2 = dependencyManager2.resolve(project, context, isTest, listOf(Scope.COMPILE, Scope.TEST))
+//        }
 
         // The directory where the classes get compiled
         val buildDirectory = if (isTest) File(project.buildDirectory, KFiles.TEST_CLASSES_DIR)

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/DynamicGraph.kt

@@ -30,28 +30,32 @@ class DynamicGraph<T> {
     private val dependedUpon = HashMultimap.create<Node<T>, Node<T>>()
     private val dependingOn = HashMultimap.create<Node<T>, Node<T>>()
 
-    fun transitiveClosure(root: T): Set<T> {
+    companion object {
-        val result = hashSetOf<T>()
+        fun <T> transitiveClosure(root: T, childrenFor: (T) -> List<T>) : List<T> {
-        val seen = hashSetOf<T>()
+            val result = arrayListOf<T>()
-        val toProcess = arrayListOf<T>().apply {
+            val seen = hashSetOf<T>()
-            add(root)
+            val toProcess = arrayListOf<T>().apply {
-        }
+                add(root)
-        while (toProcess.any()) {
-            val newToProcess = arrayListOf<T>()
-            toProcess.forEach {
-                if (! seen.contains(it)) {
-                    result.add(it)
-                    newToProcess.addAll(dependedUpon[Node(it)].map { it.value })
-                    seen.add(it)
-                }
             }
-            toProcess.clear()
+            while (toProcess.any()) {
-            toProcess.addAll(newToProcess)
+                val newToProcess = arrayListOf<T>()
+                toProcess.forEach {
+                    if (! seen.contains(it)) {
+                        result.add(it)
+                        newToProcess.addAll(childrenFor(it))
+                        seen.add(it)
+                    }
+                }
+                toProcess.clear()
+                toProcess.addAll(newToProcess)
+            }
+            return result
         }
-
-        return result
     }
 
+    fun transitiveClosure(root: T)
+            = transitiveClosure(root) { element -> dependedUpon[Node(element)].map { it.value } }
+
     fun addNode(t: T) = synchronized(nodes) {
         nodes.add(Node(t))
     }

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/GenericRunner.kt

@@ -2,6 +2,7 @@ package com.beust.kobalt.internal
 
 import com.beust.kobalt.*
 import com.beust.kobalt.api.*
+import com.beust.kobalt.maven.DependencyManager2
 import com.beust.kobalt.misc.KFiles
 import com.beust.kobalt.misc.log
 import com.google.common.annotations.VisibleForTesting
@@ -23,9 +24,13 @@ abstract class GenericTestRunner: ITestRunnerContributor {
             classpath: List<IClasspathDependency>)
         = TaskResult(runTests(project, context, classpath, configName))
 
-    override fun affinity(project: Project, context: KobaltContext) =
+    override fun affinity(project: Project, context: KobaltContext) : Int {
-            if (project.testDependencies.any { it.id.contains(dependencyName)}) IAffinity.DEFAULT_POSITIVE_AFFINITY
+        val td = Kobalt.INJECTOR.getInstance(DependencyManager2::class.java).resolve(project, context, isTest = true)
+        val result =
+            if (td.any { it.id.contains(dependencyName) }) IAffinity.DEFAULT_POSITIVE_AFFINITY
             else 0
+        return result
+    }
 
     protected fun findTestClasses(project: Project, context: KobaltContext, testConfig: TestConfig): List<String> {
         val testClassDir = KFiles.joinDir(project.buildDirectory, KFiles.TEST_CLASSES_DIR)

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/JvmCompilerPlugin.kt

@@ -9,6 +9,7 @@ import com.beust.kobalt.api.annotation.ExportedProjectProperty
 import com.beust.kobalt.api.annotation.IncrementalTask
 import com.beust.kobalt.api.annotation.Task
 import com.beust.kobalt.maven.DependencyManager
+import com.beust.kobalt.maven.DependencyManager2
 import com.beust.kobalt.maven.LocalRepo
 import com.beust.kobalt.maven.Md5
 import com.beust.kobalt.misc.*
@@ -26,6 +27,7 @@ open class JvmCompilerPlugin @Inject constructor(
         open val localRepo: LocalRepo,
         open val files: KFiles,
         open val dependencyManager: DependencyManager,
+        open val dependencyManager2: DependencyManager2,
         open val executors: KobaltExecutors,
         open val taskContributor : TaskContributor,
         val compilerUtils: CompilerUtils)
@@ -89,8 +91,10 @@ open class JvmCompilerPlugin @Inject constructor(
         val testContributor = ActorUtils.selectAffinityActor(project, context,
                 context.pluginInfo.testRunnerContributors)
         if (testContributor != null && testContributor.affinity(project, context) > 0) {
-            return testContributor.run(project, context, configName,
+//            val td1 = dependencyManager.testDependencies(project, context)
-                    dependencyManager.testDependencies(project, context))
+            val testDependencies = dependencyManager2.resolve(project, context, isTest = true)
+            val compileDependencies = dependencyManager2.resolve(project, context, isTest = false)
+            return testContributor.run(project, context, configName, testDependencies + compileDependencies)
         } else {
             log(2, "Couldn't find a test runner for project ${project.name}, did you specify dependenciesTest{}?")
             return TaskResult()

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/DependencyManager.kt

@@ -177,8 +177,8 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val
      * If this project depends on other projects, we need to include their jar file and also
      * their own dependencies
      */
-    private fun dependentProjectDependencies(project: Project?, context: KobaltContext, scopeFilters: Collection<Scope>)
+    private fun dependentProjectDependencies(project: Project?, context: KobaltContext,
-            : List<IClasspathDependency> {
+            scopeFilters: Collection<Scope>): List<IClasspathDependency> {
         if (project == null) {
             return emptyList()
         } else {

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/DependencyManager2.kt

@@ -0,0 +1,159 @@
+package com.beust.kobalt.maven
+
+import com.beust.kobalt.KobaltException
+import com.beust.kobalt.api.*
+import com.beust.kobalt.internal.DynamicGraph
+import com.beust.kobalt.maven.aether.KobaltAether
+import com.beust.kobalt.maven.aether.Scope
+import com.beust.kobalt.maven.dependency.FileDependency
+import com.beust.kobalt.misc.KFiles
+import com.google.inject.Inject
+import java.io.File
+
+class DependencyManager2 @Inject constructor(val aether: KobaltAether) {
+    /**
+     * Create an IClasspathDependency from a Maven id.
+     */
+    fun createMaven(id: String) : IClasspathDependency = aether.create(id)
+
+    /**
+     * Create an IClasspathDependency from a path.
+     */
+    fun createFile(path: String) : IClasspathDependency = FileDependency(path)
+
+    /**
+     * Parse the id and return the correct IClasspathDependency
+     */
+    fun create(id: String, projectDirectory: String?) : IClasspathDependency {
+        if (id.startsWith(FileDependency.PREFIX_FILE)) {
+            val path = if (projectDirectory != null) {
+                val idPath = id.substring(FileDependency.PREFIX_FILE.length)
+                if (! File(idPath).isAbsolute) {
+                    // If the project directory is relative, we might not be in the correct directory to locate
+                    // that file, so we'll use the absolute directory deduced from the build file path. Pick
+                    // the first one that produces an actual file
+                    val result = listOf(File(projectDirectory), Kobalt.context?.internalContext?.absoluteDir).map {
+                        File(it, idPath)
+                    }.firstOrNull {
+                        it.exists()
+                    }
+                    result ?: throw KobaltException("Couldn't find $id")
+
+                } else {
+                    File(idPath)
+                }
+            } else {
+                File(id.substring(FileDependency.PREFIX_FILE.length))
+            }
+            return createFile(path.path)
+        } else {
+            // Convert to a Kobalt id first (so that if it doesn't have a version, it gets translated to
+            // an Aether ranged id "[0,)")
+            return createMaven(MavenId.create(id).toId)
+        }
+    }
+
+    /**
+     * Resolve the dependencies for the give project based on the scope filters.
+     */
+    fun resolve(project: Project, context: KobaltContext, isTest: Boolean,
+            passedScopeFilters : List<Scope> = emptyList(),
+            passedIds: List<IClasspathDependency> = emptyList()): List<IClasspathDependency> {
+        val result = hashSetOf<IClasspathDependency>()
+        val nonMavenDependencies = hashSetOf<IClasspathDependency>()
+
+        val scopeFilters =
+            if (passedScopeFilters.isEmpty())
+                if (isTest) listOf(Scope.TEST)
+                else listOf(Scope.COMPILE)
+            else passedScopeFilters
+
+        val toDependencies = Scope.toDependencyLambda(scopeFilters)
+
+        // Make sure that classes/ and test-classes/ are always at the top of this classpath,
+        // so that older versions of that project on the classpath don't shadow them
+        if (isTest) {
+            result.add(FileDependency(KFiles.makeOutputDir(project).path))
+            result.add(FileDependency(KFiles.makeOutputTestDir(project).path))
+        }
+
+        // Passed and direct ids
+        val ids = hashSetOf<IClasspathDependency>().apply {
+            addAll(passedIds)
+            addAll(toDependencies(project))
+        }
+
+        // Contributed id's
+        val contributedIds = runClasspathContributors(project, context)
+        contributedIds.forEach {
+            if (it.isMaven) ids.add(it)
+            else nonMavenDependencies.add(it)
+        }
+
+        // Dependent project id's
+        val dependentIds = dependentProjectDependencies(project, context, toDependencies)
+        dependentIds.forEach {
+            if (it.isMaven) ids.add(it)
+            else nonMavenDependencies.add(it)
+        }
+
+        //
+        // Now we have all the id's, resolve them
+        //
+        var i = 0
+        ids.forEach {
+            val resolved = aether.resolveAll(it.id, filterScopes = scopeFilters)
+                .map { create(it.toString(), project.directory) }
+            i++
+            result.addAll(resolved)
+        }
+
+        result.addAll(nonMavenDependencies)
+
+        return result.toList()
+    }
+
+    private fun runClasspathContributors(project: Project?, context: KobaltContext) :
+            Set<IClasspathDependency> {
+        val result = hashSetOf<IClasspathDependency>()
+        context.pluginInfo.classpathContributors.forEach { it: IClasspathContributor ->
+            result.addAll(it.classpathEntriesFor(project, context))
+        }
+        return result
+    }
+
+    /**
+     * If this project depends on other projects, we need to include their jar file and also
+     * their own dependencies
+     */
+    private fun dependentProjectDependencies(project: Project?, context: KobaltContext,
+            toDependencies: (Project) -> List<IClasspathDependency>)
+            : List<IClasspathDependency> {
+        // Get the transitive closure of all the projects this project depends on
+        val transitiveProjects =
+            if (project == null) emptyList()
+            else DynamicGraph.transitiveClosure(project) { project -> project.dependsOn }
+
+        val result = arrayListOf<IClasspathDependency>()
+
+        /**
+         * Add the class directories of projects depended upon
+         */
+        fun maybeAddClassDir(classDir: String) {
+            // A project is allowed not to have any kobaltBuild/classes or test-classes directory if it doesn't have
+            // any sources
+            if (File(classDir).exists()) {
+                result.add(FileDependency(classDir))
+            }
+        }
+        transitiveProjects.forEach { p ->
+            maybeAddClassDir(KFiles.joinDir(p.directory, p.classesDir(context)))
+            maybeAddClassDir(KFiles.makeOutputTestDir(p).path)
+        }
+
+        // And add all the transitive projects
+        result.addAll(transitiveProjects.flatMap { toDependencies(it) })
+        return result
+    }
+
+}

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/aether/Aether.kt

@@ -3,6 +3,7 @@ package com.beust.kobalt.maven.aether
 import com.beust.kobalt.KobaltException
 import com.beust.kobalt.api.IClasspathDependency
 import com.beust.kobalt.api.Kobalt
+import com.beust.kobalt.api.Project
 import com.beust.kobalt.internal.KobaltSettings
 import com.beust.kobalt.internal.KobaltSettingsXml
 import com.beust.kobalt.internal.getProxy
@@ -48,6 +49,22 @@ enum class Scope(val scope: String) {
             val javaScopes = scopes.map { DependencyFilterUtils.classpathFilter(it.scope) }.toTypedArray()
             return AndDependencyFilter(KobaltAether.ExcludeOptionalDependencyFilter(), *javaScopes)
         }
+
+        /**
+         * @return a lambda that extracts the correct dependencies from a project based on the scope
+         * filters passed.
+         */
+        fun toDependencyLambda(scopes: Collection<Scope>) : (Project) -> List<IClasspathDependency> {
+            val result =
+                if (scopes.contains(Scope.COMPILE) && scopes.contains(Scope.TEST)) {
+                    { project : Project -> project.compileDependencies + project.testDependencies }
+                } else if (scopes.contains(Scope.TEST)) {
+                    { project : Project -> project.testDependencies }
+                } else {
+                    { project : Project -> project.compileDependencies }
+                }
+            return result
+        }
     }
 }
 

src/test/kotlin/com/beust/kobalt/internal/DynamicGraphTest.kt

@@ -56,11 +56,11 @@ class DynamicGraphTest {
             addEdge(1, 3)
             addEdge(2, 4)
             addEdge(6, 7)
-            assertThat(transitiveClosure(1)).isEqualTo(setOf(1, 2, 3, 4))
+            assertThat(transitiveClosure(1)).isEqualTo(listOf(1, 2, 3, 4))
-            assertThat(transitiveClosure(2)).isEqualTo(setOf(2, 4))
+            assertThat(transitiveClosure(2)).isEqualTo(listOf(2, 4))
-            assertThat(transitiveClosure(3)).isEqualTo(setOf(3))
+            assertThat(transitiveClosure(3)).isEqualTo(listOf(3))
-            assertThat(transitiveClosure(6)).isEqualTo(setOf(6, 7))
+            assertThat(transitiveClosure(6)).isEqualTo(listOf(6, 7))
-            assertThat(transitiveClosure(7)).isEqualTo(setOf(7))
+            assertThat(transitiveClosure(7)).isEqualTo(listOf(7))
             println("done")
         }
     }

src/test/kotlin/com/beust/kobalt/maven/DependencyManagerTest.kt

@@ -15,6 +15,7 @@ import org.testng.annotations.Test
 
 @Guice(modules = arrayOf(TestModule::class))
 class DependencyManagerTest @Inject constructor(val dependencyManager: DependencyManager,
+        val dependencyManager2: DependencyManager2,
         val compilerFactory: BuildFileCompiler.IFactory, override val aether: KobaltAether) : BaseTest(aether) {
 
     private fun assertContains(dependencies: List<IClasspathDependency>, vararg ids: String) {
@@ -69,4 +70,30 @@ class DependencyManagerTest @Inject constructor(val dependencyManager: Dependenc
         assertContains(dependencies, ":testng:")
         assertContains(dependencies, ":jcommander:")
     }
+
+    @Test
+    fun honorRuntimeDependenciesBetweenProjects2() {
+        val buildFileString = """
+            import com.beust.kobalt.*
+
+            val lib = project {
+                name = "lib"
+                dependencies {
+                    compile("org.testng:testng:6.9.11")
+                    runtime("com.beust:jcommander:1.48")
+                }
+            }
+
+            val p = project(lib) {
+                name = "transitive"
+            }
+        """
+
+        val compileResult = compileBuildFile(buildFileString, Args(), compilerFactory)
+        val project2 = compileResult.projects[1]
+        val dependencies = dependencyManager2.resolve(project2, Kobalt.context!!, isTest = false,
+                passedScopeFilters = listOf(Scope.COMPILE, Scope.RUNTIME))
+        assertContains(dependencies, ":testng:")
+        assertContains(dependencies, ":jcommander:")
+    }
 }