From d18c8009c840ae2cdaefa8780d08b77a4a299214 Mon Sep 17 00:00:00 2001 From: Cedric Beust Date: Tue, 26 Jul 2016 02:24:35 -0800 Subject: [PATCH] Revamp the scopes and scope filters. --- .../kotlin/com/beust/kobalt/JarGenerator.kt | 2 +- .../beust/kobalt/api/IDependencyManager.kt | 7 +- .../com/beust/kobalt/internal/JvmCompiler.kt | 2 +- .../beust/kobalt/maven/DependencyManager.kt | 50 ++++++++--- .../com/beust/kobalt/maven/aether/Aether.kt | 83 ++++++++++++------- .../plugin/application/ApplicationPlugin.kt | 5 +- .../plugin/packaging/PackagingPlugin.kt | 2 +- .../kobalt/plugin/packaging/WarGenerator.kt | 2 +- src/test/kotlin/com/beust/kobalt/BaseTest.kt | 24 ++++++ .../com/beust/kobalt/internal/ProfileTest.kt | 37 +++------ .../kobalt/maven/DependencyManagerTest.kt | 53 ++++++++++-- .../com/beust/kobalt/maven/DownloadTest.kt | 2 +- 12 files changed, 183 insertions(+), 86 deletions(-) diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/JarGenerator.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/JarGenerator.kt index 1146fd3b..e86960fe 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/JarGenerator.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/JarGenerator.kt @@ -102,7 +102,7 @@ class JarGenerator @Inject constructor(val dependencyManager: DependencyManager) context.variant.buildType.compileRuntimeDependencies + context.variant.productFlavor.compileDependencies + context.variant.productFlavor.compileRuntimeDependencies - val transitiveDependencies = dependencyManager.calculateDependencies(project, context, false, + val transitiveDependencies = dependencyManager.calculateDependencies(project, context, emptyList(), allDependencies) transitiveDependencies.map { it.jarFile.get() diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/IDependencyManager.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/IDependencyManager.kt index 3b514cf3..ea472379 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/IDependencyManager.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/IDependencyManager.kt @@ -1,5 +1,7 @@ package com.beust.kobalt.api +import com.beust.kobalt.maven.aether.Scope + /** * Manage the creation of dependencies and also provide dependencies for projects. */ @@ -33,6 +35,7 @@ interface IDependencyManager { * @return the classpath for this project, including the IClasspathContributors. * allDependencies is typically either compileDependencies or testDependencies */ - fun calculateDependencies(project: Project?, context: KobaltContext, isTest: Boolean = false, - vararg allDependencies: List): List + fun calculateDependencies(project: Project?, context: KobaltContext, + scopeFilters: Collection = emptyList(), + vararg passedDependencies: List): List } \ No newline at end of file diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/JvmCompiler.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/JvmCompiler.kt index cb614c94..dd612256 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/JvmCompiler.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/JvmCompiler.kt @@ -25,7 +25,7 @@ class JvmCompiler @Inject constructor(val dependencyManager: DependencyManager) // Dependencies val allDependencies = (info.dependencies - + dependencyManager.calculateDependencies(project, context!!, allDependencies = info.dependencies)) + + dependencyManager.calculateDependencies(project, context!!, passedDependencies = info.dependencies)) .distinct() // Plugins that add flags to the compiler diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/DependencyManager.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/DependencyManager.kt index d12bfdfa..76371a39 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/DependencyManager.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/DependencyManager.kt @@ -3,6 +3,7 @@ package com.beust.kobalt.maven import com.beust.kobalt.KobaltException import com.beust.kobalt.api.* import com.beust.kobalt.maven.aether.KobaltAether +import com.beust.kobalt.maven.aether.Scope import com.beust.kobalt.maven.dependency.FileDependency import com.beust.kobalt.misc.KFiles import com.beust.kobalt.misc.KobaltExecutors @@ -82,16 +83,40 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val /** * @return the classpath for this project, including the IClasspathContributors. - * allDependencies is typically either compileDependencies or testDependencies + * allDependencies is typically either compileDependencies or testDependencies. If no dependencies + * are passed, they are calculated from the scope filters. */ - override fun calculateDependencies(project: Project?, context: KobaltContext, isTest: Boolean, - vararg allDependencies: List): List { + override fun calculateDependencies(project: Project?, context: KobaltContext, + scopeFilters: Collection, + vararg passedDependencies: List): List { val result = arrayListOf() + + /** + * Extract the correct dependencies from the project based on the scope filters. + */ + fun filtersToDependencies(project: Project, scopes: Collection): List { + return arrayListOf().apply { + if (scopes.contains(Scope.COMPILE)) { + addAll(project.compileDependencies) + } + if (scopes.contains(Scope.RUNTIME)) { + addAll(project.compileRuntimeDependencies) + } + if (scopes.contains(Scope.TEST)) { + addAll(project.testDependencies) + } + } + } + + val allDependencies : Array> = + if (project == null || passedDependencies.any()) passedDependencies + else arrayOf(filtersToDependencies(project, scopeFilters)) + allDependencies.forEach { dependencies -> - result.addAll(transitiveClosure(dependencies, isTest, project?.name)) + result.addAll(transitiveClosure(dependencies, scopeFilters, project?.name)) } result.addAll(runClasspathContributors(project, context)) - result.addAll(dependentProjectDependencies(project, context)) + result.addAll(dependentProjectDependencies(project, context, scopeFilters)) // Dependencies get reordered by transitiveClosure() but since we just added a bunch of new ones, // we need to reorder them again in case we're adding dependencies that are already present @@ -113,13 +138,14 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val * Return the transitive closure of the dependencies *without* running the classpath contributors. * TODO: This should be private, everyone should be calling calculateDependencies(). */ - fun transitiveClosure(dependencies : List, isTest: Boolean = false, + fun transitiveClosure(dependencies : List, + scopeFilter: Collection = emptyList(), requiredBy: String? = null): List { val result = arrayListOf() dependencies.forEach { result.add(it) if (it.isMaven) { - val resolved = aether.resolveAll(it.id, isTest).map { it.toString() } + val resolved = aether.resolveAll(it.id, null, scopeFilter).map { it.toString() } result.addAll(resolved.map { create(it) }) } } @@ -151,8 +177,8 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val * If this project depends on other projects, we need to include their jar file and also * their own dependencies */ - private fun dependentProjectDependencies( - project: Project?, context: KobaltContext) : List { + private fun dependentProjectDependencies(project: Project?, context: KobaltContext, scopeFilters: Collection) + : List { if (project == null) { return emptyList() } else { @@ -165,7 +191,7 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val result.add(FileDependency(KFiles.joinDir(p.directory, p.classesDir(context)))) } } - val otherDependencies = calculateDependencies(p, context, false, p.compileDependencies) + val otherDependencies = calculateDependencies(p, context, scopeFilters) result.addAll(otherDependencies) } @@ -177,6 +203,7 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val : List { val transitive = hashSetOf() with(project) { + val scopeFilters = arrayListOf(Scope.COMPILE) context.variant.let { variant -> val deps = arrayListOf(compileDependencies, compileProvidedDependencies, variant.buildType.compileDependencies, @@ -187,9 +214,10 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val if (isTest) { deps.add(testDependencies) deps.add(testProvidedDependencies) + scopeFilters.add(Scope.TEST) } deps.filter { it.any() }.forEach { - transitive.addAll(calculateDependencies(project, context, isTest, it)) + transitive.addAll(calculateDependencies(project, context, scopeFilters, it)) } } } diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/aether/Aether.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/aether/Aether.kt index cb9f6ced..25c118d2 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/aether/Aether.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/aether/Aether.kt @@ -32,11 +32,28 @@ import org.eclipse.aether.util.filter.DependencyFilterUtils import java.io.File import java.util.concurrent.Future +enum class Scope(val scope: String) { + COMPILE(JavaScopes.COMPILE), + PROVIDED(JavaScopes.PROVIDED), + SYSTEM(JavaScopes.SYSTEM), + RUNTIME(JavaScopes.RUNTIME), + TEST(JavaScopes.TEST) + ; + + companion object { + /** + * @return a filter that excludes optional dependencies and allows all the scopes passed in parameter. + */ + fun toFilter(scopes: Collection): DependencyFilter { + val javaScopes = scopes.map { DependencyFilterUtils.classpathFilter(it.scope) }.toTypedArray() + return AndDependencyFilter(KobaltAether.ExcludeOptionalDependencyFilter(), *javaScopes) + } + } +} + class DependencyResult(val dependency: IClasspathDependency, val repoUrl: String) class KobaltAether @Inject constructor (val settings: KobaltSettings, val aether: Aether) { - val localRepo: File get() = settings.localCache - /** * Create an IClasspathDependency from a Kobalt id. */ @@ -50,14 +67,16 @@ class KobaltAether @Inject constructor (val settings: KobaltSettings, val aether DependencyResult(AetherDependency(it.artifact), it.repository.toString()) } - fun resolveAll(id: String, isTest: Boolean): List { - val results = aether.resolve(DefaultArtifact(id), isTest) + fun resolveAll(id: String, artifactScope: Scope? = null, filterScopes: Collection = emptyList()) + : List { + val results = aether.resolve(DefaultArtifact(id), artifactScope, filterScopes) return results.map { it.artifact.toString() } } - fun resolve(id: String, isTest: Boolean = false): DependencyResult { + fun resolve(id: String, artifactScope: Scope? = null, filterScopes: Collection = emptyList()) + : DependencyResult { log(ConsoleRepositoryListener.LOG_LEVEL, "Resolving $id") - val result = resolveToArtifact(id, isTest) + val result = resolveToArtifact(id, artifactScope, filterScopes) if (result != null) { return DependencyResult(AetherDependency(result.artifact), result.repository.toString()) } else { @@ -65,9 +84,10 @@ class KobaltAether @Inject constructor (val settings: KobaltSettings, val aether } } - fun resolveToArtifact(id: String, isTest: Boolean = false): ArtifactResult? { + fun resolveToArtifact(id: String, artifactScope: Scope? = null, filterScopes: Collection = emptyList()) + : ArtifactResult? { log(ConsoleRepositoryListener.LOG_LEVEL, "Resolving $id") - val results = aether.resolve(DefaultArtifact(MavenId.toKobaltId(id)), isTest) + val results = aether.resolve(DefaultArtifact(MavenId.toKobaltId(id)), artifactScope, filterScopes) if (results.size > 0) { return results[0] } else { @@ -87,17 +107,11 @@ class KobaltAether @Inject constructor (val settings: KobaltSettings, val aether } @Singleton -class Aether(val localRepo: File, val settings: KobaltSettings, val eventBus: EventBus) { +class Aether(localRepo: File, val settings: KobaltSettings, val eventBus: EventBus) { private val system = Booter.newRepositorySystem() private val session = Booter.newRepositorySystemSession(system, localRepo, settings, eventBus) - private val classpathFilter = AndDependencyFilter( - KobaltAether.ExcludeOptionalDependencyFilter(), - DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE), - DependencyFilterUtils.classpathFilter(JavaScopes.TEST)) - - private val testClasspathFilter = AndDependencyFilter( - KobaltAether.ExcludeOptionalDependencyFilter(), - DependencyFilterUtils.classpathFilter(JavaScopes.TEST)) +// private val classpathFilter = Scopes.toFilter(Scopes.COMPILE, Scopes.TEST) +// private val testClasspathFilter = Scopes.toFilter(Scopes.TEST) private val kobaltRepositories: List get() = Kobalt.repos.map { @@ -111,9 +125,9 @@ class Aether(val localRepo: File, val settings: KobaltSettings, val eventBus: Ev } } - private fun collectRequest(artifact: Artifact, isTest: Boolean): CollectRequest { + private fun collectRequest(artifact: Artifact, scope: Scope?): CollectRequest { with(CollectRequest()) { - root = Dependency(artifact, if (isTest) JavaScopes.TEST else JavaScopes.COMPILE) + root = Dependency(artifact, scope?.scope) repositories = kobaltRepositories return this @@ -126,8 +140,8 @@ class Aether(val localRepo: File, val settings: KobaltSettings, val eventBus: Ev if (resolved != null) { val newArtifact = DefaultArtifact(artifact.groupId, artifact.artifactId, artifact.extension, resolved.highestVersion.toString()) - val artifactResult = resolve(newArtifact) - if (artifactResult != null && artifactResult.size > 0) { + val artifactResult = resolve(newArtifact, null, emptyList()) + if (artifactResult.any()) { return artifactResult[0] } else { throw KobaltException("Couldn't find latest artifact for $group:$artifactId") @@ -143,7 +157,7 @@ class Aether(val localRepo: File, val settings: KobaltSettings, val eventBus: Ev return result } - fun resolve(artifact: Artifact, isTest: Boolean = false): List { + fun resolve(artifact: Artifact, artifactScope: Scope?, filterScopes: Collection): List { fun manageException(ex: Exception, artifact: Artifact): List { if (artifact.extension == "pom") { // Only display a warning for .pom files. Not resolving a .jar or other artifact @@ -154,8 +168,8 @@ class Aether(val localRepo: File, val settings: KobaltSettings, val eventBus: Ev } try { - val dependencyRequest = DependencyRequest(collectRequest(artifact, isTest), - if (isTest) testClasspathFilter else classpathFilter) + val scopeFilter = Scope.toFilter(filterScopes) + val dependencyRequest = DependencyRequest(collectRequest(artifact, artifactScope), scopeFilter) val result = system.resolveDependencies(session, dependencyRequest).artifactResults return result } catch(ex: ArtifactNotFoundException) { @@ -165,10 +179,10 @@ class Aether(val localRepo: File, val settings: KobaltSettings, val eventBus: Ev } } - fun transitiveDependencies(artifact: Artifact) = directDependencies(artifact) +// fun transitiveDependencies(artifact: Artifact) = directDependencies(artifact) - fun directDependencies(artifact: Artifact, isTest: Boolean = false): CollectResult? - = system.collectDependencies(session, collectRequest(artifact, isTest)) + fun directDependencies(artifact: Artifact, artifactScope: Scope? = null): CollectResult? + = system.collectDependencies(session, collectRequest(artifact, artifactScope)) } class AetherDependency(val artifact: Artifact) : IClasspathDependency, Comparable { @@ -194,12 +208,16 @@ class AetherDependency(val artifact: Artifact) : IClasspathDependency, Comparabl if (file.exists()) { CompletedFuture(file) } else { - val td = aether.resolve(artifact) - val newFile = td[0].artifact.file - if (newFile != null) { - CompletedFuture(newFile) + val td = aether.resolve(artifact, null, emptyList()) + if (td.any()) { + val newFile = td[0].artifact.file + if (newFile != null) { + CompletedFuture(newFile) + } else { + CompletedFuture(File("DOESNOTEXIST $id")) // will be filtered out + } } else { - CompletedFuture(File("DONOTEXIST")) // will be filtered out + CompletedFuture(File("DOESNOTEXIST $id")) } } } @@ -277,3 +295,4 @@ fun main(argv: Array) { // println("Artifact: " + d) } + diff --git a/src/main/kotlin/com/beust/kobalt/plugin/application/ApplicationPlugin.kt b/src/main/kotlin/com/beust/kobalt/plugin/application/ApplicationPlugin.kt index 85efffed..fdd36316 100644 --- a/src/main/kotlin/com/beust/kobalt/plugin/application/ApplicationPlugin.kt +++ b/src/main/kotlin/com/beust/kobalt/plugin/application/ApplicationPlugin.kt @@ -7,6 +7,7 @@ import com.beust.kobalt.api.annotation.Task import com.beust.kobalt.archive.Archives import com.beust.kobalt.internal.ActorUtils import com.beust.kobalt.maven.DependencyManager +import com.beust.kobalt.maven.aether.Scope import com.beust.kobalt.misc.KFiles import com.beust.kobalt.misc.KobaltExecutors import com.beust.kobalt.misc.RunCommand @@ -105,7 +106,9 @@ class ApplicationPlugin @Inject constructor(val configActor: ConfigActor, vararg ids: String) { + ids.forEach { id -> + assertThat(dependencies.any { it.id.contains(id) }).isTrue() + } + } @Test(description = "Make sure that COMPILE scope dependencies get resolved properly") fun testScopeDependenciesShouldBeDownloaded() { val testDeps = listOf(dependencyManager.create("org.testng:testng:6.9.11")) - fun assertContains(dependencies: List, vararg ids: String) { - ids.forEach { id -> - assertThat(dependencies.any { it.id.contains(id) }).isTrue() - } - } - // Should only resolve to TestNG - dependencyManager.transitiveClosure(testDeps, isTest = false).let { dependencies -> + dependencyManager.transitiveClosure(testDeps, listOf(Scope.COMPILE)).let { dependencies -> assertThat(dependencies.any { it.id.contains(":jcommander:") }).isFalse() assertContains(dependencies, ":testng:") } // Should resolve to TestNG and its dependencies - dependencyManager.transitiveClosure(testDeps, isTest = true).let { dependencies -> + dependencyManager.transitiveClosure(testDeps, listOf(Scope.TEST)).let { dependencies -> assertContains(dependencies, ":jcommander:") assertContains(dependencies, ":bsh:") assertContains(dependencies, ":ant:") + assertContains(dependencies, ":ant-launcher:") + assertContains(dependencies, ":testng:") } } + @Test + fun honorRuntimeDependenciesBetweenProjects() { + val buildFileString = """ + import com.beust.kobalt.* + + val lib = project { + name = "lib" + dependencies { + compile("org.testng:testng:6.9.11") + runtime("com.beust:jcommander:1.48") + } + } + + val p = project(lib) { + name = "transitive" + } + """ + + val compileResult = compileBuildFile(buildFileString, Args(), compilerFactory) + val project2 = compileResult.projects[1] + val dependencies = dependencyManager.calculateDependencies(project2, Kobalt.context!!, + listOf(Scope.COMPILE, Scope.RUNTIME), + project2.compileDependencies + project2.compileRuntimeDependencies) + assertContains(dependencies, ":testng:") + assertContains(dependencies, ":jcommander:") + } } diff --git a/src/test/kotlin/com/beust/kobalt/maven/DownloadTest.kt b/src/test/kotlin/com/beust/kobalt/maven/DownloadTest.kt index c5fa9b31..c7d09bda 100644 --- a/src/test/kotlin/com/beust/kobalt/maven/DownloadTest.kt +++ b/src/test/kotlin/com/beust/kobalt/maven/DownloadTest.kt @@ -160,7 +160,7 @@ class DownloadTest @Inject constructor( @Test fun variablesShouldBeExpanded() { val dep = dependencyManager.createMaven("org.mapdb:mapdb:3.0.0-M3") - val closure = dependencyManager.transitiveClosure(listOf(dep), false, "") + val closure = dependencyManager.transitiveClosure(listOf(dep), requiredBy = "") val d = closure.filter { it.id.contains("eclipse-collections-api")} Assert.assertEquals(d.size, 1) }