From 1ae9baa97d53dfe10a6e05d86813693637e21ecf Mon Sep 17 00:00:00 2001 From: Cedric Beust Date: Thu, 31 Mar 2016 23:01:47 -0800 Subject: [PATCH] Make "assemble" incremental. --- .../com/beust/kobalt/IncrementalTaskInfo.kt | 9 +- .../com/beust/kobalt/api/KobaltContext.kt | 10 +- .../kobalt/internal/IncrementalManager.kt | 104 ++++++++++-------- .../kobalt/internal/JvmCompilerPlugin.kt | 28 ++--- .../main/kotlin/com/beust/kobalt/maven/Md5.kt | 2 +- .../plugin/packaging/PackagingPlugin.kt | 63 +++++------ 6 files changed, 112 insertions(+), 104 deletions(-) diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/IncrementalTaskInfo.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/IncrementalTaskInfo.kt index 661e0c68..b1c56e5a 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/IncrementalTaskInfo.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/IncrementalTaskInfo.kt @@ -1,5 +1,6 @@ package com.beust.kobalt +import com.beust.kobalt.api.KobaltContext import com.beust.kobalt.api.Project /** @@ -7,10 +8,10 @@ import com.beust.kobalt.api.Project * calculated by Kobalt. If they differ, the task gets run. If they are equal, outputChecksums are then compared. * @param outputChecksum The checksum for the output of this task. If null, the output is absent * and the task will be run. If non null, it gets compared against the checksum of the previous run and - * if they differ, the task gets run. Note that this parameter is a closure and not a direct value - * because Kobalt needs to call it twice: once before the task and once after a successful execution (to store it). + * if they differ, the task gets run. * @param task The task to run. */ -class IncrementalTaskInfo(val inputChecksum: String?, +class IncrementalTaskInfo(val inputChecksum: () -> String?, val outputChecksum: () -> String?, - val task: (Project) -> TaskResult) + val task: (Project) -> TaskResult, + val context: KobaltContext) diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/KobaltContext.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/KobaltContext.kt index 2d9cbd78..30f76e93 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/KobaltContext.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/KobaltContext.kt @@ -8,7 +8,7 @@ import com.beust.kobalt.internal.PluginInfo import com.beust.kobalt.maven.DependencyManager import com.beust.kobalt.misc.KobaltExecutors -public class KobaltContext(val args: Args) { +class KobaltContext(val args: Args) { var variant: Variant = Variant() val profiles = arrayListOf() @@ -20,6 +20,14 @@ public class KobaltContext(val args: Args) { fun findPlugin(name: String) = Plugins.findPlugin(name) + /** + * When an incremental task decides it's up to date, it sets this boolean to true so that subsequent + * tasks in that project can be skipped as well. This is an internal field that should only be set by Kobalt. + */ + private val incrementalSuccesses = hashSetOf() + fun previousTaskWasIncrementalSuccess(projectName: String) = incrementalSuccesses.contains(projectName) ?: false + fun setIncrementalSuccess(projectName: String) = incrementalSuccesses.add(projectName) + // // Injected // diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/IncrementalManager.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/IncrementalManager.kt index 3587b092..64997291 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/IncrementalManager.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/IncrementalManager.kt @@ -84,62 +84,72 @@ class IncrementalManager(val fileName: String = IncrementalManager.BUILD_INFO_FI val taskName = project.name + ":" + shortTaskName var upToDate = false var taskOutputChecksum : String? = null - // - // First, compare the input checksums - // - inputChecksumFor(taskName)?.let { inputChecksum -> - val dependsOnDirtyProjects = project.projectExtra.dependsOnDirtyProjects(project) - if (inputChecksum == iti.inputChecksum && ! dependsOnDirtyProjects) { - // - // Input checksums are equal, compare the output checksums - // - outputChecksumFor(taskName)?.let { outputChecksum -> - taskOutputChecksum = iti.outputChecksum() - if (outputChecksum == taskOutputChecksum) { - upToDate = true - } else { - logIncremental(LEVEL, "Incremental task $taskName output is out of date, running it") - } - } - } else { - if (dependsOnDirtyProjects) { - logIncremental(LEVEL, "Project ${project.name} depends on dirty project, running $taskName") - } else { - logIncremental(LEVEL, "Incremental task $taskName input is out of date, running it" - + " old: $inputChecksum new: ${iti.inputChecksum}") - } - project.projectExtra.isDirty = true - } - } - if (! upToDate) { + if (iti.context.previousTaskWasIncrementalSuccess(project.name)) { // - // The task is out of date, invoke the task on the IncrementalTaskInfo object + // If the previous task was an incremental success, no need to run // - val result = iti.task(project) - if (result.success) { - logIncremental(LEVEL, "Incremental task $taskName done running, saving checksums") - iti.inputChecksum?.let { - saveInputChecksum(taskName, it) - logIncremental(LEVEL, " input checksum \"$it\" saved") - } - // Important to rerun the checksum here since the output of the task might have changed it - iti.outputChecksum()?.let { - saveOutputChecksum(taskName, it) - logIncremental(LEVEL, " output checksum \"$it\" saved") - } - } - result + logIncremental(LEVEL, "Previous incremental task was a success, not running $shortTaskName") + TaskResult() } else { // - // Identical input and output checksums, don't run the task + // First, compare the input checksums // - logIncremental(LEVEL, "Incremental task \"$taskName\" is up to date, not running it") - TaskResult() + inputChecksumFor(taskName)?.let { inputChecksum -> + val dependsOnDirtyProjects = project.projectExtra.dependsOnDirtyProjects(project) + if (inputChecksum == iti.inputChecksum() && !dependsOnDirtyProjects) { + // + // Input checksums are equal, compare the output checksums + // + outputChecksumFor(taskName)?.let { outputChecksum -> + taskOutputChecksum = iti.outputChecksum() + if (outputChecksum == taskOutputChecksum) { + upToDate = true + } else { + logIncremental(LEVEL, "Incremental task $taskName output is out of date, running it") + } + } + } else { + if (dependsOnDirtyProjects) { + logIncremental(LEVEL, "Project ${project.name} depends on dirty project, running $taskName") + } else { + logIncremental(LEVEL, "Incremental task $taskName input is out of date, running it" + + " old: $inputChecksum new: ${iti.inputChecksum()}") + } + project.projectExtra.isDirty = true + } + } + + if (!upToDate) { + // + // The task is out of date, invoke the task on the IncrementalTaskInfo object + // + val result = iti.task(project) + if (result.success) { + logIncremental(LEVEL, "Incremental task $taskName done running, saving checksums") + iti.inputChecksum()?.let { + saveInputChecksum(taskName, it) + logIncremental(LEVEL, " input checksum \"$it\" saved") + } + // Important to rerun the checksum here since the output of the task might have changed it + iti.outputChecksum()?.let { + saveOutputChecksum(taskName, it) + logIncremental(LEVEL, " output checksum \"$it\" saved") + } + } + result + } else { + // + // Identical input and output checksums, don't run the task + // + logIncremental(LEVEL, "Incremental task \"$taskName\" is up to date, not running it") + iti.context.setIncrementalSuccess(project.name) + TaskResult() + } } } } - val LEVEL = 3 + val LEVEL = 2 private fun logIncremental(level: Int, s: String) = log(level, " INC - $s") } diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/JvmCompilerPlugin.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/JvmCompilerPlugin.kt index 4067bec3..c6839b61 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/JvmCompilerPlugin.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/JvmCompilerPlugin.kt @@ -143,22 +143,22 @@ open class JvmCompilerPlugin @Inject constructor( runAfter = arrayOf(JvmCompilerPlugin.TASK_COMPILE)) fun taskCompileTest(project: Project): IncrementalTaskInfo { sourceTestDirectories.addAll(context.variant.sourceDirectories(project, context, SourceSet.of(isTest = true))) - val inputChecksum = Md5.toMd5Directories(context.testSourceDirectories(project).map { - File(project.directory, it.path) - }) return IncrementalTaskInfo( - inputChecksum = inputChecksum, - outputChecksum = { - Md5.toMd5Directories(listOf(KFiles.makeOutputTestDir(project))) - }, - task = { project -> doTaskCompileTest(project) } + inputChecksum = { + Md5.toMd5Directories(context.testSourceDirectories(project).map { File(project.directory, it.path)}) + }, + outputChecksum = { + Md5.toMd5Directories(listOf(KFiles.makeOutputTestDir(project))) + }, + task = { project -> doTaskCompileTest(project)}, + context = context ) } @IncrementalTask(name = JvmCompilerPlugin.TASK_COMPILE, description = "Compile the project") fun taskCompile(project: Project): IncrementalTaskInfo { // Generate the BuildConfig before invoking sourceDirectories() since that call - // might add the buildConfig source directori + // might add the buildConfig source directories val sourceDirectory = context.variant.maybeGenerateBuildConfig(project, context) if (sourceDirectory != null) { sourceDirectories.add(sourceDirectory) @@ -167,15 +167,15 @@ open class JvmCompilerPlugin @Inject constructor( // Set up the source files now that we have the variant sourceDirectories.addAll(context.variant.sourceDirectories(project, context, SourceSet.of(isTest = false))) - val inputChecksum = Md5.toMd5Directories(context.sourceDirectories(project).map { - File(project.directory, it.path) - }) return IncrementalTaskInfo( - inputChecksum = inputChecksum, + inputChecksum = { + Md5.toMd5Directories(context.sourceDirectories(project).map { File(project.directory, it.path) }) + }, outputChecksum = { Md5.toMd5Directories(listOf(File(project.directory, project.classesDir(context)))) }, - task = { project -> doTaskCompile(project) } + task = { project -> doTaskCompile(project) }, + context = context ) } diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/Md5.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/Md5.kt index b3c330fe..339efaa1 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/Md5.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/maven/Md5.kt @@ -30,7 +30,7 @@ public class Md5 { fileCount++ } else { val files = KFiles.findRecursively(file) // , { f -> f.endsWith("java")}) - log(2, " Calculating checksum of ${files.size} files in $file") + log(2, " Calculating checksum of ${files.size} files in $file") files.map { File(file, it) }.filter { diff --git a/src/main/kotlin/com/beust/kobalt/plugin/packaging/PackagingPlugin.kt b/src/main/kotlin/com/beust/kobalt/plugin/packaging/PackagingPlugin.kt index 3ed25496..8657c11c 100644 --- a/src/main/kotlin/com/beust/kobalt/plugin/packaging/PackagingPlugin.kt +++ b/src/main/kotlin/com/beust/kobalt/plugin/packaging/PackagingPlugin.kt @@ -1,14 +1,11 @@ package com.beust.kobalt.plugin.packaging -import com.beust.kobalt.JarGenerator -import com.beust.kobalt.KobaltException -import com.beust.kobalt.TaskResult +import com.beust.kobalt.* import com.beust.kobalt.api.* import com.beust.kobalt.api.annotation.Directive import com.beust.kobalt.api.annotation.ExportedProjectProperty import com.beust.kobalt.api.annotation.Task import com.beust.kobalt.archive.* -import com.beust.kobalt.glob import com.beust.kobalt.internal.IncrementalManager import com.beust.kobalt.internal.JvmCompilerPlugin import com.beust.kobalt.maven.DependencyManager @@ -26,7 +23,8 @@ class PackagingPlugin @Inject constructor(val dependencyManager : DependencyMana val executors: KobaltExecutors, val jarGenerator: JarGenerator, val warGenerator: WarGenerator, val zipGenerator: ZipGenerator, val taskContributor: TaskContributor, val pomFactory: PomGenerator.IFactory, val configActor: ConfigActor) - : BasePlugin(), ITaskContributor, IAssemblyContributor, IConfigActor by configActor { + : BasePlugin(), ITaskContributor, IIncrementalAssemblyContributor, + IConfigActor by configActor { companion object { const val PLUGIN_NAME = "Packaging" @@ -53,39 +51,30 @@ class PackagingPlugin @Inject constructor(val dependencyManager : DependencyMana runTask = { doTaskAssemble(project) }) } - // IAssemblyContributor - override fun assemble(project: Project, context: KobaltContext) : TaskResult { - try { - project.projectProperties.put(PACKAGES, packages) - packages.filter { it.project.name == project.name }.forEach { pkg -> - pkg.jars.forEach { jarGenerator.generateJar(pkg.project, context, it) } - pkg.wars.forEach { warGenerator.generateWar(pkg.project, context, it) } - pkg.zips.forEach { zipGenerator.generateZip(pkg.project, context, it) } - if (pkg.generatePom) { - pomFactory.create(project).generate() + /** + * "assemble" is an incremental task but with a twist. Because it can be costly to determine if any + * of the class files generated in the previous phase is new or not, we just don't do that and always + * return "null" for both input and output checksums, which would cause that task to always be rerun. + * However, we are depending on Kobalt's cascading incremental management to skip up whenever appropriate: + * whenever a previous incremental task was a success, all following incremental tasks are automatically + * skipped. + */ + override fun assemble(project: Project, context: KobaltContext) : IncrementalTaskInfo { + return IncrementalTaskInfo({ null }, { null }, { project -> + try { + project.projectProperties.put(PACKAGES, packages) + packages.filter { it.project.name == project.name }.forEach { pkg -> + pkg.jars.forEach { jarGenerator.generateJar(pkg.project, context, it) } + pkg.wars.forEach { warGenerator.generateWar(pkg.project, context, it) } + pkg.zips.forEach { zipGenerator.generateZip(pkg.project, context, it) } + if (pkg.generatePom) { + pomFactory.create(project).generate() + } } - } - return TaskResult() - } catch(ex: Exception) { - throw KobaltException(ex) - } - } - - private fun doAssemble(project: Project, context: KobaltContext) : TaskResult { - try { - project.projectProperties.put(PACKAGES, packages) - packages.filter { it.project.name == project.name }.forEach { pkg -> - pkg.jars.forEach { jarGenerator.generateJar(pkg.project, context, it) } - pkg.wars.forEach { warGenerator.generateWar(pkg.project, context, it) } - pkg.zips.forEach { zipGenerator.generateZip(pkg.project, context, it) } - if (pkg.generatePom) { - pomFactory.create(project).generate() - } - } - return TaskResult() - } catch(ex: Exception) { - throw KobaltException(ex) - } + TaskResult() + } catch(ex: Exception) { + throw KobaltException(ex) + }}, context) } @Task(name = TASK_ASSEMBLE, description = "Package the artifacts",