From 4bdc12def50af3bcd5f8a3c7cc8d685ad15b80ea Mon Sep 17 00:00:00 2001 From: Cedric Beust Date: Fri, 3 Mar 2017 13:52:25 -0800 Subject: [PATCH] Incremental compilation. --- .../src/main/kotlin/com/beust/kobalt/Args.kt | 6 +- .../beust/kobalt/api/CompilerActionInfo.kt | 3 +- .../com/beust/kobalt/api/KobaltContext.kt | 1 + .../beust/kobalt/internal/CompilerUtils.kt | 2 +- .../kotlin/com/beust/kobalt/misc/KFiles.kt | 30 ++++++- .../com/beust/kobalt/app/BuildFileCompiler.kt | 7 +- .../com/beust/kobalt/app/BuildScriptUtil.kt | 18 +--- .../com/beust/kobalt/plugin/apt/AptPlugin.kt | 2 +- .../kobalt/plugin/kotlin/KotlinCompiler.kt | 82 +++++++++++++++++-- src/test/kotlin/com/beust/kobalt/BaseTest.kt | 8 +- 10 files changed, 121 insertions(+), 38 deletions(-) diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/Args.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/Args.kt index b2b7ec97..443a48de 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/Args.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/Args.kt @@ -57,9 +57,6 @@ class Args { @Parameter(names = arrayOf("--noIncremental"), description = "Turn off incremental builds") var noIncremental: Boolean = false - @Parameter(names = arrayOf("--parallel"), description = "Build all the projects in parallel whenever possible") - var parallel: Boolean = true - @Parameter(names = arrayOf("--plugins"), description = "Comma-separated list of plug-in Maven id's") var pluginIds: String? = null @@ -82,6 +79,9 @@ class Args { @Parameter(names = arrayOf("--projectInfo"), description = "Display information about the current projects") var projectInfo: Boolean = false + @Parameter(names = arrayOf("--noIncrementalKotlin"), description = "Disable incremental Kotlin compilation") + var noIncrementalKotlin: Boolean = false + @Parameter(names = arrayOf("--sequential"), description = "Build all the projects in sequence") var sequential: Boolean = false diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/CompilerActionInfo.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/CompilerActionInfo.kt index f0be2eb5..d2521255 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/CompilerActionInfo.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/CompilerActionInfo.kt @@ -11,4 +11,5 @@ data class CompilerActionInfo(val directory: String?, val suffixesBeingCompiled: List, val outputDir: File, val compilerArgs: List, - val friendPaths: List) + val friendPaths: List, + val forceRecompile: Boolean) diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/KobaltContext.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/KobaltContext.kt index 983c5961..57939069 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/KobaltContext.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/KobaltContext.kt @@ -19,6 +19,7 @@ import java.io.File class KobaltContext(val args: Args) { lateinit var variant: Variant val profiles = arrayListOf() + var forceRecompile: Boolean = false init { args.profiles?.split(',')?.filterNotNull()?.forEach { diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/CompilerUtils.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/CompilerUtils.kt index a340662b..e17ee4d4 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/CompilerUtils.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/CompilerUtils.kt @@ -167,7 +167,7 @@ class CompilerUtils @Inject constructor(val files: KFiles, val dependencyManager // Finally, alter the info with the compiler interceptors before returning it val initialActionInfo = CompilerActionInfo(projectDirectory.path, classpath, allSources, sourceSuffixes, buildDirectory, emptyList() /* the flags will be provided by flag contributors */, - emptyList()) + emptyList(), context.forceRecompile) val result = context.pluginInfo.compilerInterceptors.fold(initialActionInfo, { ai, interceptor -> interceptor.intercept(project, context, ai) }) diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/misc/KFiles.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/misc/KFiles.kt index bdd0cca5..04fee461 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/misc/KFiles.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/misc/KFiles.kt @@ -5,14 +5,12 @@ import com.beust.kobalt.api.Kobalt import com.beust.kobalt.api.Project import com.beust.kobalt.internal.build.BuildFile import com.beust.kobalt.maven.Md5 -import java.io.File -import java.io.IOException -import java.io.InputStream -import java.io.OutputStream +import java.io.* import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.nio.file.StandardCopyOption +import java.util.jar.JarInputStream class KFiles { /** @@ -170,6 +168,30 @@ class KFiles { return result } + /** + * List the files contained in a directory or a jar file. + */ + fun listFiles(file: File, block: (String) -> Unit) { + if (file.isDirectory) { + KFiles.findRecursively(file).forEach { + block(it) + } + } else if (file.name.endsWith(".jar")) { + FileInputStream(file).use { + JarInputStream(it).use { stream -> + var entry = stream.nextJarEntry + while (entry != null) { + block(entry.name) + entry = stream.nextJarEntry; + } + } + } + + } else { + throw KobaltException("Can't list files of a file: " + file) + } + } + fun copyRecursively(from: File, to: File, replaceExisting: Boolean = true, deleteFirst: Boolean = false, onError: (File, IOException) -> OnErrorAction = { _, exception -> throw exception }) { // Need to wait until copyRecursively supports an overwrite: Boolean = false parameter diff --git a/src/main/kotlin/com/beust/kobalt/app/BuildFileCompiler.kt b/src/main/kotlin/com/beust/kobalt/app/BuildFileCompiler.kt index 2f9db0fc..9d0c4405 100644 --- a/src/main/kotlin/com/beust/kobalt/app/BuildFileCompiler.kt +++ b/src/main/kotlin/com/beust/kobalt/app/BuildFileCompiler.kt @@ -61,6 +61,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil context.resolver = resolver context.pomGeneratorFactory = pomGeneratorFactory context.logger = parallelLogger + context.forceRecompile = forceRecompile Kobalt.context = context // The list of dynamic plug-ins has to be a companion since it's modified directly from @@ -71,7 +72,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil // // Find all the projects in the build file, possibly compiling them // - val projectResult = findProjects(context, forceRecompile) + val projectResult = findProjects(context) return projectResult } @@ -81,7 +82,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil class FindProjectResult(val context: KobaltContext, val projects: List, val pluginUrls: List, val taskResult: TaskResult) - private fun findProjects(context: KobaltContext, forceRecompile: Boolean): FindProjectResult { + private fun findProjects(context: KobaltContext): FindProjectResult { var errorTaskResult: TaskResult? = null val projects = arrayListOf() buildFiles.forEach { buildFile -> @@ -110,7 +111,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil KFiles.saveFile(modifiedBuildFile, parsedBuildFile.buildScriptCode) val taskResult = maybeCompileBuildFile(context, BuildFile(Paths.get(modifiedBuildFile.path), "Modified ${Constants.BUILD_FILE_NAME}", buildFile.realPath), - buildScriptJarFile, pluginUrls, forceRecompile) + buildScriptJarFile, pluginUrls, context.forceRecompile) if (taskResult.success) { projects.addAll(buildScriptUtil.runBuildScriptJarFile(buildScriptJarFile, pluginUrls, context)) } else { diff --git a/src/main/kotlin/com/beust/kobalt/app/BuildScriptUtil.kt b/src/main/kotlin/com/beust/kobalt/app/BuildScriptUtil.kt index 0bbc9bb6..9e913779 100644 --- a/src/main/kotlin/com/beust/kobalt/app/BuildScriptUtil.kt +++ b/src/main/kotlin/com/beust/kobalt/app/BuildScriptUtil.kt @@ -17,12 +17,9 @@ import com.beust.kobalt.misc.warn import com.beust.kobalt.plugin.KobaltPlugin import com.google.inject.Inject import java.io.File -import java.io.FileInputStream -import java.io.InputStream import java.lang.reflect.Modifier import java.net.URL import java.net.URLClassLoader -import java.util.jar.JarInputStream class BuildScriptUtil @Inject constructor(val plugins: Plugins, val files: KFiles, val taskManager: TaskManager) { @@ -37,7 +34,6 @@ class BuildScriptUtil @Inject constructor(val plugins: Plugins, val files: KFile */ fun runBuildScriptJarFile(buildScriptJarFile: File, urls: List, context: KobaltContext) : List { - var stream : InputStream? = null // The jar files used to load the plug-ins are: // - all the plug-ins found in the build file // - kobalt's own jar file @@ -55,13 +51,9 @@ class BuildScriptUtil @Inject constructor(val plugins: Plugins, val files: KFile // // Classload all the jar files and invoke their methods // - try { - stream = JarInputStream(FileInputStream(buildScriptJarFile)) - var entry = stream.nextJarEntry - + if (buildScriptJarFile.exists()) { val classes = hashSetOf>() - while (entry != null) { - val name = entry.name; + KFiles.listFiles(buildScriptJarFile) { name -> if (name.endsWith(".class")) { val className = name.substring(0, name.length - 6).replace("/", ".") try { @@ -75,7 +67,6 @@ class BuildScriptUtil @Inject constructor(val plugins: Plugins, val files: KFile warn("Couldn't find class $className") } } - entry = stream.nextJarEntry; } // Invoke all the "val" found on the _DefaultPackage class (the Build.kt file) @@ -103,10 +94,9 @@ class BuildScriptUtil @Inject constructor(val plugins: Plugins, val files: KFile taskManager.addIncrementalTask(defaultPlugin, method, it) } - }} + } + } } - } finally { - stream?.close() } validateProjectNames(projects) diff --git a/src/main/kotlin/com/beust/kobalt/plugin/apt/AptPlugin.kt b/src/main/kotlin/com/beust/kobalt/plugin/apt/AptPlugin.kt index 666e009d..2aa3156d 100644 --- a/src/main/kotlin/com/beust/kobalt/plugin/apt/AptPlugin.kt +++ b/src/main/kotlin/com/beust/kobalt/plugin/apt/AptPlugin.kt @@ -91,7 +91,7 @@ class AptPlugin @Inject constructor(val dependencyManager: DependencyManager, va val dependencies = dependencyManager.calculateDependencies(project, context) val info = CompilerActionInfo(sourceDir.absolutePath, dependencies, listOf(javaFile.absolutePath), listOf("java"), File(sourceDir, "generated"), - listOf(), listOf()) + listOf(), listOf(), context.forceRecompile) val results = compilerUtils.invokeCompiler(project, context, javaCompiler, info) } diff --git a/src/main/kotlin/com/beust/kobalt/plugin/kotlin/KotlinCompiler.kt b/src/main/kotlin/com/beust/kobalt/plugin/kotlin/KotlinCompiler.kt index cb38210b..66d78b8d 100644 --- a/src/main/kotlin/com/beust/kobalt/plugin/kotlin/KotlinCompiler.kt +++ b/src/main/kotlin/com/beust/kobalt/plugin/kotlin/KotlinCompiler.kt @@ -13,7 +13,10 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity import org.jetbrains.kotlin.cli.common.messages.MessageCollector import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler import org.jetbrains.kotlin.config.Services +import org.jetbrains.kotlin.incremental.ICReporter +import org.jetbrains.kotlin.incremental.makeIncrementally import java.io.File +import java.nio.file.Files import java.util.* import javax.inject.Inject import javax.inject.Singleton @@ -26,6 +29,7 @@ import kotlin.properties.Delegates @Singleton class KotlinCompiler @Inject constructor( val files: KFiles, + val cliArgs: Args, val dependencyManager: DependencyManager, val executors: KobaltExecutors, val settings: KobaltSettings, @@ -37,10 +41,17 @@ class KotlinCompiler @Inject constructor( override fun compile(project: Project?, info: CompilerActionInfo): TaskResult { val projectName = project?.name val version = settings.kobaltCompilerVersion + var filesToCompile = 0 if (! info.outputDir.path.endsWith("ript.jar")) { // Don't display the message if compiling Build.kt - kobaltLog.log(projectName!!, 1, - " Kotlin $version compiling " + Strings.pluralizeAll(info.sourceFiles.size, "file")) + filesToCompile = + info.sourceFiles.map(::File).map { + if (it.isDirectory) KFiles.findRecursively(it).size else 1 + }.reduce { a, b -> + a + b + } + kobaltLog.log(projectName ?: "", 1, + " Kotlin $version compiling " + Strings.pluralizeAll(filesToCompile, "file")) } val cp = compilerFirst(info.dependencies.map { it.jarFile.get() }) val infoDir = info.directory @@ -79,7 +90,7 @@ class KotlinCompiler @Inject constructor( } else { return invokeCompilerDirectly(projectName ?: "kobalt-" + Random().nextInt(), outputDir, - classpath, info.sourceFiles, info.friendPaths.toTypedArray()) + info, classpath, filesToCompile) } } @@ -120,8 +131,10 @@ class KotlinCompiler @Inject constructor( return TaskResult(result == 0, "Error while compiling") } - private fun invokeCompilerDirectly(projectName: String, outputDir: String?, classpathString: String, - sourceFiles: List, friends: Array): TaskResult { + private fun invokeCompilerDirectly(projectName: String, outputDir: String?, info: CompilerActionInfo, + classpathString: String, filesToCompile: Int): TaskResult { + val sourceFiles = info.sourceFiles + val friends = info.friendPaths.toTypedArray() val args = K2JVMCompilerArguments().apply { moduleName = projectName destination = outputDir @@ -209,11 +222,64 @@ class KotlinCompiler @Inject constructor( // TODO: experimental should be removed as soon as it becomes standard System.setProperty("kotlin.incremental.compilation.experimental", "true") - val exitCode = K2JVMCompiler().exec(collector, Services.Builder().build(), args) - val result = TaskResult(exitCode == ExitCode.OK) + val result = + if (cliArgs.noIncrementalKotlin) { + log(2, "Kotlin incremental compilation is disabled") + val exitCode = K2JVMCompiler().exec(collector, Services.Builder().build(), args) + TaskResult(exitCode == ExitCode.OK) + } else { + log(2, "Kotlin incremental compilation is enabled") + compileIncrementally(filesToCompile, sourceFiles, outputDir, info, args, collector) + TaskResult() + } return result } + private fun compileIncrementally(filesToCompile: Int, sourceFiles: List, outputDir: String?, + info: CompilerActionInfo, + args: K2JVMCompilerArguments, + collector: MessageCollector) { + val compiledFiles = arrayListOf() + val reporter = object : ICReporter { + override fun pathsAsString(files: Iterable): String { + return files.joinToString { it.absolutePath } +// return files.joinToString { it.relativeTo(workingDir).path } + } + + override fun report(message: () -> String) { + log(2, " ICReport: ${message()}") + } + + override fun reportCompileIteration(sourceFiles: Collection, exitCode: ExitCode) { + log(2, " ICCompileIteration Compiled files: ${pathsAsString(sourceFiles)}") + compiledFiles.addAll(sourceFiles) + } + } + incrementalCompile(sourceFiles, File(outputDir), info.forceRecompile, args, collector, reporter) + if (filesToCompile > compiledFiles.size) { + log(1, " Actual files that needed to be compiled: " + compiledFiles.size) + } + } + + /** + * Invoke the incremental compiler. + */ + fun incrementalCompile(sourceFiles: List, outputDirectory: File, forceRecompile: Boolean, + args: K2JVMCompilerArguments, + messageCollector: MessageCollector, reporter: ICReporter) { + // If asked to force recompile, create a brand new cachesDir, otherwise reuse the existing one + val cachesDir = + if (forceRecompile) Files.createTempDirectory("kobalt-").toFile() + else File(outputDirectory.parent, outputDirectory.name + "-ic-caches") + + val sourceRoots = sourceFiles.map(::File).map { if (it.isFile) it.parentFile else it }.toSet() + try { + makeIncrementally(cachesDir, sourceRoots, args, messageCollector, reporter) + } catch(ex: Exception) { + throw KobaltException(ex.message, ex) + } + } + /** * Invoke the Kotlin compiler by reflection to make sure we use the class defined * in the kotlin-embeddable jar file. At the time of this writing, the dokka fatJar @@ -310,7 +376,7 @@ class KotlinCompiler @Inject constructor( emptyList() } val info = CompilerActionInfo(project?.directory, dependencies, sourceFiles, listOf("kt"), outputDir, args, - friendPaths) + friendPaths, context?.forceRecompile ?: false) return jvmCompiler.doCompile(project, context, compilerAction, info, if (context != null) compilerUtils.sourceCompilerFlags(project, context, info) else emptyList()) diff --git a/src/test/kotlin/com/beust/kobalt/BaseTest.kt b/src/test/kotlin/com/beust/kobalt/BaseTest.kt index 77691bbc..9f82990d 100644 --- a/src/test/kotlin/com/beust/kobalt/BaseTest.kt +++ b/src/test/kotlin/com/beust/kobalt/BaseTest.kt @@ -8,7 +8,7 @@ import com.beust.kobalt.internal.KobaltPluginXml import com.beust.kobalt.internal.PluginInfo import com.beust.kobalt.internal.build.BuildFile import org.testng.annotations.BeforeClass -import java.io.File +import java.nio.file.Files import java.nio.file.Paths import java.util.* @@ -44,7 +44,8 @@ open class BaseTest(val compilerFactory: BuildFileCompiler.IFactory? = null) { * interfering with other tests. */ fun compileBuildFile(buildFileText: String, args: Args = Args()): BuildFileCompiler.FindProjectResult { - val tmpBuildFile = File.createTempFile("kobaltTest", "").apply { + val tmpBaseDir = Files.createTempDirectory("kobaltTest") + val tmpBuildFile = Files.createTempFile(tmpBaseDir, "kobaltTest", "").toFile().apply { deleteOnExit() writeText(buildFileText) } @@ -56,6 +57,7 @@ open class BaseTest(val compilerFactory: BuildFileCompiler.IFactory? = null) { val pluginInfo = PluginInfo(KobaltPluginXml(), null, null).apply { projectContributors.add(jvmCompilerPlugin) } - return compilerFactory!!.create(listOf(thisBuildFile), pluginInfo).compileBuildFiles(args) + return compilerFactory!!.create(listOf(thisBuildFile), pluginInfo).compileBuildFiles(args, + forceRecompile = true) } } \ No newline at end of file