Incremental compilation.

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/Args.kt

@@ -57,9 +57,6 @@ class Args {
     @Parameter(names = arrayOf("--noIncremental"), description = "Turn off incremental builds")
     var noIncremental: Boolean = false
 
-    @Parameter(names = arrayOf("--parallel"), description = "Build all the projects in parallel whenever possible")
-    var parallel: Boolean = true
-
     @Parameter(names = arrayOf("--plugins"), description = "Comma-separated list of plug-in Maven id's")
     var pluginIds: String? = null
 
@@ -82,6 +79,9 @@ class Args {
     @Parameter(names = arrayOf("--projectInfo"), description = "Display information about the current projects")
     var projectInfo: Boolean = false
 
+    @Parameter(names = arrayOf("--noIncrementalKotlin"), description = "Disable incremental Kotlin compilation")
+    var noIncrementalKotlin: Boolean = false
+
     @Parameter(names = arrayOf("--sequential"), description = "Build all the projects in sequence")
     var sequential: Boolean = false
 

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/CompilerActionInfo.kt

@@ -11,4 +11,5 @@ data class CompilerActionInfo(val directory: String?,
         val suffixesBeingCompiled: List<String>,
         val outputDir: File,
         val compilerArgs: List<String>,
-        val friendPaths: List<String>)
+        val friendPaths: List<String>,
+        val forceRecompile: Boolean)

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/api/KobaltContext.kt

@@ -19,6 +19,7 @@ import java.io.File
 class KobaltContext(val args: Args) {
     lateinit var variant: Variant
     val profiles = arrayListOf<String>()
+    var forceRecompile: Boolean = false
 
     init {
         args.profiles?.split(',')?.filterNotNull()?.forEach {

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/CompilerUtils.kt

@@ -167,7 +167,7 @@ class CompilerUtils @Inject constructor(val files: KFiles, val dependencyManager
         // Finally, alter the info with the compiler interceptors before returning it
         val initialActionInfo = CompilerActionInfo(projectDirectory.path, classpath, allSources,
                 sourceSuffixes, buildDirectory, emptyList() /* the flags will be provided by flag contributors */,
-                emptyList())
+                emptyList(), context.forceRecompile)
         val result = context.pluginInfo.compilerInterceptors.fold(initialActionInfo, { ai, interceptor ->
             interceptor.intercept(project, context, ai)
         })

modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/misc/KFiles.kt

@@ -5,14 +5,12 @@ import com.beust.kobalt.api.Kobalt
 import com.beust.kobalt.api.Project
 import com.beust.kobalt.internal.build.BuildFile
 import com.beust.kobalt.maven.Md5
-import java.io.File
+import java.io.*
-import java.io.IOException
-import java.io.InputStream
-import java.io.OutputStream
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
 import java.nio.file.StandardCopyOption
+import java.util.jar.JarInputStream
 
 class KFiles {
     /**
@@ -170,6 +168,30 @@ class KFiles {
             return result
         }
 
+        /**
+         * List the files contained in a directory or a jar file.
+         */
+        fun listFiles(file: File, block: (String) -> Unit) {
+            if (file.isDirectory) {
+                KFiles.findRecursively(file).forEach {
+                    block(it)
+                }
+            } else if (file.name.endsWith(".jar")) {
+                FileInputStream(file).use {
+                    JarInputStream(it).use { stream ->
+                        var entry = stream.nextJarEntry
+                        while (entry != null) {
+                            block(entry.name)
+                            entry = stream.nextJarEntry;
+                        }
+                    }
+                }
+
+            } else {
+                throw KobaltException("Can't list files of a file: " + file)
+            }
+        }
+
         fun copyRecursively(from: File, to: File, replaceExisting: Boolean = true, deleteFirst: Boolean = false,
                 onError: (File, IOException) -> OnErrorAction = { _, exception -> throw exception }) {
             // Need to wait until copyRecursively supports an overwrite: Boolean = false parameter

src/main/kotlin/com/beust/kobalt/app/BuildFileCompiler.kt

@@ -61,6 +61,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil
         context.resolver = resolver
         context.pomGeneratorFactory = pomGeneratorFactory
         context.logger = parallelLogger
+        context.forceRecompile = forceRecompile
         Kobalt.context = context
 
         // The list of dynamic plug-ins has to be a companion since it's modified directly from
@@ -71,7 +72,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil
         //
         // Find all the projects in the build file, possibly compiling them
         //
-        val projectResult = findProjects(context, forceRecompile)
+        val projectResult = findProjects(context)
 
         return projectResult
     }
@@ -81,7 +82,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil
     class FindProjectResult(val context: KobaltContext, val projects: List<Project>, val pluginUrls: List<URL>,
             val taskResult: TaskResult)
 
-    private fun findProjects(context: KobaltContext, forceRecompile: Boolean): FindProjectResult {
+    private fun findProjects(context: KobaltContext): FindProjectResult {
         var errorTaskResult: TaskResult? = null
         val projects = arrayListOf<Project>()
         buildFiles.forEach { buildFile ->
@@ -110,7 +111,7 @@ class BuildFileCompiler @Inject constructor(@Assisted("buildFiles") val buildFil
             KFiles.saveFile(modifiedBuildFile, parsedBuildFile.buildScriptCode)
             val taskResult = maybeCompileBuildFile(context, BuildFile(Paths.get(modifiedBuildFile.path),
                     "Modified ${Constants.BUILD_FILE_NAME}", buildFile.realPath),
-                    buildScriptJarFile, pluginUrls, forceRecompile)
+                    buildScriptJarFile, pluginUrls, context.forceRecompile)
             if (taskResult.success) {
                 projects.addAll(buildScriptUtil.runBuildScriptJarFile(buildScriptJarFile, pluginUrls, context))
             } else {

src/main/kotlin/com/beust/kobalt/app/BuildScriptUtil.kt

@@ -17,12 +17,9 @@ import com.beust.kobalt.misc.warn
 import com.beust.kobalt.plugin.KobaltPlugin
 import com.google.inject.Inject
 import java.io.File
-import java.io.FileInputStream
-import java.io.InputStream
 import java.lang.reflect.Modifier
 import java.net.URL
 import java.net.URLClassLoader
-import java.util.jar.JarInputStream
 
 class BuildScriptUtil @Inject constructor(val plugins: Plugins, val files: KFiles,
         val taskManager: TaskManager) {
@@ -37,7 +34,6 @@ class BuildScriptUtil @Inject constructor(val plugins: Plugins, val files: KFile
      */
     fun runBuildScriptJarFile(buildScriptJarFile: File, urls: List<URL>,
             context: KobaltContext) : List<Project> {
-        var stream : InputStream? = null
         // The jar files used to load the plug-ins are:
         // - all the plug-ins found in the build file
         // - kobalt's own jar file
@@ -55,13 +51,9 @@ class BuildScriptUtil @Inject constructor(val plugins: Plugins, val files: KFile
         //
         // Classload all the jar files and invoke their methods
         //
-        try {
+        if (buildScriptJarFile.exists()) {
-            stream = JarInputStream(FileInputStream(buildScriptJarFile))
-            var entry = stream.nextJarEntry
-
             val classes = hashSetOf<Class<*>>()
-            while (entry != null) {
+            KFiles.listFiles(buildScriptJarFile) { name ->
-                val name = entry.name;
                 if (name.endsWith(".class")) {
                     val className = name.substring(0, name.length - 6).replace("/", ".")
                     try {
@@ -75,7 +67,6 @@ class BuildScriptUtil @Inject constructor(val plugins: Plugins, val files: KFile
                         warn("Couldn't find class $className")
                     }
                 }
-                entry = stream.nextJarEntry;
             }
 
             // Invoke all the "val" found on the _DefaultPackage class (the Build.kt file)
@@ -103,10 +94,9 @@ class BuildScriptUtil @Inject constructor(val plugins: Plugins, val files: KFile
                             taskManager.addIncrementalTask(defaultPlugin, method, it)
                         }
 
-                    }}
+                    }
+                }
             }
-        } finally {
-            stream?.close()
         }
 
         validateProjectNames(projects)

src/main/kotlin/com/beust/kobalt/plugin/apt/AptPlugin.kt

@@ -91,7 +91,7 @@ class AptPlugin @Inject constructor(val dependencyManager: DependencyManager, va
             val dependencies = dependencyManager.calculateDependencies(project, context)
             val info = CompilerActionInfo(sourceDir.absolutePath, dependencies,
                     listOf(javaFile.absolutePath), listOf("java"), File(sourceDir, "generated"),
-                    listOf(), listOf())
+                    listOf(), listOf(), context.forceRecompile)
 
             val results = compilerUtils.invokeCompiler(project, context, javaCompiler, info)
         }

src/main/kotlin/com/beust/kobalt/plugin/kotlin/KotlinCompiler.kt

@@ -13,7 +13,10 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
 import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.incremental.ICReporter
+import org.jetbrains.kotlin.incremental.makeIncrementally
 import java.io.File
+import java.nio.file.Files
 import java.util.*
 import javax.inject.Inject
 import javax.inject.Singleton
@@ -26,6 +29,7 @@ import kotlin.properties.Delegates
 @Singleton
 class KotlinCompiler @Inject constructor(
         val files: KFiles,
+        val cliArgs: Args,
         val dependencyManager: DependencyManager,
         val executors: KobaltExecutors,
         val settings: KobaltSettings,
@@ -37,10 +41,17 @@ class KotlinCompiler @Inject constructor(
         override fun compile(project: Project?, info: CompilerActionInfo): TaskResult {
             val projectName = project?.name
             val version = settings.kobaltCompilerVersion
+            var filesToCompile = 0
             if (! info.outputDir.path.endsWith("ript.jar")) {
                 // Don't display the message if compiling Build.kt
-                kobaltLog.log(projectName!!, 1,
+                filesToCompile =
-                        "  Kotlin $version compiling " + Strings.pluralizeAll(info.sourceFiles.size, "file"))
+                        info.sourceFiles.map(::File).map {
+                            if (it.isDirectory) KFiles.findRecursively(it).size else 1
+                        }.reduce { a, b ->
+                            a + b
+                        }
+                kobaltLog.log(projectName ?: "", 1,
+                        "  Kotlin $version compiling " + Strings.pluralizeAll(filesToCompile, "file"))
             }
             val cp = compilerFirst(info.dependencies.map { it.jarFile.get() })
             val infoDir = info.directory
@@ -79,7 +90,7 @@ class KotlinCompiler @Inject constructor(
 
             } else {
                 return invokeCompilerDirectly(projectName ?: "kobalt-" + Random().nextInt(), outputDir,
-                        classpath, info.sourceFiles, info.friendPaths.toTypedArray())
+                        info, classpath, filesToCompile)
             }
         }
 
@@ -120,8 +131,10 @@ class KotlinCompiler @Inject constructor(
             return TaskResult(result == 0, "Error while compiling")
         }
 
-        private fun invokeCompilerDirectly(projectName: String, outputDir: String?, classpathString: String,
+        private fun invokeCompilerDirectly(projectName: String, outputDir: String?, info: CompilerActionInfo,
-                sourceFiles: List<String>, friends: Array<String>): TaskResult {
+                classpathString: String, filesToCompile: Int): TaskResult {
+            val sourceFiles = info.sourceFiles
+            val friends = info.friendPaths.toTypedArray()
             val args = K2JVMCompilerArguments().apply {
                 moduleName = projectName
                 destination = outputDir
@@ -209,11 +222,64 @@ class KotlinCompiler @Inject constructor(
             // TODO: experimental should be removed as soon as it becomes standard
             System.setProperty("kotlin.incremental.compilation.experimental", "true")
 
-            val exitCode = K2JVMCompiler().exec(collector, Services.Builder().build(), args)
+            val result =
-            val result = TaskResult(exitCode == ExitCode.OK)
+                if (cliArgs.noIncrementalKotlin) {
+                    log(2, "Kotlin incremental compilation is disabled")
+                    val exitCode = K2JVMCompiler().exec(collector, Services.Builder().build(), args)
+                    TaskResult(exitCode == ExitCode.OK)
+                } else {
+                    log(2, "Kotlin incremental compilation is enabled")
+                    compileIncrementally(filesToCompile, sourceFiles, outputDir, info, args, collector)
+                    TaskResult()
+                }
             return result
         }
 
+        private fun compileIncrementally(filesToCompile: Int, sourceFiles: List<String>, outputDir: String?,
+                info: CompilerActionInfo,
+                args: K2JVMCompilerArguments,
+                collector: MessageCollector) {
+            val compiledFiles = arrayListOf<File>()
+            val reporter = object : ICReporter {
+                override fun pathsAsString(files: Iterable<File>): String {
+                    return files.joinToString { it.absolutePath }
+//                    return files.joinToString { it.relativeTo(workingDir).path }
+                }
+
+                override fun report(message: () -> String) {
+                    log(2, "    ICReport: ${message()}")
+                }
+
+                override fun reportCompileIteration(sourceFiles: Collection<File>, exitCode: ExitCode) {
+                    log(2, "    ICCompileIteration Compiled files: ${pathsAsString(sourceFiles)}")
+                    compiledFiles.addAll(sourceFiles)
+                }
+            }
+            incrementalCompile(sourceFiles, File(outputDir), info.forceRecompile, args, collector, reporter)
+            if (filesToCompile > compiledFiles.size) {
+                log(1, "  Actual files that needed to be compiled: " + compiledFiles.size)
+            }
+        }
+
+        /**
+         * Invoke the incremental compiler.
+         */
+        fun incrementalCompile(sourceFiles: List<String>, outputDirectory: File, forceRecompile: Boolean,
+                args: K2JVMCompilerArguments,
+                messageCollector: MessageCollector, reporter: ICReporter) {
+            // If asked to force recompile, create a brand new cachesDir, otherwise reuse the existing one
+            val cachesDir =
+                if (forceRecompile) Files.createTempDirectory("kobalt-").toFile()
+                else File(outputDirectory.parent, outputDirectory.name + "-ic-caches")
+
+            val sourceRoots = sourceFiles.map(::File).map { if (it.isFile) it.parentFile else it }.toSet()
+            try {
+                makeIncrementally(cachesDir, sourceRoots, args, messageCollector, reporter)
+            } catch(ex: Exception) {
+                throw KobaltException(ex.message, ex)
+            }
+        }
+
         /**
          * Invoke the Kotlin compiler by reflection to make sure we use the class defined
          * in the kotlin-embeddable jar file. At the time of this writing, the dokka fatJar
@@ -310,7 +376,7 @@ class KotlinCompiler @Inject constructor(
                 emptyList<String>()
             }
         val info = CompilerActionInfo(project?.directory, dependencies, sourceFiles, listOf("kt"), outputDir, args,
-                friendPaths)
+                friendPaths, context?.forceRecompile ?: false)
 
         return jvmCompiler.doCompile(project, context, compilerAction, info,
                 if (context != null) compilerUtils.sourceCompilerFlags(project, context, info) else emptyList())

src/test/kotlin/com/beust/kobalt/BaseTest.kt

@@ -8,7 +8,7 @@ import com.beust.kobalt.internal.KobaltPluginXml
 import com.beust.kobalt.internal.PluginInfo
 import com.beust.kobalt.internal.build.BuildFile
 import org.testng.annotations.BeforeClass
-import java.io.File
+import java.nio.file.Files
 import java.nio.file.Paths
 import java.util.*
 
@@ -44,7 +44,8 @@ open class BaseTest(val compilerFactory: BuildFileCompiler.IFactory? = null) {
      * interfering with other tests.
      */
     fun compileBuildFile(buildFileText: String, args: Args = Args()): BuildFileCompiler.FindProjectResult {
-        val tmpBuildFile = File.createTempFile("kobaltTest", "").apply {
+        val tmpBaseDir = Files.createTempDirectory("kobaltTest")
+        val tmpBuildFile = Files.createTempFile(tmpBaseDir, "kobaltTest", "").toFile().apply {
             deleteOnExit()
             writeText(buildFileText)
         }
@@ -56,6 +57,7 @@ open class BaseTest(val compilerFactory: BuildFileCompiler.IFactory? = null) {
         val pluginInfo = PluginInfo(KobaltPluginXml(), null, null).apply {
             projectContributors.add(jvmCompilerPlugin)
         }
-        return compilerFactory!!.create(listOf(thisBuildFile), pluginInfo).compileBuildFiles(args)
+        return compilerFactory!!.create(listOf(thisBuildFile), pluginInfo).compileBuildFiles(args,
+                forceRecompile = true)
     }
 }