1
0
Fork 0
mirror of https://github.com/ethauvin/kobalt.git synced 2025-04-26 00:17:11 -07:00

Merge branch 'master' of github.com:cbeust/kobalt

This commit is contained in:
Cedric Beust 2015-10-14 19:03:34 -07:00
commit 05c6904f74
16 changed files with 154 additions and 155 deletions

View file

@ -1 +1 @@
kobalt.version=0.183
kobalt.version=0.189

View file

@ -24,9 +24,16 @@ private fun parseArgs(argv: Array<String>): Main.RunInfo {
}
public fun main(argv: Array<String>) {
val result = mainNoExit(argv)
if (result != 0) {
System.exit(result)
}
}
public fun mainNoExit(argv: Array<String>) : Int {
val (jc, args) = parseArgs(argv)
Kobalt.INJECTOR = Guice.createInjector(MainModule(args))
Kobalt.INJECTOR.getInstance(Main::class.java).run(jc, args)
return Kobalt.INJECTOR.getInstance(Main::class.java).run(jc, args)
}
private class Main @Inject constructor(
@ -44,13 +51,13 @@ private class Main @Inject constructor(
data class RunInfo(val jc: JCommander, val args: Args)
public fun run(jc: JCommander, args: Args) {
public fun run(jc: JCommander, args: Args) : Int {
var result = 0
val latestVersionFuture = github.latestKobaltVersion
benchmark("Build", {
println(Banner.get() + Kobalt.version + "\n")
// runTest()
runWithArgs(jc, args)
result = runWithArgs(jc, args)
executors.shutdown()
debug("All done")
})
@ -67,6 +74,7 @@ private class Main @Inject constructor(
log(1, it )
}
}
return result
}
public class Worker<T>(val runNodes: ArrayList<T>, val n: T) : IWorker<T> {
@ -94,7 +102,8 @@ private class Main @Inject constructor(
private val SCRIPT_JAR = "buildScript.jar"
private fun runWithArgs(jc: JCommander, args: Args) {
private fun runWithArgs(jc: JCommander, args: Args) : Int {
var result = 0
val p = if (args.buildFile != null) File(args.buildFile) else findBuildFile()
args.buildFile = p.absolutePath
val buildFile = BuildFile(Paths.get(p.absolutePath), p.name)
@ -154,10 +163,14 @@ private class Main @Inject constructor(
//
// Launch the build
//
taskManager.runTargets(args.targets, allProjects)
val thisResult = taskManager.runTargets(args.targets, allProjects)
if (result == 0) {
result = thisResult
}
}
}
}
return result
}
private fun findBuildFile(): File {

View file

@ -4,7 +4,7 @@ import com.beust.kobalt.misc.NamedThreadFactory
import com.beust.kobalt.misc.ToString
import com.beust.kobalt.misc.log
import com.google.common.collect.ArrayListMultimap
import java.util.*
import com.google.common.collect.HashMultimap
import java.util.concurrent.*
open class TaskResult2<T>(success: Boolean, val value: T) : TaskResult(success) {
@ -40,13 +40,17 @@ public class DynamicGraphExecutor<T>(val graph: DynamicGraph<T>,
val executor = Executors.newFixedThreadPool(5, NamedThreadFactory("DynamicGraphExecutor"))
val completion = ExecutorCompletionService<TaskResult2<T>>(executor)
public fun run() {
/**
* @return 0 if all went well, > 0 otherwise
*/
public fun run() : Int {
var lastResult = TaskResult()
while (graph.freeNodes.size() > 0) {
log(3, "Current count: ${graph.nodeCount}")
synchronized(graph) {
val freeNodes = graph.freeNodes
freeNodes.forEach { graph.setStatus(it, DynamicGraph.Status.RUNNING)}
log(3, "submitting free nodes ${freeNodes}")
log(3, "submitting free nodes $freeNodes")
val callables : List<IWorker<T>> = factory.createWorkers(freeNodes)
callables.forEach { completion.submit(it) }
var n = callables.size()
@ -56,7 +60,8 @@ public class DynamicGraphExecutor<T>(val graph: DynamicGraph<T>,
try {
val future = completion.take()
val taskResult = future.get(2, TimeUnit.SECONDS)
log(3, "Received task result ${taskResult}")
lastResult = taskResult
log(3, "Received task result $taskResult")
n--
graph.setStatus(taskResult.value,
if (taskResult.success) {
@ -71,6 +76,7 @@ public class DynamicGraphExecutor<T>(val graph: DynamicGraph<T>,
}
}
executor.shutdown()
return if (lastResult.success) 0 else 1
}
}
@ -78,21 +84,19 @@ public class DynamicGraphExecutor<T>(val graph: DynamicGraph<T>,
* Representation of the graph of methods.
*/
public class DynamicGraph<T> {
private val DEBUG = false
private val nodesReady = linkedSetOf<T>()
private val nodesRunning = linkedSetOf<T>()
private val nodesFinished = linkedSetOf<T>()
private val nodesInError = linkedSetOf<T>()
private val nodesSkipped = linkedSetOf<T>()
private val dependedUpon = ArrayListMultimap.create<T, T>()
private val dependingOn = ArrayListMultimap.create<T, T>()
private val dependedUpon = HashMultimap.create<T, T>()
private val dependingOn = HashMultimap.create<T, T>()
/**
* Define a comparator for the nodes of this graph, which will be used
* to order the free nodes when they are asked.
*/
public val comparator : Comparator<T>? = null
// public val comparator : Comparator<T>? = null
enum class Status {
READY, RUNNING, FINISHED, ERROR, SKIPPED
@ -146,14 +150,14 @@ public class DynamicGraph<T> {
// }
// }
log(3, "freeNodes: ${result}")
log(3, "freeNodes: $result")
return result
}
/**
* @return a list of all the nodes that have a status other than FINISHED.
*/
private fun getUnfinishedNodes(nodes: List<T>) : Collection<T> {
private fun getUnfinishedNodes(nodes: Set<T>) : Collection<T> {
val result = hashSetOf<T>()
nodes.forEach { node ->
if (nodesReady.contains(node) || nodesRunning.contains(node)) {
@ -176,7 +180,7 @@ public class DynamicGraph<T> {
private fun setSkipStatus(node: T, status: Status) {
dependingOn.get(node).forEach {
if (! nodesSkipped.contains(it)) {
log(3, "Node skipped: ${it}")
log(3, "Node skipped: $it")
nodesSkipped.add(it)
nodesReady.remove(it)
setSkipStatus(it, status)
@ -194,7 +198,7 @@ public class DynamicGraph<T> {
Status.RUNNING -> nodesRunning.add(node)
Status.FINISHED -> nodesFinished.add(node)
Status.ERROR -> {
log(3, "Node in error: ${node}")
log(3, "Node in error: $node")
nodesReady.remove(node)
nodesInError.add(node)
setSkipStatus(node, status)

View file

@ -43,8 +43,8 @@ public class TaskManager @Inject constructor(val plugins: Plugins, val args: Arg
fun matches(projectName: String) = project == null || project == projectName
}
public fun runTargets(targets: List<String>, projects: List<Project>) {
val tasksAlreadyRun = hashSetOf<String>()
public fun runTargets(targets: List<String>, projects: List<Project>) : Int {
var result = 0
projects.forEach { project ->
val projectName = project.name!!
val tasksByNames = hashMapOf<String, PluginTask>()
@ -58,8 +58,8 @@ public class TaskManager @Inject constructor(val plugins: Plugins, val args: Arg
log(1, " Building project ${project.name}")
log(1, "")
val graph = DynamicGraph<PluginTask>()
targets.forEach { target ->
val graph = DynamicGraph<PluginTask>()
val ti = TaskInfo(target)
if (ti.matches(projectName)) {
@ -83,10 +83,7 @@ public class TaskManager @Inject constructor(val plugins: Plugins, val args: Arg
if (currentFreeTask.size() == 1) {
currentFreeTask.get(0).let {
val thisTaskInfo = TaskInfo(projectName, it.name)
if (! tasksAlreadyRun.contains(thisTaskInfo.id)) {
graph.addNode(it)
tasksAlreadyRun.add(thisTaskInfo.id)
}
graph.addNode(it)
}
}
@ -99,11 +96,7 @@ public class TaskManager @Inject constructor(val plugins: Plugins, val args: Arg
rb.forEach {
val to = tasksByNames.get(it)
if (to != null) {
val taskInfo = TaskInfo(projectName, to.name)
if (! tasksAlreadyRun.contains(taskInfo.id)) {
graph.addEdge(pluginTask, to)
tasksAlreadyRun.add(taskInfo.id)
}
graph.addEdge(pluginTask, to)
} else {
throw KobaltException("Should have found $it")
}
@ -124,27 +117,32 @@ public class TaskManager @Inject constructor(val plugins: Plugins, val args: Arg
}
}
}
log(2, "About to run graph:\n ${graph.dump()} ")
//
// Now that we have a full graph, run it
//
val factory = object : IThreadWorkerFactory<PluginTask> {
override public fun createWorkers(nodes: List<PluginTask>): List<IWorker<PluginTask>> {
val result = arrayListOf<IWorker<PluginTask>>()
nodes.forEach {
result.add(TaskWorker(arrayListOf(it), args.dryRun))
}
return result
}
}
val executor = DynamicGraphExecutor(graph, factory)
executor.run()
}
}
//
// Now that we have a full graph, run it
//
log(2, "About to run graph:\n ${graph.dump()} ")
val factory = object : IThreadWorkerFactory<PluginTask> {
override public fun createWorkers(nodes: List<PluginTask>): List<IWorker<PluginTask>> {
val result = arrayListOf<IWorker<PluginTask>>()
nodes.forEach {
result.add(TaskWorker(arrayListOf(it), args.dryRun))
}
return result
}
}
val executor = DynamicGraphExecutor(graph, factory)
val thisResult = executor.run()
if (result == 0) {
result = thisResult
}
}
return result
}
/**

View file

@ -55,6 +55,7 @@ public class Http {
.put(RequestBody.create(MEDIA_TYPE_BINARY, file))
.build()
log(2, "Uploading $file to $url")
val response = OkHttpClient().newCall(request).execute()
if (! response.isSuccessful) {
error(response)

View file

@ -57,6 +57,6 @@ public class PomGenerator @Inject constructor(@Assisted val project: Project) {
val pomFile = SimpleDep(project.group!!, project.artifactId!!, project.version!!).toPomFileName()
val outputFile = File(outputDir, pomFile)
outputFile.writeText(s.toString(), Charset.defaultCharset())
log(1, "Wrote $outputFile")
log(1, " Wrote $outputFile")
}
}

View file

@ -11,7 +11,7 @@ class NamedThreadFactory(val n: String) : ThreadFactory {
override
public fun newThread(r: Runnable) : Thread {
val result = Thread(r)
result.setName(name + "-" + result.getId())
result.name = name + "-" + result.id
return result
}
}
@ -25,7 +25,7 @@ class KobaltExecutor(name: String, threadCount: Int)
var ex : Throwable? = null
if (t == null && r is Future<*>) {
try {
if (r.isDone()) r.get();
if (r.isDone) r.get();
} catch (ce: CancellationException) {
ex = ce;
} catch (ee: ExecutionException) {
@ -52,8 +52,8 @@ public class KobaltExecutors {
miscExecutor.shutdown()
}
fun <T> completionService(name: String, threadCount: Int,
maxMs: Long, tasks: List<Callable<T>>) : List<T> {
fun <T> completionService(name: String, threadCount: Int, maxMs: Long, tasks: List<Callable<T>>,
progress: (T) -> Unit = {}) : List<T> {
val result = arrayListOf<T>()
val executor = newExecutor(name, threadCount)
val cs = ExecutorCompletionService<T>(executor)
@ -64,6 +64,7 @@ public class KobaltExecutors {
while (i < tasks.size() && remainingMs >= 0) {
var start = System.currentTimeMillis()
val r = cs.take().get(remainingMs, TimeUnit.MILLISECONDS)
progress(r)
result.add(r)
remainingMs -= (System.currentTimeMillis() - start)
log(2, "Received $r, remaining: $remainingMs ms")

View file

@ -4,54 +4,9 @@ import com.beust.kobalt.api.Kobalt
import java.text.SimpleDateFormat
import java.util.*
interface KobaltLogger {
companion object {
public var LOG_LEVEL : Int = 1
val logger : Logger get() =
if (Kobalt.context != null) {
Logger(Kobalt.context!!.args.dev)
} else {
Logger(false)
}
fun log(level: Int, s: String) {
if (level <= LOG_LEVEL) {
logger.log("Logger", s)
}
}
fun warn(s: String, e: Throwable? = null) {
logger.warn("Logger", s, e)
}
fun debug(s: String) {
logger.debug(s)
}
}
final fun log(level: Int = 1, message: String) {
if (level <= LOG_LEVEL) {
logger.log("Logger", message)
}
}
final fun debug(message: String) {
logger.debug(message)
}
final fun error(message: String, e: Throwable? = null) {
logger.error("Logger", "***** $message", e)
}
final fun warn(message: String, e: Throwable? = null) {
logger.warn("Logger", message, e)
}
}
fun Any.log(level: Int, text: String) {
fun Any.log(level: Int, text: String, newLine : Boolean = true) {
if (level <= KobaltLogger.LOG_LEVEL) {
KobaltLogger.logger.log(javaClass.simpleName, text)
KobaltLogger.logger.log(javaClass.simpleName, text, newLine)
}
}
@ -67,6 +22,19 @@ fun Any.error(text: String, e: Throwable? = null) {
KobaltLogger.logger.error(javaClass.simpleName, text, e)
}
interface KobaltLogger {
companion object {
public var LOG_LEVEL: Int = 1
val logger: Logger get() =
if (Kobalt.context != null) {
Logger(Kobalt.context!!.args.dev)
} else {
Logger(false)
}
}
}
class Logger(val dev: Boolean) {
val FORMAT = SimpleDateFormat("HH:mm:ss.SSS")
@ -88,6 +56,9 @@ class Logger(val dev: Boolean) {
final fun warn(tag: String, message: String, e: Throwable? = null) =
println(getPattern("W", "***** WARNING ", tag, message))
final fun log(tag: String, message: String) =
println(getPattern("L", "", tag, message))
final fun log(tag: String, message: String, newLine: Boolean) =
with(getPattern("L", "", tag, message)) {
if (newLine) println(this)
else print("\r" + this)
}
}

View file

@ -65,8 +65,8 @@ public class JavaPlugin @Inject constructor(
// pb.redirectError(File("/tmp/kobalt-err"))
// pb.redirectOutput(File("/tmp/kobalt-out"))
val line = args.join(" ")
log(1, "Compiling ${sourceFiles.size()} files with classpath size ${cpList.size()}")
log(2, "Compiling ${project}:\n${line}")
log(1, " Compiling ${sourceFiles.size()} files with classpath size ${cpList.size()}")
log(2, " Compiling $project:\n$line")
val process = pb.start()
val errorCode = process.waitFor()

View file

@ -6,6 +6,7 @@ import com.beust.kobalt.internal.TaskResult
import com.beust.kobalt.maven.*
import com.beust.kobalt.misc.KobaltExecutors
import com.beust.kobalt.misc.log
import org.jetbrains.kotlin.cli.common.CLICompiler
import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
import java.io.File
import javax.inject.Inject
@ -52,7 +53,7 @@ class KotlinCompiler @Inject constructor(override val localRepo : LocalRepo,
validateClasspath(classpathList)
log(2, "Compiling ${source.size()} files with classpath:\n " + classpathList.join("\n "))
K2JVMCompiler.main(arrayOf(
CLICompiler.doMainNoExit(K2JVMCompiler(), arrayOf(
"-d", output,
"-classpath", classpathList.join(File.pathSeparator), *source.toTypedArray(),
*args.toTypedArray()))

View file

@ -83,7 +83,7 @@ public class KotlinPlugin @Inject constructor(
outputDirectory: String): TaskResult {
File(outputDirectory).mkdirs()
log(1, "Compiling ${sources.size()} files with classpath size ${cpList.size()}")
log(1, " Compiling ${sources.size()} files with classpath size ${cpList.size()}")
return kotlinCompilePrivate {
classpath(cpList.map { it.jarFile.get().absolutePath })

View file

@ -218,7 +218,7 @@ public class PackagingPlugin @Inject constructor(val dependencyManager : Depende
log(2, "Added ${includedFiles.size()} files to ${result}")
outStream.flush()
outStream.close()
log(1, "Created ${result}")
log(1, " Created ${result}")
return result
}

View file

@ -44,7 +44,7 @@ open public class UnauthenticatedJCenterApi @Inject constructor(open val http: H
public class JCenterApi @Inject constructor (@Nullable @Assisted("username") val username: String?,
@Nullable @Assisted("password") val password: String?,
override val http: Http, val gpg: Gpg) : UnauthenticatedJCenterApi(http) {
override val http: Http, val gpg: Gpg, val executors: KobaltExecutors) : UnauthenticatedJCenterApi(http) {
interface IFactory {
fun create(@Nullable @Assisted("username") username: String?,
@ -100,8 +100,6 @@ public class JCenterApi @Inject constructor (@Nullable @Assisted("username") val
private fun upload(files: List<File>, configuration : JCenterConfiguration?, fileToPath: (File) -> String,
generateMd5: Boolean = false, generateAsc: Boolean) : TaskResult {
val successes = arrayListOf<File>()
val failures = hashMapOf<File, String>()
val filesToUpload = arrayListOf<File>()
if (generateAsc) {
@ -127,37 +125,48 @@ public class JCenterApi @Inject constructor (@Nullable @Assisted("username") val
val options = arrayListOf<String>()
if (configuration?.publish == true) options.add("publish=1")
//
// TODO: These files should be uploaded from a thread pool instead of serially
//
log(1, "Found ${filesToUpload.size()} artifacts to upload")
filesToUpload.forEach {
var path = fileToPath(it)
if (options.size() > 0) {
path += "?" + options.join("&")
}
log(1, " Uploading $it to $path")
http.uploadFile(username, password, path, it,
{ r: Response -> successes.add(it) },
{ r: Response ->
val jo = parseResponse(r.body().string())
failures.put(it, jo.string("message") ?: "No message found")
})
val optionPath = StringBuffer()
if (options.size() > 0) {
optionPath.append("?" + options.join("&"))
}
val result: TaskResult
if (successes.size() == filesToUpload.size()) {
log(1, "All artifacts successfully uploaded")
result = TaskResult(true)
//
// Uploads can'be done in parallel or JCenter rejects them
//
val fileCount = filesToUpload.size()
if (fileCount > 0) {
log(1, " Found $fileCount artifacts to upload: " + filesToUpload.get(0)
+ if (fileCount > 1) "..." else "")
var i = 1
val errorMessages = arrayListOf<String>()
fun dots(total: Int, list: List<Boolean>) : String {
val spaces : String = Array(total - list.size(), { " " }).join("")
return "|" + list.map { if (it) "." else "X" }.join("") + spaces + "|"
}
val results = arrayListOf<Boolean>()
filesToUpload.forEach { file ->
http.uploadFile(username, password, fileToPath(file) + optionPath, file,
{ r: Response -> results.add(true)},
{ r: Response ->
results.add(false)
val jo = parseResponse(r.body().string())
errorMessages.add(jo.string("message") ?: "No message found")
})
val end = if (i >= fileCount) "\n" else ""
log(1, " Uploading " + (i++) + " / $fileCount " + dots(fileCount, results) + end, false)
}
if (errorMessages.isEmpty()) {
return TaskResult()
} else {
error("Errors while uploading:\n" + errorMessages.map { " $it" }.join("\n"))
return TaskResult(false, errorMessages.join("\n"))
}
} else {
result = TaskResult(false, failures.values().join(" "))
error("Failed to upload ${failures.size()} files:")
failures.forEach{ entry ->
error(" - ${entry.key} : ${entry.value}")
}
warn("Found no artifacts to upload")
return TaskResult()
}
return result
}
}

View file

@ -55,7 +55,7 @@ public class Wrapper {
if (url != null) {
readProperties(result, url.openConnection().inputStream)
} else {
throw IllegalArgumentException("Couldn't find ${KOBALT_PROPERTIES}")
throw IllegalArgumentException("Couldn't find $KOBALT_PROPERTIES")
}
return result
@ -64,7 +64,7 @@ public class Wrapper {
private fun initWrapperFile(version: String) {
val config = File(WRAPPER_DIR, KOBALT_WRAPPER_PROPERTIES)
if (! config.exists()) {
KFiles.saveFile(config, "${PROPERTY_VERSION}=${version}")
KFiles.saveFile(config, "$PROPERTY_VERSION=$version")
}
properties.load(FileReader(config))
}
@ -84,16 +84,16 @@ public class Wrapper {
val version = properties.getProperty(PROPERTY_VERSION)
initWrapperFile(version)
log(2, "Wrapper version: ${wrapperVersion}")
log(2, "Wrapper version: $wrapperVersion")
val fileName = "${FILE_NAME}-${wrapperVersion}.zip"
val fileName = "$FILE_NAME-$wrapperVersion.zip"
File(KFiles.distributionsDir).mkdirs()
val localZipFile = Paths.get(KFiles.distributionsDir, fileName)
val zipOutputDir = KFiles.distributionsDir + "/" + wrapperVersion
val kobaltJarFile = Paths.get(zipOutputDir, "kobalt/wrapper/${FILE_NAME}-${wrapperVersion}.jar")
val kobaltJarFile = Paths.get(zipOutputDir, "kobalt/wrapper/$FILE_NAME-$wrapperVersion.jar")
if (!Files.exists(localZipFile) || !Files.exists(kobaltJarFile)) {
log(1, "Downloading ${fileName}")
val fullUrl = "${URL}/${fileName}"
log(1, "Downloading $fileName")
val fullUrl = "$URL/$fileName"
val body = Http().get(fullUrl)
if (body.code == 200) {
if (!Files.exists(localZipFile)) {
@ -105,9 +105,9 @@ public class Wrapper {
// the BufferedSource returned in the ResponseBody
Files.copy(ins, target)
}
log(2, "${localZipFile} downloaded, extracting it")
log(2, "$localZipFile downloaded, extracting it")
} else {
log(2, "${localZipFile} already exists, extracting it")
log(2, "$localZipFile already exists, extracting it")
}
//
@ -124,16 +124,16 @@ public class Wrapper {
entryFile.mkdirs()
} else {
val dest = Paths.get(zipOutputDir, entryFile.path)
log(2, " Writing ${entry.name} to ${dest}")
log(2, " Writing ${entry.name} to $dest")
Files.createDirectories(dest.parent)
Files.copy(zipFile.getInputStream(entry),
dest,
java.nio.file.StandardCopyOption.REPLACE_EXISTING)
}
}
log(2, "${localZipFile} extracted")
log(2, "$localZipFile extracted")
} else {
error("Couldn't download ${URL}")
error("Couldn't download $URL")
}
}
@ -141,7 +141,7 @@ public class Wrapper {
// Copy the wrapper files in the current kobalt/wrapper directory
//
log(2, "Copying the wrapper files...")
arrayListOf(KOBALTW, "kobalt/wrapper/${FILE_NAME}-wrapper.jar").forEach {
arrayListOf(KOBALTW, "kobalt/wrapper/$FILE_NAME-wrapper.jar").forEach {
val from = Paths.get(zipOutputDir, it)
val to = Paths.get(File(".").absolutePath, it)
KFiles.copy(from, to, java.nio.file.StandardCopyOption.REPLACE_EXISTING)
@ -169,7 +169,7 @@ public class Wrapper {
args.addAll(argv)
val pb = ProcessBuilder(args)
pb.inheritIO()
log(1, "Launching\n ${args.join(" ")}")
log(2, "Launching\n ${args.join(" ")}")
val process = pb.start()
process.waitFor()
}

View file

@ -1 +1 @@
kobalt.version=0.183
kobalt.version=0.189

View file

@ -3,6 +3,7 @@ package com.beust.kobalt.internal
import com.beust.kobalt.maven.KobaltException
import com.beust.kobalt.misc.KobaltLogger
import com.beust.kobalt.misc.Topological
import com.beust.kobalt.misc.log
import javafx.concurrent.Worker
import org.testng.Assert
import org.testng.annotations.Test