diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/DG.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/DG.kt index 3f47c66c..9a2f2646 100644 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/DG.kt +++ b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/DG.kt @@ -1,6 +1,7 @@ package com.beust.kobalt.internal import com.beust.kobalt.KobaltException +import com.beust.kobalt.TaskResult import com.beust.kobalt.misc.NamedThreadFactory import com.beust.kobalt.misc.error import com.beust.kobalt.misc.log @@ -9,6 +10,10 @@ import java.lang.reflect.InvocationTargetException import java.util.* import java.util.concurrent.* +open class TaskResult2(success: Boolean, errorMessage: String?, val value: T) : TaskResult(success, errorMessage) { + override fun toString() = com.beust.kobalt.misc.toString("TaskResult", "value", value, "success", success) +} + class Node(val value: T) { override fun hashCode() = value!!.hashCode() override fun equals(other: Any?) : Boolean { diff --git a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/DynamicGraph.kt b/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/DynamicGraph.kt deleted file mode 100644 index ffb01624..00000000 --- a/modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/DynamicGraph.kt +++ /dev/null @@ -1,288 +0,0 @@ -package com.beust.kobalt.internal - -import com.beust.kobalt.KobaltException -import com.beust.kobalt.TaskResult -import com.beust.kobalt.misc.* -import com.google.common.collect.HashMultimap -import java.lang.reflect.InvocationTargetException -import java.util.concurrent.* - -open class TaskResult2(success: Boolean, errorMessage: String?, val value: T) : TaskResult(success, errorMessage) { - override fun toString() = toString("TaskResult", "value", value, "success", success) -} - -interface IWorker : Callable> { - /** - * @return list of tasks this worker is working on. - */ - // val tasks : List - - /** - * @return the priority of this task. - */ - val priority : Int -} - -interface IThreadWorkerFactory { - - /** - * Creates {@code IWorker} for specified set of tasks. It is not necessary that - * number of workers returned be same as number of tasks entered. - * - * @param nodes tasks that need to be executed - * @return list of workers - */ - fun createWorkers(nodes: List) : List> -} - -class DynamicGraphExecutor(val graph: DynamicGraph, - val factory: IThreadWorkerFactory) { - val executor = Executors.newFixedThreadPool(5, NamedThreadFactory("DynamicGraphExecutor")) - val completion = ExecutorCompletionService>(executor) - - /** - * @return 0 if all went well, > 0 otherwise - */ - fun run() : Int { - var lastResult = TaskResult() - var gotError = false - var nodesRunning = 0 - while (graph.freeNodes.size > 0 && ! gotError) { - log(3, "Current node count: ${graph.nodeCount}") - synchronized(graph) { - val freeNodes = graph.freeNodes - freeNodes.forEach { graph.setStatus(it, DynamicGraph.Status.RUNNING)} - log(3, " ==> Submitting " + freeNodes) - val callables : List> = factory.createWorkers(freeNodes) - callables.forEach { completion.submit(it) } - nodesRunning += callables.size - - // When a callable ends, see if it freed a node. If not, keep looping - while (graph.nodesRunning.size > 0 && graph.freeNodes.size == 0 && ! gotError) { - try { - val future = completion.take() - val taskResult = future.get(2, TimeUnit.SECONDS) - lastResult = taskResult - log(3, " <== Received task result $taskResult") - graph.setStatus(taskResult.value, - if (taskResult.success) { - DynamicGraph.Status.FINISHED - } else { - DynamicGraph.Status.ERROR - }) - } catch(ex: TimeoutException) { - log(2, "Time out") - } catch(ex: Exception) { - val ite = ex.cause - if (ite is InvocationTargetException) { - if (ite.targetException is KobaltException) { - throw (ex.cause as InvocationTargetException).targetException - } else { - error("Error: ${ite.cause?.message}", ite.cause) - gotError = true - } - } else { - error("Error: ${ex.message}", ex) - gotError = true - } - } - } - } - } - executor.shutdown() - if (graph.freeNodes.size == 0 && graph.nodesReady.size > 0) { - if (KobaltLogger.LOG_LEVEL > 1) { - throw KobaltException("Couldn't find any free nodes but a few nodes still haven't run, there is " + - "a cycle in the dependencies.\n Nodes left: " + graph.dump(graph.nodesReady)) - } else { - error("Error during the build") - lastResult = TaskResult(false) - } - } - return if (lastResult.success) 0 else 1 - } -} - -/** - * Representation of the graph of methods. - */ -class DynamicGraph { - val nodesReady = linkedSetOf() - val nodesRunning = linkedSetOf() - private val nodesFinished = linkedSetOf() - private val nodesInError = linkedSetOf() - private val nodesSkipped = linkedSetOf() - private val dependedUpon = HashMultimap.create() - private val dependingOn = HashMultimap.create() - - /** - * Define a comparator for the nodes of this graph, which will be used - * to order the free nodes when they are asked. - */ -// val comparator : Comparator? = null - - enum class Status { - READY, RUNNING, FINISHED, ERROR, SKIPPED - } - - /** - * Add a node to the graph. - */ - fun addNode(value: T) : T { - nodes.add(value) - nodesReady.add(value) - return value - } - - /** - * Add an edge between two nodes, which don't have to already be in the graph - * (they will be added by this method). Makes "to" depend on "from". - */ - fun addEdge(from: T, to: T) { - log(3, "Node $from depends on $to") - nodes.add(from) - nodes.add(to) - val fromNode = addNode(from) - val toNode = addNode(to) - dependingOn.put(toNode, fromNode) - dependedUpon.put(fromNode, toNode) - } - - /** - * @return a set of all the nodes that don't depend on any other nodes. - */ - val freeNodes : List - get() { - val result = arrayListOf() - nodesReady.forEach { m -> - // A node is free if... - - val du = dependedUpon.get(m) - // - no other nodes depend on it - if (! dependedUpon.containsKey(m)) { - result.add(m) - } else if (getUnfinishedNodes(du).size == 0) { - result.add(m) - } - } - - // Sort the free nodes if requested (e.g. priorities) -// if (! result.isEmpty()) { -// if (comparator != null) { -// Collections.sort(result, comparator) -// debug("Nodes after sorting:" + result.get(0)) -// } -// } - - log(3, " freeNodes: $result") - return result - } - - /** - * @return a list of all the nodes that have a status other than FINISHED. - */ - private fun getUnfinishedNodes(nodes: Set) : Collection { - val result = hashSetOf() - nodes.forEach { node -> - if (nodesReady.contains(node) || nodesRunning.contains(node)) { - result.add(node); - } - } - return result; - } - - /** - * Set the status for a set of nodes. - */ - fun setStatus(nodes: Collection, status: Status) { - nodes.forEach { setStatus(it, status) } - } - - /** - * Mark all dependees of this node SKIPPED - */ - private fun setSkipStatus(node: T, status: Status) { - dependingOn.get(node).forEach { - if (! nodesSkipped.contains(it)) { - log(3, "Node skipped: $it") - nodesSkipped.add(it) - nodesReady.remove(it) - setSkipStatus(it, status) - } - } - } - - /** - * Set the status for a node. - */ - fun setStatus(node: T, status: Status) { - removeNode(node); - when(status) { - Status.READY -> nodesReady.add(node) - Status.RUNNING -> nodesRunning.add(node) - Status.FINISHED -> nodesFinished.add(node) - Status.ERROR -> { - log(3, "Node in error: $node") - nodesReady.remove(node) - nodesInError.add(node) - setSkipStatus(node, status) - } - else -> { - throw IllegalArgumentException() - } - } - } - - private fun removeNode(node: T) { - if (! nodesReady.remove(node)) { - if (! nodesRunning.remove(node)) { - nodesFinished.remove(node) - } - } - } - - /** - * @return the number of nodes in this graph. - */ - val nodeCount: Int - get() = nodesReady.size + nodesRunning.size + nodesFinished.size - - override fun toString() : String { - val result = StringBuilder("[DynamicGraph ") - result.append("\n Ready:" + nodesReady) - result.append("\n Running:" + nodesRunning) - result.append("\n Finished:" + nodesFinished) - result.append("\n Edges:\n") - // dependingOn.entrySet().forEach { es -> - // result.append(" " + es.getKey() + "\n"); - // es.getValue().forEach { t -> - // result.append(" " + t + "\n"); - // } - // } - result.append("]"); - return result.toString(); - } - - val nodes = hashSetOf() - - fun dump(nodes: Collection) : String { - val result = StringBuffer() - result.append("************ Graph dump ***************\n") - val free = arrayListOf() - nodes.forEach { node -> - val d = dependedUpon.get(node) - if (d == null || d.isEmpty()) { - free.add(node) - } - } - - result.append("Free nodes: $free").append("\n Dependent nodes:\n") - nodes.forEach { - result.append(" $it -> ${dependedUpon.get(it)}\n") - } - return result.toString() - } - - fun dump() = dump(nodesReady) -} - diff --git a/src/test/kotlin/com/beust/kobalt/internal/DynamicGraphTest.kt b/src/test/kotlin/com/beust/kobalt/internal/DynamicGraphTest.kt index dc72abcb..7e576bda 100644 --- a/src/test/kotlin/com/beust/kobalt/internal/DynamicGraphTest.kt +++ b/src/test/kotlin/com/beust/kobalt/internal/DynamicGraphTest.kt @@ -14,17 +14,17 @@ class DynamicGraphTest { Assert.assertEquals(h, e) } - private fun createFactory(runNodes: ArrayList, errorFunction: (T) -> Boolean) : IThreadWorkerFactory { - return object: IThreadWorkerFactory { - override fun createWorkers(nodes: List): List> { - val result = arrayListOf>() + private fun createFactory(runNodes: ArrayList, errorFunction: (T) -> Boolean) : IThreadWorkerFactory2 { + return object: IThreadWorkerFactory2 { + override fun createWorkers(nodes: Collection): List> { + val result = arrayListOf>() nodes.forEach { result.add(Worker(runNodes, it, errorFunction)) } return result } } } - class Worker(val runNodes: ArrayList, val n: T, val errorFunction: (T) -> Boolean) : IWorker { + class Worker(val runNodes: ArrayList, val n: T, val errorFunction: (T) -> Boolean) : IWorker2 { override val priority = 0 override fun call() : TaskResult2 { @@ -36,14 +36,14 @@ class DynamicGraphTest { @Test fun testExecutor() { - val dg = DynamicGraph(); + val dg = DG(); dg.addEdge("compile", "runApt") dg.addEdge("compile", "generateVersion") val runNodes = arrayListOf() val factory = createFactory(runNodes, { true }) - DynamicGraphExecutor(dg, factory).run() + DGExecutor(dg, factory).run() Assert.assertEquals(runNodes.size, 3) } @@ -51,7 +51,7 @@ class DynamicGraphTest { @Test private fun testExecutorWithSkip() { - val g = DynamicGraph() + val g = DG() // 2 and 3 depend on 1, 4 depend on 3, 10 depends on 4 // 3 will blow up, which should make 4 and 10 skipped g.addEdge(2, 1) @@ -61,7 +61,7 @@ class DynamicGraphTest { g.addEdge(5, 2) val runNodes = arrayListOf() val factory = createFactory(runNodes, { n -> n != 3 }) - val ex = DynamicGraphExecutor(g, factory) + val ex = DGExecutor(g, factory) ex.run() Thread.`yield`() Assert.assertTrue(! runNodes.contains(4)) @@ -69,60 +69,63 @@ class DynamicGraphTest { } @Test - public fun test8() { - val dg = DG() - dg.addEdge("b1", "a1") - dg.addEdge("b1", "a2") - dg.addEdge("b2", "a1") - dg.addEdge("b2", "a2") - dg.addEdge("c1", "b1") - dg.addEdge("c1", "b2") - dg.addNode("x") - dg.addNode("y") - assertFreeNodesEquals(dg, arrayOf("a1", "a2", "y", "x")) + fun test8() { + DG().apply { + addEdge("b1", "a1") + addEdge("b1", "a2") + addEdge("b2", "a1") + addEdge("b2", "a2") + addEdge("c1", "b1") + addEdge("c1", "b2") + addNode("x") + addNode("y") + assertFreeNodesEquals(this, arrayOf("a1", "a2", "y", "x")) - dg.removeNode("a1") - assertFreeNodesEquals(dg, arrayOf()) + removeNode("a1") + assertFreeNodesEquals(this, arrayOf()) - dg.removeNode("a2") - assertFreeNodesEquals(dg, arrayOf("b1", "b2")) + removeNode("a2") + assertFreeNodesEquals(this, arrayOf("b1", "b2")) - dg.removeNode("b1") - assertFreeNodesEquals(dg, arrayOf()) + removeNode("b1") + assertFreeNodesEquals(this, arrayOf()) - dg.removeNode("b2") - assertFreeNodesEquals(dg, arrayOf("c1")) + removeNode("b2") + assertFreeNodesEquals(this, arrayOf("c1")) + } } @Test - public fun test2() { - val dg = DG() - dg.addEdge("b1", "a1") - dg.addEdge("b1", "a2") - dg.addNode("x") - assertFreeNodesEquals(dg, arrayOf("a1", "a2", "x" )) + fun test2() { + DG().apply { + addEdge("b1", "a1") + addEdge("b1", "a2") + addNode("x") + assertFreeNodesEquals(this, arrayOf("a1", "a2", "x")) - dg.removeNode("a1") - assertFreeNodesEquals(dg, arrayOf("a2", "x")) + removeNode("a1") + assertFreeNodesEquals(this, arrayOf("a2", "x")) - dg.removeNode("a2") - assertFreeNodesEquals(dg, arrayOf("b1", "x")) + removeNode("a2") + assertFreeNodesEquals(this, arrayOf("b1", "x")) - dg.removeNode("b1") - assertFreeNodesEquals(dg, arrayOf("x")) + removeNode("b1") + assertFreeNodesEquals(this, arrayOf("x")) + } } @Test fun topologicalSort() { - val dg = Topological() - dg.addEdge("b1", "a1") - dg.addEdge("b1", "a2") - dg.addEdge("b2", "a1") - dg.addEdge("b2", "a2") - dg.addEdge("c1", "b1") - dg.addEdge("c1", "b2") - val sorted = dg.sort(arrayListOf("a1", "a2", "b1", "b2", "c1", "x", "y")) - Assert.assertEquals(sorted, arrayListOf("a1", "a2", "x", "y", "b1", "b2", "c1")) + Topological().apply { + addEdge("b1", "a1") + addEdge("b1", "a2") + addEdge("b2", "a1") + addEdge("b2", "a2") + addEdge("c1", "b1") + addEdge("c1", "b2") + val sorted = sort(arrayListOf("a1", "a2", "b1", "b2", "c1", "x", "y")) + Assert.assertEquals(sorted, arrayListOf("a1", "a2", "x", "y", "b1", "b2", "c1")) + } } @Test