erik/kobalt
erik/kobalt
1
0
Fork 0
mirror of https://github.com/ethauvin/kobalt.git synced 2025-04-26 16:28:12 -07:00
Code Releases Activity

Remove old DynamicGraph.

Browse source
This commit is contained in:
Cedric Beust 2016-04-17 10:28:23 -07:00
parent 735b85a0f0
commit 63ba0ea63e
3 changed files with 57 additions and 337 deletions
Download patch file Download diff file Expand all files Collapse all files

5
modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/DG.kt
View file

@ -1,6 +1,7 @@
package com.beust.kobalt.internal
import com.beust.kobalt.KobaltException
import com.beust.kobalt.TaskResult
import com.beust.kobalt.misc.NamedThreadFactory
import com.beust.kobalt.misc.error
import com.beust.kobalt.misc.log
@ -9,6 +10,10 @@ import java.lang.reflect.InvocationTargetException
import java.util.*
import java.util.concurrent.*
open class TaskResult2<T>(success: Boolean, errorMessage: String?, val value: T) : TaskResult(success, errorMessage) {
override fun toString() = com.beust.kobalt.misc.toString("TaskResult", "value", value, "success", success)
}
class Node<T>(val value: T) {
override fun hashCode() = value!!.hashCode()
override fun equals(other: Any?) : Boolean {

288
modules/kobalt-plugin-api/src/main/kotlin/com/beust/kobalt/internal/DynamicGraph.kt
View file

@ -1,288 +0,0 @@
package com.beust.kobalt.internal
import com.beust.kobalt.KobaltException
import com.beust.kobalt.TaskResult
import com.beust.kobalt.misc.*
import com.google.common.collect.HashMultimap
import java.lang.reflect.InvocationTargetException
import java.util.concurrent.*
open class TaskResult2<T>(success: Boolean, errorMessage: String?, val value: T) : TaskResult(success, errorMessage) {
override fun toString() = toString("TaskResult", "value", value, "success", success)
}
interface IWorker<T> : Callable<TaskResult2<T>> {
/**
* @return list of tasks this worker is working on.
*/
// val tasks : List<T>
/**
* @return the priority of this task.
*/
val priority : Int
}
interface IThreadWorkerFactory<T> {
/**
* Creates {@code IWorker} for specified set of tasks. It is not necessary that
* number of workers returned be same as number of tasks entered.
*
* @param nodes tasks that need to be executed
* @return list of workers
*/
fun createWorkers(nodes: List<T>) : List<IWorker<T>>
}
class DynamicGraphExecutor<T>(val graph: DynamicGraph<T>,
val factory: IThreadWorkerFactory<T>) {
val executor = Executors.newFixedThreadPool(5, NamedThreadFactory("DynamicGraphExecutor"))
val completion = ExecutorCompletionService<TaskResult2<T>>(executor)
/**
* @return 0 if all went well, > 0 otherwise
*/
fun run() : Int {
var lastResult = TaskResult()
var gotError = false
var nodesRunning = 0
while (graph.freeNodes.size > 0 && ! gotError) {
log(3, "Current node count: ${graph.nodeCount}")
synchronized(graph) {
val freeNodes = graph.freeNodes
freeNodes.forEach { graph.setStatus(it, DynamicGraph.Status.RUNNING)}
log(3, " ==> Submitting " + freeNodes)
val callables : List<IWorker<T>> = factory.createWorkers(freeNodes)
callables.forEach { completion.submit(it) }
nodesRunning += callables.size
// When a callable ends, see if it freed a node. If not, keep looping
while (graph.nodesRunning.size > 0 && graph.freeNodes.size == 0 && ! gotError) {
try {
val future = completion.take()
val taskResult = future.get(2, TimeUnit.SECONDS)
lastResult = taskResult
log(3, " <== Received task result $taskResult")
graph.setStatus(taskResult.value,
if (taskResult.success) {
DynamicGraph.Status.FINISHED
} else {
DynamicGraph.Status.ERROR
})
} catch(ex: TimeoutException) {
log(2, "Time out")
} catch(ex: Exception) {
val ite = ex.cause
if (ite is InvocationTargetException) {
if (ite.targetException is KobaltException) {
throw (ex.cause as InvocationTargetException).targetException
} else {
error("Error: ${ite.cause?.message}", ite.cause)
gotError = true
}
} else {
error("Error: ${ex.message}", ex)
gotError = true
}
}
}
}
}
executor.shutdown()
if (graph.freeNodes.size == 0 && graph.nodesReady.size > 0) {
if (KobaltLogger.LOG_LEVEL > 1) {
throw KobaltException("Couldn't find any free nodes but a few nodes still haven't run, there is " +
"a cycle in the dependencies.\n Nodes left: " + graph.dump(graph.nodesReady))
} else {
error("Error during the build")
lastResult = TaskResult(false)
}
}
return if (lastResult.success) 0 else 1
}
}
/**
* Representation of the graph of methods.
*/
class DynamicGraph<T> {
val nodesReady = linkedSetOf<T>()
val nodesRunning = linkedSetOf<T>()
private val nodesFinished = linkedSetOf<T>()
private val nodesInError = linkedSetOf<T>()
private val nodesSkipped = linkedSetOf<T>()
private val dependedUpon = HashMultimap.create<T, T>()
private val dependingOn = HashMultimap.create<T, T>()
/**
* Define a comparator for the nodes of this graph, which will be used
* to order the free nodes when they are asked.
*/
// val comparator : Comparator<T>? = null
enum class Status {
READY, RUNNING, FINISHED, ERROR, SKIPPED
}
/**
* Add a node to the graph.
*/
fun addNode(value: T) : T {
nodes.add(value)
nodesReady.add(value)
return value
}
/**
* Add an edge between two nodes, which don't have to already be in the graph
* (they will be added by this method). Makes "to" depend on "from".
*/
fun addEdge(from: T, to: T) {
log(3, "Node $from depends on $to")
nodes.add(from)
nodes.add(to)
val fromNode = addNode(from)
val toNode = addNode(to)
dependingOn.put(toNode, fromNode)
dependedUpon.put(fromNode, toNode)
}
/**
* @return a set of all the nodes that don't depend on any other nodes.
*/
val freeNodes : List<T>
get() {
val result = arrayListOf<T>()
nodesReady.forEach { m ->
// A node is free if...
val du = dependedUpon.get(m)
// - no other nodes depend on it
if (! dependedUpon.containsKey(m)) {
result.add(m)
} else if (getUnfinishedNodes(du).size == 0) {
result.add(m)
}
}
// Sort the free nodes if requested (e.g. priorities)
// if (! result.isEmpty()) {
// if (comparator != null) {
// Collections.sort(result, comparator)
// debug("Nodes after sorting:" + result.get(0))
// }
// }
log(3, " freeNodes: $result")
return result
}
/**
* @return a list of all the nodes that have a status other than FINISHED.
*/
private fun getUnfinishedNodes(nodes: Set<T>) : Collection<T> {
val result = hashSetOf<T>()
nodes.forEach { node ->
if (nodesReady.contains(node) || nodesRunning.contains(node)) {
result.add(node);
}
}
return result;
}
/**
* Set the status for a set of nodes.
*/
fun setStatus(nodes: Collection<T>, status: Status) {
nodes.forEach { setStatus(it, status) }
}
/**
* Mark all dependees of this node SKIPPED
*/
private fun setSkipStatus(node: T, status: Status) {
dependingOn.get(node).forEach {
if (! nodesSkipped.contains(it)) {
log(3, "Node skipped: $it")
nodesSkipped.add(it)
nodesReady.remove(it)
setSkipStatus(it, status)
}
}
}
/**
* Set the status for a node.
*/
fun setStatus(node: T, status: Status) {
removeNode(node);
when(status) {
Status.READY -> nodesReady.add(node)
Status.RUNNING -> nodesRunning.add(node)
Status.FINISHED -> nodesFinished.add(node)
Status.ERROR -> {
log(3, "Node in error: $node")
nodesReady.remove(node)
nodesInError.add(node)
setSkipStatus(node, status)
}
else -> {
throw IllegalArgumentException()
}
}
}
private fun removeNode(node: T) {
if (! nodesReady.remove(node)) {
if (! nodesRunning.remove(node)) {
nodesFinished.remove(node)
}
}
}
/**
* @return the number of nodes in this graph.
*/
val nodeCount: Int
get() = nodesReady.size + nodesRunning.size + nodesFinished.size
override fun toString() : String {
val result = StringBuilder("[DynamicGraph ")
result.append("\n Ready:" + nodesReady)
result.append("\n Running:" + nodesRunning)
result.append("\n Finished:" + nodesFinished)
result.append("\n Edges:\n")
// dependingOn.entrySet().forEach { es ->
// result.append(" " + es.getKey() + "\n");
// es.getValue().forEach { t ->
// result.append(" " + t + "\n");
// }
// }
result.append("]");
return result.toString();
}
val nodes = hashSetOf<T>()
fun dump(nodes: Collection<T>) : String {
val result = StringBuffer()
result.append("************ Graph dump ***************\n")
val free = arrayListOf<T>()
nodes.forEach { node ->
val d = dependedUpon.get(node)
if (d == null || d.isEmpty()) {
free.add(node)
}
}
result.append("Free nodes: $free").append("\n Dependent nodes:\n")
nodes.forEach {
result.append(" $it -> ${dependedUpon.get(it)}\n")
}
return result.toString()
}
fun dump() = dump(nodesReady)
}

101
src/test/kotlin/com/beust/kobalt/internal/DynamicGraphTest.kt
View file

@ -14,17 +14,17 @@ class DynamicGraphTest {
Assert.assertEquals(h, e)
}
private fun <T> createFactory(runNodes: ArrayList<T>, errorFunction: (T) -> Boolean) : IThreadWorkerFactory<T> {
return object: IThreadWorkerFactory<T> {
override fun createWorkers(nodes: List<T>): List<IWorker<T>> {
val result = arrayListOf<IWorker<T>>()
private fun <T> createFactory(runNodes: ArrayList<T>, errorFunction: (T) -> Boolean) : IThreadWorkerFactory2<T> {
return object: IThreadWorkerFactory2<T> {
override fun createWorkers(nodes: Collection<T>): List<IWorker2<T>> {
val result = arrayListOf<IWorker2<T>>()
nodes.forEach { result.add(Worker(runNodes, it, errorFunction)) }
return result
}
}
}
class Worker<T>(val runNodes: ArrayList<T>, val n: T, val errorFunction: (T) -> Boolean) : IWorker<T> {
class Worker<T>(val runNodes: ArrayList<T>, val n: T, val errorFunction: (T) -> Boolean) : IWorker2<T> {
override val priority = 0
override fun call() : TaskResult2<T> {
@ -36,14 +36,14 @@ class DynamicGraphTest {
@Test
fun testExecutor() {
val dg = DynamicGraph<String>();
val dg = DG<String>();
dg.addEdge("compile", "runApt")
dg.addEdge("compile", "generateVersion")
val runNodes = arrayListOf<String>()
val factory = createFactory(runNodes, { true })
DynamicGraphExecutor(dg, factory).run()
DGExecutor(dg, factory).run()
Assert.assertEquals(runNodes.size, 3)
}
@ -51,7 +51,7 @@ class DynamicGraphTest {
@Test
private fun testExecutorWithSkip() {
val g = DynamicGraph<Int>()
val g = DG<Int>()
// 2 and 3 depend on 1, 4 depend on 3, 10 depends on 4
// 3 will blow up, which should make 4 and 10 skipped
g.addEdge(2, 1)
@ -61,7 +61,7 @@ class DynamicGraphTest {
g.addEdge(5, 2)
val runNodes = arrayListOf<Int>()
val factory = createFactory(runNodes, { n -> n != 3 })
val ex = DynamicGraphExecutor(g, factory)
val ex = DGExecutor(g, factory)
ex.run()
Thread.`yield`()
Assert.assertTrue(! runNodes.contains(4))
@ -69,60 +69,63 @@ class DynamicGraphTest {
}
@Test
public fun test8() {
val dg = DG<String>()
dg.addEdge("b1", "a1")
dg.addEdge("b1", "a2")
dg.addEdge("b2", "a1")
dg.addEdge("b2", "a2")
dg.addEdge("c1", "b1")
dg.addEdge("c1", "b2")
dg.addNode("x")
dg.addNode("y")
assertFreeNodesEquals(dg, arrayOf("a1", "a2", "y", "x"))
fun test8() {
DG<String>().apply {
addEdge("b1", "a1")
addEdge("b1", "a2")
addEdge("b2", "a1")
addEdge("b2", "a2")
addEdge("c1", "b1")
addEdge("c1", "b2")
addNode("x")
addNode("y")
assertFreeNodesEquals(this, arrayOf("a1", "a2", "y", "x"))
dg.removeNode("a1")
assertFreeNodesEquals(dg, arrayOf<String>())
removeNode("a1")
assertFreeNodesEquals(this, arrayOf<String>())
dg.removeNode("a2")
assertFreeNodesEquals(dg, arrayOf("b1", "b2"))
removeNode("a2")
assertFreeNodesEquals(this, arrayOf("b1", "b2"))
dg.removeNode("b1")
assertFreeNodesEquals(dg, arrayOf<String>())
removeNode("b1")
assertFreeNodesEquals(this, arrayOf<String>())
dg.removeNode("b2")
assertFreeNodesEquals(dg, arrayOf("c1"))
removeNode("b2")
assertFreeNodesEquals(this, arrayOf("c1"))
}
}
@Test
public fun test2() {
val dg = DG<String>()
dg.addEdge("b1", "a1")
dg.addEdge("b1", "a2")
dg.addNode("x")
assertFreeNodesEquals(dg, arrayOf("a1", "a2", "x" ))
fun test2() {
DG<String>().apply {
addEdge("b1", "a1")
addEdge("b1", "a2")
addNode("x")
assertFreeNodesEquals(this, arrayOf("a1", "a2", "x"))
dg.removeNode("a1")
assertFreeNodesEquals(dg, arrayOf("a2", "x"))
removeNode("a1")
assertFreeNodesEquals(this, arrayOf("a2", "x"))
dg.removeNode("a2")
assertFreeNodesEquals(dg, arrayOf("b1", "x"))
removeNode("a2")
assertFreeNodesEquals(this, arrayOf("b1", "x"))
dg.removeNode("b1")
assertFreeNodesEquals(dg, arrayOf("x"))
removeNode("b1")
assertFreeNodesEquals(this, arrayOf("x"))
}
}
@Test
fun topologicalSort() {
val dg = Topological<String>()
dg.addEdge("b1", "a1")
dg.addEdge("b1", "a2")
dg.addEdge("b2", "a1")
dg.addEdge("b2", "a2")
dg.addEdge("c1", "b1")
dg.addEdge("c1", "b2")
val sorted = dg.sort(arrayListOf("a1", "a2", "b1", "b2", "c1", "x", "y"))
Assert.assertEquals(sorted, arrayListOf("a1", "a2", "x", "y", "b1", "b2", "c1"))
Topological<String>().apply {
addEdge("b1", "a1")
addEdge("b1", "a2")
addEdge("b2", "a1")
addEdge("b2", "a2")
addEdge("c1", "b1")
addEdge("c1", "b2")
val sorted = sort(arrayListOf("a1", "a2", "b1", "b2", "c1", "x", "y"))
Assert.assertEquals(sorted, arrayListOf("a1", "a2", "x", "y", "b1", "b2", "c1"))
}
}
@Test