1
0
Fork 0
mirror of https://github.com/ethauvin/kobalt.git synced 2025-04-26 08:27:12 -07:00

Fix the task dependency bug.

This commit is contained in:
Cedric Beust 2016-05-04 02:09:47 -08:00
parent 1e932d2f38
commit 5279a19966
4 changed files with 316 additions and 141 deletions

View file

@ -17,6 +17,7 @@ class DynamicTask(override val plugin: IPlugin, override val name: String, overr
val reverseDependsOn: List<String> = listOf<String>(),
val runBefore: List<String> = listOf<String>(),
val runAfter: List<String> = listOf<String>(),
val alwaysRunAfter: List<String> = listOf<String>(),
val closure: (Project) -> TaskResult) : ITask {
override fun call(): TaskResult2<ITask> {

View file

@ -21,7 +21,10 @@ annotation class Task(
val runBefore: Array<String> = arrayOf(),
/** Ordering: tasks this task runs after */
val runAfter: Array<String> = arrayOf()
val runAfter: Array<String> = arrayOf(),
/** Wrapper tasks */
val alwaysRunAfter: Array<String> = arrayOf()
)
@Retention(AnnotationRetention.RUNTIME)
@ -38,8 +41,11 @@ annotation class IncrementalTask(
/** Tasks that this task depends on */
val runBefore: Array<String> = arrayOf(),
/** Ordering: tasks this task runs after */
val runAfter: Array<String> = arrayOf()
/** Ordering: tasks this task runs after */
val runAfter: Array<String> = arrayOf(),
/** Wrapper tasks */
val alwaysRunAfter: Array<String> = arrayOf()
)
/**

View file

@ -25,6 +25,7 @@ class TaskManager @Inject constructor(val args: Args,
private val reverseDependsOn = TreeMultimap.create<String, String>()
private val runBefore = TreeMultimap.create<String, String>()
private val runAfter = TreeMultimap.create<String, String>()
private val alwaysRunAfter = TreeMultimap.create<String, String>()
/**
* Dependency: task2 depends on task 1.
@ -34,7 +35,7 @@ class TaskManager @Inject constructor(val args: Args,
/**
* Dependency: task2 depends on task 1.
*/
fun reverseDependsOn(task1: String, task2: String) = reverseDependsOn.put(task1, task2)
fun reverseDependsOn(task1: String, task2: String) = reverseDependsOn.put(task2, task1)
/**
* Ordering: task1 runs before task 2.
@ -46,6 +47,11 @@ class TaskManager @Inject constructor(val args: Args,
*/
fun runAfter(task1: String, task2: String) = runAfter.put(task2, task1)
/**
* Wrapper task: task2 runs after task 1.
*/
fun alwaysRunAfter(task1: String, task2: String) = alwaysRunAfter.put(task2, task1)
data class TaskInfo(val id: String) {
constructor(project: String, task: String) : this(project + ":" + task)
@ -53,6 +59,7 @@ class TaskManager @Inject constructor(val args: Args,
get() = if (id.contains(":")) id.split(":")[0] else null
val taskName: String
get() = if (id.contains(":")) id.split(":")[1] else id
fun matches(projectName: String) = project == null || project == projectName
}
@ -64,7 +71,7 @@ class TaskManager @Inject constructor(val args: Args,
* There can be multiple tasks by the same name (e.g. PackagingPlugin and AndroidPlugin both
* define "install"), so return a multimap.
*/
fun tasksByNames(project: Project): ListMultimap<String, out ITask> {
fun tasksByNames(project: Project): ListMultimap<String, ITask> {
return ArrayListMultimap.create<String, ITask>().apply {
annotationTasks.filter {
it.project.name == project.name
@ -79,7 +86,7 @@ class TaskManager @Inject constructor(val args: Args,
}
}
fun runTargets(taskNames: List<String>, projects: List<Project>) : RunTargetResult {
fun runTargets(taskNames: List<String>, projects: List<Project>): RunTargetResult {
var result = 0
val failedProjects = hashSetOf<String>()
val messages = Collections.synchronizedList(arrayListOf<String>())
@ -109,8 +116,8 @@ class TaskManager @Inject constructor(val args: Args,
log(3, " $it: " + tasksByNames.get(it))
}
val graph = createGraph(project.name, taskNames, tasksByNames,
dependsOn, reverseDependsOn, runBefore, runAfter,
val graph = createGraph2(project.name, taskNames, tasksByNames,
dependsOn, reverseDependsOn, runBefore, runAfter, alwaysRunAfter,
{ task: ITask -> task.name },
{ task: ITask -> task.plugin.accept(project) })
@ -121,7 +128,7 @@ class TaskManager @Inject constructor(val args: Args,
val factory = object : IThreadWorkerFactory<ITask> {
override fun createWorkers(nodes: Collection<ITask>)
= nodes.map { TaskWorker(listOf(it), args.dryRun, messages) }
= nodes.map { TaskWorker(listOf(it), args.dryRun, messages) }
}
val executor = DynamicGraphExecutor(graph, factory)
@ -138,134 +145,250 @@ class TaskManager @Inject constructor(val args: Args,
return RunTargetResult(result, messages)
}
/**
* Create a dynamic graph representing all the tasks that need to be run.
*/
val LOG_LEVEL = 1
@VisibleForTesting
fun <T> createGraph(projectName: String, taskNames: List<String>, nodeMap: Multimap<String, out T>,
fun <T> createGraph2(projectName: String, taskNames: List<String>, nodeMap: Multimap<String, T>,
dependsOn: Multimap<String, String>,
reverseDependsOn: Multimap<String, String>,
runBefore: Multimap<String, String>,
runAfter: Multimap<String, String>,
alwaysRunAfter: Multimap<String, String>,
toName: (T) -> String,
accept: (T) -> Boolean):
DynamicGraph<T> {
val result = DynamicGraph<T>()
taskNames.forEach { fullTaskName ->
val ti = TaskInfo(fullTaskName)
if (!nodeMap.keys().contains(ti.taskName)) {
throw KobaltException("Unknown task: $fullTaskName")
val newToProcess = arrayListOf<T>()
val seen = hashSetOf<String>()
val always = ArrayListMultimap.create<String, String>()
alwaysRunAfter.keySet().forEach { k ->
alwaysRunAfter[k].forEach { v ->
always.put(v, k)
}
}
//
// Turn the task names into the more useful TaskInfo and do some sanity checking on the way
//
val taskInfos = taskNames.map { TaskInfo(it) }.filter {
if (!nodeMap.keys().contains(it.taskName)) {
throw KobaltException("Unknown task: $it")
}
it.matches(projectName)
}
if (ti.matches(projectName)) {
val tiTaskName = ti.taskName
nodeMap[tiTaskName].forEach { task ->
if (task != null && accept(task)) {
val toProcess = arrayListOf(task)
val seen = hashSetOf<String>()
val newToProcess = hashSetOf<T>()
val toProcess = ArrayList(taskInfos)
fun maybeAddEdge(task: T, mm: Multimap<String, String>, isDependency: Boolean,
reverseEdges: Boolean = false) : Boolean {
var added = false
val taskName = toName(task)
mm[taskName]?.forEach { toName ->
val addEdge = isDependency || (!isDependency && taskNames.contains(toName))
if (addEdge) {
nodeMap[toName].forEach { to ->
if (reverseEdges) {
log(3, " Adding reverse edge \"$to\" -> \"$task\" it=\"$toName\"")
added = true
result.addEdge(to, task)
} else {
log(3, " Adding edge \"$task\" -> \"$to\"")
added = true
result.addEdge(task, to)
}
if (!seen.contains(toName(to))) {
log(3, " New node to process: \"$to\"")
newToProcess.add(to)
} else {
log(3, " Already seen: $to")
}
}
}
}
return added
}
while (toProcess.size > 0) {
fun reverseMultimap(mm: Multimap<String, String>) : Multimap<String, String> {
val result = TreeMultimap.create<String, String>()
mm.keySet().forEach { key ->
mm[key].forEach { value ->
result.put(value, key)
}
}
return result
}
// These two maps indicate reversed dependencies so we want to have
// a reverse map for them so we consider all the cases. For example,
// if we are looking at task "a", we want to find all the relationships
// that depend on "a" and also all the relationships that "a" depends on
val invertedReverseDependsOn = reverseMultimap(reverseDependsOn)
val invertedRunBefore = reverseMultimap(runBefore)
val invertedDependsOn = reverseMultimap(dependsOn)
val invertedRunAfter = reverseMultimap(runAfter)
while (toProcess.size > 0) {
log(3, " New batch of nodes to process: $toProcess")
toProcess.filter { !seen.contains(toName(it)) }.forEach { current ->
result.addNode(current)
seen.add(toName(current))
if (! maybeAddEdge(current, invertedDependsOn, true, true)) {
maybeAddEdge(current, dependsOn, true, false)
}
if (! maybeAddEdge(current, invertedRunAfter, false, true)) {
maybeAddEdge(current, runAfter, false, false)
}
if (! maybeAddEdge(current, reverseDependsOn, true, true)) {
maybeAddEdge(current, invertedReverseDependsOn, true, false)
}
if (! maybeAddEdge(current, runBefore, false, true)) {
maybeAddEdge(current, invertedRunBefore, false, false)
}
newToProcess.addAll(dependsOn[toName(current)].flatMap { nodeMap[it] })
}
toProcess.clear()
toProcess.addAll(newToProcess)
newToProcess.clear()
}
fun addEdge(result: DynamicGraph<T>, from: String, to: String, newToProcess: ArrayList<T>, text: String) {
val froms = nodeMap[from]
froms.forEach { f: T ->
nodeMap[to].forEach { t: T ->
val tn = toName(t)
log(LOG_LEVEL, " Adding edge ($text) $f -> $t")
result.addEdge(f, t)
newToProcess.add(t)
}
}
} else {
log(3, "Task $fullTaskName does not match the current project $projectName, skipping it")
}
/**
* Whenever a task is added to the graph, we also add its alwaysRunAfter tasks.
*/
fun processAlways(always: Multimap<String, String>, node: T) {
log(LOG_LEVEL, " Processing always for $node")
always[toName(node)]?.let { to: Collection<String> ->
to.forEach { t ->
nodeMap[t].forEach { from ->
log(LOG_LEVEL, " Adding always edge $from -> $node")
result.addEdge(from, node)
}
}
log(LOG_LEVEL, " ... done processing always for $node")
}
}
log(LOG_LEVEL, " Current batch to process: $toProcess")
val invertedReverseDependsOn = reverseMultimap(reverseDependsOn)
toProcess.forEach { taskInfo ->
val taskName = taskInfo.taskName
log(LOG_LEVEL, " ***** Current node: $taskName")
nodeMap[taskName].forEach { processAlways(always, it) }
//
// dependsOn and reverseDependsOn are considered for all tasks, explicit and implicit
//
dependsOn[taskName].forEach { to ->
addEdge(result, taskName, to, newToProcess, "dependsOn")
}
reverseDependsOn[taskName].forEach { from ->
addEdge(result, from, taskName, newToProcess, "reverseDependsOn")
}
invertedReverseDependsOn[taskName].forEach { to ->
addEdge(result, taskName, to, newToProcess, "invertedReverseDependsOn")
}
//
// runBefore and runAfter (task ordering) are only considered for explicit tasks (tasks that were
// explicitly requested by the user)
//
runBefore[taskName].forEach { from ->
if (taskNames.contains(from)) {
addEdge(result, from, taskName, newToProcess, "runBefore")
}
}
runAfter[taskName].forEach { to ->
if (taskNames.contains(to)) {
addEdge(result, taskName, to, newToProcess, "runAfter")
}
}
seen.add(taskName)
}
newToProcess.forEach { processAlways(always, it) }
toProcess.clear()
toProcess.addAll(newToProcess.filter { ! seen.contains(toName(it))}.map { TaskInfo(toName(it)) })
newToProcess.clear()
}
return result
}
private fun reverseMultimap(mm: Multimap<String, String>) : Multimap<String, String> {
val result = TreeMultimap.create<String, String>()
mm.keySet().forEach { key ->
mm[key].forEach { value ->
result.put(value, key)
}
}
return result
}
/**
* Create a dynamic graph representing all the tasks that need to be run.
*/
// @VisibleForTesting
// fun <T> createGraph(projectName: String, taskNames: List<String>, nodeMap: Multimap<String, out T>,
// dependsOn: Multimap<String, String>,
// reverseDependsOn: Multimap<String, String>,
// runBefore: Multimap<String, String>,
// runAfter: Multimap<String, String>,
// toName: (T) -> String,
// accept: (T) -> Boolean):
// DynamicGraph<T> {
//
// val result = DynamicGraph<T>()
// taskNames.forEach { fullTaskName ->
// val ti = TaskInfo(fullTaskName)
// if (!nodeMap.keys().contains(ti.taskName)) {
// throw KobaltException("Unknown task: $fullTaskName")
// }
//
// if (ti.matches(projectName)) {
// val tiTaskName = ti.taskName
// nodeMap[tiTaskName].forEach { task ->
// if (task != null && accept(task)) {
// val toProcess = arrayListOf(task)
// val newToProcess = hashSetOf<T>()
//
// fun maybeAddEdge(task: T, mm: Multimap<String, String>,
// seen: Set<String>,
// isDependency: Boolean,
// reverseEdges: Boolean = false): Boolean {
// var added = false
// val taskName = toName(task)
// mm[taskName]?.forEach { toName ->
// val addEdge = isDependency || (!isDependency && taskNames.contains(toName))
// if (addEdge) {
// nodeMap[toName].forEach { to ->
// if (reverseEdges) {
// log(3, " Adding reverse edge \"$to\" -> \"$task\" it=\"$toName\"")
// added = true
// result.addEdge(to, task)
// } else {
// log(3, " Adding edge \"$task\" -> \"$to\"")
// added = true
// result.addEdge(task, to)
// }
// if (!seen.contains(toName(to))) {
// log(3, " New node to process: \"$to\"")
// newToProcess.add(to)
// } else {
// log(3, " Already seen: $to")
// }
// }
// }
// }
// return added
// }
//
// // These two maps indicate reversed dependencies so we want to have
// // a reverse map for them so we consider all the cases. For example,
// // if we are looking at task "a", we want to find all the relationships
// // that depend on "a" and also all the relationships that "a" depends on
// val invertedReverseDependsOn = reverseMultimap(reverseDependsOn)
// val invertedRunBefore = reverseMultimap(runBefore)
// val invertedDependsOn = reverseMultimap(dependsOn)
// val invertedRunAfter = reverseMultimap(runAfter)
//
// val seen = hashSetOf<String>()
// while (toProcess.size > 0) {
// log(3, " New batch of nodes to process: $toProcess")
// toProcess.filter { !seen.contains(toName(it)) }.forEach { current ->
// result.addNode(current)
//
// if (! maybeAddEdge(current, invertedDependsOn, seen, true, true)) {
// maybeAddEdge(current, dependsOn, seen, true, false)
// }
// if (! maybeAddEdge(current, invertedRunAfter, seen, false, true)) {
// maybeAddEdge(current, runAfter, seen, false, false)
// }
// if (! maybeAddEdge(current, reverseDependsOn, seen, true, true)) {
// maybeAddEdge(current, invertedReverseDependsOn, seen, true, false)
// }
// if (! maybeAddEdge(current, runBefore, seen, false, true)) {
// maybeAddEdge(current, invertedRunBefore, seen, false, false)
// }
//
// seen.add(toName(current))
//
// newToProcess.addAll(dependsOn[toName(current)].flatMap { nodeMap[it] })
// }
// toProcess.clear()
// toProcess.addAll(newToProcess)
// newToProcess.clear()
// }
// }
// }
// } else {
// log(3, "Task $fullTaskName does not match the current project $projectName, skipping it")
// }
// }
// return result
// }
/**
* Find the free tasks of the graph.
*/
private fun <T> calculateFreeTasks(tasksByNames: Multimap<String, T>, runBefore: TreeMultimap<String, String>,
reverseAfter: HashMap<String,
String>)
: Collection<T> {
val freeTaskMap = hashMapOf<String, T>()
tasksByNames.keys().forEach {
if (! runBefore.containsKey(it) && ! reverseAfter.containsKey(it)) {
tasksByNames[it].forEach { t ->
freeTaskMap.put(it, t)
}
}
}
return freeTaskMap.values
}
// private fun <T> calculateFreeTasks(tasksByNames: Multimap<String, T>, runBefore: TreeMultimap<String, String>,
// reverseAfter: HashMap<String,
// String>)
// : Collection<T> {
// val freeTaskMap = hashMapOf<String, T>()
// tasksByNames.keys().forEach {
// if (! runBefore.containsKey(it) && ! reverseAfter.containsKey(it)) {
// tasksByNames[it].forEach { t ->
// freeTaskMap.put(it, t)
// }
// }
// }
//
// return freeTaskMap.values
// }
/////
// Manage the tasks
@ -278,6 +401,7 @@ class TaskManager @Inject constructor(val args: Args,
class TaskAnnotation(val method: Method, val plugin: IPlugin, val name: String, val description: String,
val dependsOn: Array<String>, val reverseDependsOn: Array<String>,
val runBefore: Array<String>, val runAfter: Array<String>,
val alwaysRunAfter: Array<String>,
val callable: (Project) -> TaskResult) {
override fun toString() = "[TaskAnnotation $name]"
}
@ -287,7 +411,7 @@ class TaskManager @Inject constructor(val args: Args,
*/
fun toTaskAnnotation(method: Method, plugin: IPlugin, ta: Task)
= TaskAnnotation(method, plugin, ta.name, ta.description, ta.dependsOn, ta.reverseDependsOn,
ta.runBefore, ta.runAfter,
ta.runBefore, ta.runAfter, ta.alwaysRunAfter,
{ project ->
method.invoke(plugin, project) as TaskResult
})
@ -298,7 +422,7 @@ class TaskManager @Inject constructor(val args: Args,
*/
fun toTaskAnnotation(method: Method, plugin: IPlugin, ta: IncrementalTask)
= TaskAnnotation(method, plugin, ta.name, ta.description, ta.dependsOn, ta.reverseDependsOn,
ta.runBefore, ta.runAfter,
ta.runBefore, ta.runAfter, ta.alwaysRunAfter,
incrementalManagerFactory.create().toIncrementalTaskClosure(ta.name, { project ->
method.invoke(plugin, project) as IncrementalTaskInfo
}))
@ -327,7 +451,7 @@ class TaskManager @Inject constructor(val args: Args,
dynamicTasks.forEach { task ->
projects.filter { task.plugin.accept(it) }.forEach { project ->
addTask(task.plugin, project, task.name, task.doc,
task.dependsOn, task.reverseDependsOn, task.runBefore, task.runAfter,
task.dependsOn, task.reverseDependsOn, task.runBefore, task.runAfter, task.alwaysRunAfter,
task.closure)
}
}
@ -352,7 +476,7 @@ class TaskManager @Inject constructor(val args: Args,
addTask(plugin, project, annotation.name, annotation.description,
annotation.dependsOn.toList(), annotation.reverseDependsOn.toList(),
annotation.runBefore.toList(), annotation.runAfter.toList(),
task)
annotation.alwaysRunAfter.toList(), task)
}
fun addTask(plugin: IPlugin, project: Project, name: String, description: String = "",
@ -360,6 +484,7 @@ class TaskManager @Inject constructor(val args: Args,
reverseDependsOn: List<String> = listOf<String>(),
runBefore: List<String> = listOf<String>(),
runAfter: List<String> = listOf<String>(),
alwaysRunAfter: List<String> = listOf<String>(),
task: (Project) -> TaskResult) {
annotationTasks.add(
object : BasePluginTask(plugin, name, description, project) {
@ -372,6 +497,7 @@ class TaskManager @Inject constructor(val args: Args,
reverseDependsOn.forEach { reverseDependsOn(it, name) }
runBefore.forEach { runBefore(it, name) }
runAfter.forEach { runAfter(it, name) }
alwaysRunAfter.forEach { alwaysRunAfter(it, name) }
}
/**