1
0
Fork 0
mirror of https://github.com/ethauvin/kobalt.git synced 2025-04-26 08:27:12 -07:00

Merge branch 'master' of github.com:cbeust/kobalt

This commit is contained in:
Cedric Beust 2016-03-28 18:24:59 -07:00
commit 7d163310da
11 changed files with 61 additions and 450 deletions

View file

@ -3,7 +3,6 @@ package com.beust.kobalt
import com.beust.kobalt.api.IClasspathDependency
import com.beust.kobalt.maven.LocalRepo
import com.beust.kobalt.maven.MavenId
import com.beust.kobalt.maven.RepoFinder
import com.beust.kobalt.maven.aether.KobaltAether
import com.beust.kobalt.misc.KobaltExecutors
import com.beust.kobalt.misc.Node
@ -14,7 +13,7 @@ import java.util.*
/**
* Display information about a Maven id.
*/
class ResolveDependency @Inject constructor(val repoFinder: RepoFinder,
class ResolveDependency @Inject constructor(
val localRepo: LocalRepo,
val aether: KobaltAether,
val executors: KobaltExecutors) {

View file

@ -1,86 +0,0 @@
package com.beust.kobalt.maven
import com.beust.kobalt.HostConfig
import com.beust.kobalt.misc.KFiles
import com.beust.kobalt.misc.log
import com.beust.kobalt.misc.warn
import com.google.common.cache.CacheBuilder
import com.google.common.cache.CacheLoader
import com.google.common.cache.LoadingCache
import com.google.inject.assistedinject.Assisted
import java.io.File
import java.nio.file.Files
import java.nio.file.Paths
import java.nio.file.StandardCopyOption
import java.util.concurrent.Callable
import java.util.concurrent.ExecutorService
import java.util.concurrent.Future
import javax.inject.Inject
import javax.inject.Singleton
/**
* Manages the download of files from a given HostConfig.
*/
@Singleton
class DownloadManager @Inject constructor(val factory: ArtifactFetcher.IFactory) {
class Key(val hostInfo: HostConfig, val fileName: String, val executor: ExecutorService) {
override fun equals(other: Any?): Boolean {
return (other as Key).hostInfo.url == hostInfo.url
}
override fun hashCode(): Int {
return hostInfo.url.hashCode()
}
}
private val CACHE : LoadingCache<Key, Future<File>> = CacheBuilder.newBuilder()
.build(object : CacheLoader<Key, Future<File>>() {
override fun load(key: Key): Future<File> {
return key.executor.submit(factory.create(key.hostInfo, key.fileName))
}
})
fun download(hostInfo: HostConfig, fileName: String, executor: ExecutorService)
: Future<File> = CACHE.get(Key(hostInfo, fileName, executor))
}
/**
* Fetches an artifact (a file in a Maven repo, .jar, -javadoc.jar, ...) to the given local file.
*/
class ArtifactFetcher @Inject constructor(@Assisted("hostInfo") val hostInfo: HostConfig,
@Assisted("fileName") val fileName: String,
val files: KFiles) : Callable<File> {
interface IFactory {
fun create(@Assisted("hostInfo") hostInfo: HostConfig, @Assisted("fileName") fileName: String) : ArtifactFetcher
}
override fun call() : File {
val k = Kurl(hostInfo.copy(url = hostInfo.url + ".md5"))
val remoteMd5 =
if (k.exists) k.string.trim(' ', '\t', '\n').substring(0, 32)
else null
val tmpFile = Paths.get(fileName + ".tmp")
val file = Paths.get(fileName)
val url = hostInfo.url
if (! Files.exists(file)) {
with(tmpFile.toFile()) {
parentFile.mkdirs()
Kurl(hostInfo).toFile(this)
}
log(2, "Done downloading, renaming $tmpFile to $file")
Files.move(tmpFile, file, StandardCopyOption.REPLACE_EXISTING)
log(1, " Downloaded $url")
log(2, " to $file")
}
val localMd5 = Md5.toMd5(file.toFile())
if (remoteMd5 != null && remoteMd5 != localMd5) {
warn("MD5 not matching for $url")
} else {
log(2, "No md5 found for $url, skipping md5 check")
}
return file.toFile()
}
}

View file

@ -24,19 +24,16 @@ class DependencyManager @Inject constructor(val executors: KobaltExecutors, val
*/
override fun create(id: String) : IClasspathDependency {
if (id.startsWith(FileDependency.PREFIX_FILE)) {
return FileDependency(id.substring(FileDependency.PREFIX_FILE.length))
return createFile(id.substring(FileDependency.PREFIX_FILE.length))
} else {
val mavenId = MavenId.create(id)
val result = if (mavenId.hasVersion) aether.create(id)
else aether.create(id + "(0,]")
return result
return createMaven(id)
}
}
/**
* Create an IClasspathDependency from a Maven id.
*/
override fun createMaven(id: String) : IClasspathDependency = create(id)
override fun createMaven(id: String) : IClasspathDependency = aether.create(id)
/**
* Create an IClasspathDependency from a path.

View file

@ -40,11 +40,13 @@ class MavenId private constructor(val groupId: String, val artifactId: String, v
MavenId(groupId, artifactId, extension, version)
}
fun toKobaltId(id: String) = if (id.endsWith(":")) id + "(0,]" else id
/**
* The main entry point to create Maven Id's. Id's created by this function
* will run through IMavenIdInterceptors.
*/
fun create(id: String) : MavenId {
fun create(originalId: String) : MavenId {
val id = toKobaltId(originalId)
var originalMavenId = createNoInterceptors(id)
var interceptedMavenId = originalMavenId
val interceptors = Kobalt.context?.pluginInfo?.mavenIdInterceptors

View file

@ -1,88 +0,0 @@
package com.beust.kobalt.maven
import com.beust.kobalt.HostConfig
import com.beust.kobalt.api.Kobalt
import com.beust.kobalt.misc.KobaltExecutors
import com.beust.kobalt.misc.Version
import com.beust.kobalt.misc.log
import com.beust.kobalt.misc.warn
import com.google.common.cache.CacheBuilder
import com.google.common.cache.CacheLoader
import com.google.common.cache.LoadingCache
import java.util.concurrent.ExecutorCompletionService
import java.util.concurrent.TimeUnit
import javax.inject.Inject
/**
* Find the repo that contains the given dependency among a list of repos. Searches are performed in parallel and
* cached so we never make a network call for the same dependency more than once.
*/
class RepoFinder @Inject constructor(val executors: KobaltExecutors, val finderFactory: RepoFinderCallable.IFactory) {
fun findCorrectRepo(id: String) = FOUND_REPOS.get(id)
/**
* archiveUrl: full URL
*/
data class RepoResult(val hostConfig: HostConfig, val version: Version? = null,
val archiveUrl: String? = null, val snapshotVersion: Version? = null) {
val found = archiveUrl != null
val localPath = archiveUrl?.substring(hostConfig.url.length)
// If it's a snapshot, we download a specific timestamped jar file but we need to save it under
// the SNAPSHOT-3.2.jar name.
val path = if (snapshotVersion != null && snapshotVersion.snapshotTimestamp != null && localPath != null) {
val ind = localPath.indexOf(snapshotVersion.snapshotTimestamp)
val lastDot = localPath.lastIndexOf(".")
val result = localPath.substring(0, ind) + "SNAPSHOT" +
localPath.substring(lastDot)
result
} else {
localPath
}
}
private val FOUND_REPOS: LoadingCache<String, RepoResult> = CacheBuilder.newBuilder()
.build(object : CacheLoader<String, RepoResult>() {
override fun load(key: String): RepoResult {
return loadCorrectRepo(key)
}})
/**
* Schedule an HTTP request to each repo in its own thread.
*/
private fun loadCorrectRepo(id: String): RepoResult {
val executor = executors.newExecutor("RepoFinder-$id", Kobalt.repos.size)
val cs = ExecutorCompletionService<List<RepoResult>>(executor)
val results = arrayListOf<RepoResult>()
try {
log(2, "Looking for $id")
Kobalt.repos.forEach { cs.submit(finderFactory.create(id, it)) }
for (i in 0..Kobalt.repos.size - 1) {
try {
val repos = cs.take().get(2000, TimeUnit.MILLISECONDS)
repos.forEach { result ->
if (result.found) {
log(2, "Located $id in ${result.hostConfig.url}")
results.add(result)
} else {
log(3, " Result for repo #$i: $result")
}
}
} catch(ex: Exception) {
warn("Error: $ex")
}
}
} finally {
executor.shutdownNow()
}
if (results.size > 0) {
// results.sortByDescending { Versions.toLongVersion(it.version) }
results.sort({ left, right -> right.version!!.compareTo(left.version!!) })
return results[0]
} else {
return RepoResult(HostConfig(""), Version.of(id))
}
}
}

View file

@ -1,193 +0,0 @@
package com.beust.kobalt.maven
import com.beust.kobalt.HostConfig
import com.beust.kobalt.maven.dependency.FileDependency
import com.beust.kobalt.misc.Version
import com.beust.kobalt.misc.log
import com.beust.kobalt.misc.warn
import com.google.inject.Inject
import com.google.inject.assistedinject.Assisted
import kotlinx.dom.asElementList
import kotlinx.dom.parseXml
import org.w3c.dom.NodeList
import java.io.File
import java.util.concurrent.Callable
import javax.xml.xpath.XPathConstants
import javax.xml.xpath.XPathFactory
/**
* Execute a single HTTP request to one repo. This Callable can return more than one RepoResult
* if the artifact we're tying to locate is a container pom (in which case, we'll return one
* positive RepoResult for each of the artifacts listed in that .pom file). For example:
* http://repo1.maven.org/maven2/nl/komponents/kovenant/kovenant/3.0.0/
*/
class RepoFinderCallable @Inject constructor(@Assisted val id: String,
@Assisted val repo: HostConfig, val localRepo: LocalRepo, val pomFactory: Pom.IFactory,
val dependencyManager: DependencyManager)
: Callable<List<RepoFinder .RepoResult>> {
interface IFactory {
fun create(@Assisted id: String, @Assisted repo: HostConfig) : RepoFinderCallable
}
override fun call(): List<RepoFinder.RepoResult> {
val repoUrl = repo.url
log(2, " Checking $repoUrl for $id")
val mavenId = MavenId.create(id)
val groupId = mavenId.groupId
val artifactId = mavenId.artifactId
if (mavenId.version == null) {
val ud = UnversionedDep(groupId, artifactId)
val isLocal = repoUrl.startsWith(FileDependency.PREFIX_FILE)
val path = ud.toMetadataXmlPath(false, isLocal)
val foundVersion = findCorrectVersionRelease(path, repoUrl)
// When looking up a versionless id, never return a SNAPSHOT
if (foundVersion != null && ! foundVersion.contains("SNAPSHOT")) {
return listOf(RepoFinder.RepoResult(repo, Version.of(foundVersion), repoUrl + path))
} else {
return listOf(RepoFinder.RepoResult(repo))
}
} else {
val version = Version.of(mavenId.version)
if (version.isSnapshot()) {
val dep = SimpleDep(mavenId)
val isLocal = repoUrl.startsWith(FileDependency.PREFIX_FILE)
val metadataXmlPath = dep.toMetadataXmlPath(false, isLocal, version.version)
val snapshotVersion =
if (isLocal) version
else findSnapshotVersion(metadataXmlPath, repoUrl, mavenId.version)
if (snapshotVersion != null) {
val url = repoUrl + dep.toDirectory(fileSystem = false, v = dep.version) +
dep.artifactId + "-" + snapshotVersion.noSnapshotVersion +
"-" + snapshotVersion.snapshotTimestamp + ".jar"
return listOf(RepoFinder.RepoResult(repo, version, url, snapshotVersion))
} else {
return listOf(RepoFinder.RepoResult(repo))
}
} else if (version.isRangedVersion() ) {
val foundVersion = findRangedVersion(SimpleDep(mavenId), repoUrl)
if (foundVersion != null) {
return listOf(RepoFinder.RepoResult(repo, foundVersion))
} else {
return listOf(RepoFinder.RepoResult(repo))
}
} else {
val dep = SimpleDep(mavenId)
// Try to find the jar file
val depPomFile = dep.toPomFile(dep.version)
val attemptPaths = listOf(dep.toJarFile(dep.version), dep.toAarFile(dep.version), depPomFile)
val attemptUrls = attemptPaths.map { repo.copy(url = repo.url + it )} +
attemptPaths.map { repo.copy(url = repo.url + File(it).parentFile.path.replace("\\", "/")) }
val firstFound = attemptUrls.map { Kurl(it)}.firstOrNull { it.exists }
if (firstFound != null) {
val url = firstFound.hostInfo.url
if (url.endsWith("ar")) {
log(3, "Result for $repoUrl for $id: $firstFound")
return listOf(RepoFinder.RepoResult(repo, Version.of(dep.version), firstFound.hostInfo.url))
} else if (url.endsWith(".pom")) {
log(2, "Found container pom: " + firstFound)
File(localRepo.toFullPath(depPomFile)).let { pomFile ->
pomFile.parentFile.mkdirs()
Kurl(HostConfig(url)).toFile(pomFile)
val pom2 = Pom2.parse(pomFile, dependencyManager).value
val result = arrayListOf<RepoFinder.RepoResult>()
if (pom2 != null) {
val dependencies = pom2.pomProject.dependencies
dependencies.map { it.id(pom2) }.forEach {
result.addAll(RepoFinderCallable(it, repo, localRepo, pomFactory,
dependencyManager).call())
}
} else {
warn("Couldn't parse $pomFile")
}
return result
}
} else {
return listOf(RepoFinder.RepoResult(repo, Version.of(dep.version), firstFound.hostInfo.url))
}
} else {
log(3, "Couldn't find $dep on $repoUrl")
return emptyList()
}
}
}
}
val XPATH_FACTORY = XPathFactory.newInstance();
val XPATH = XPATH_FACTORY.newXPath();
private fun findCorrectVersionRelease(metadataPath: String, repoUrl: String): String? {
val XPATHS = listOf(
XPATH.compile("/metadata/version"),
XPATH.compile("/metadata/versioning/latest"),
XPATH.compile("/metadata/versioning/release"))
// No version in this dependency, find out the most recent one by parsing maven-metadata.xml, if it exists
val url = repoUrl + metadataPath
try {
val doc = parseXml(url)
arrayListOf(XPATHS.forEach {
val result = it.evaluate(doc, XPathConstants.STRING) as String
if (! result.isEmpty()) {
return result
}
})
} catch(ex: Exception) {
log(2, "Couldn't find metadata at $url: ${ex.message}")
}
return null
}
fun findRangedVersion(dep: SimpleDep, repoUrl: String): Version? {
val l = listOf(dep.groupId.replace(".", "/"), dep.artifactId.replace(".", "/"), "maven-metadata.xml")
var metadataPath = l.joinToString("/")
val versionsXpath = XPATH.compile("/metadata/versioning/versions/version")
// No version in this dependency, find out the most recent one by parsing maven-metadata.xml, if it exists
val url = repoUrl + metadataPath
try {
val doc = parseXml(url)
val version = Version.of(dep.version)
if(version.isRangedVersion()) {
val versions = (versionsXpath.evaluate(doc, XPathConstants.NODESET) as NodeList)
.asElementList().map { Version.of(it.textContent) }
return version.select(versions)
} else {
return Version.of(XPATH.compile("/metadata/versioning/versions/version/$version")
.evaluate(doc, XPathConstants.STRING) as String)
}
} catch(ex: Exception) {
log(2, "Couldn't find metadata at ${url}")
}
return null
}
fun findSnapshotVersion(metadataPath: String, repoUrl: String, snapshotVersion: String): Version? {
val timestamp = XPATH.compile("/metadata/versioning/snapshot/timestamp")
val buildNumber = XPATH.compile("/metadata/versioning/snapshot/buildNumber")
// No version in this dependency, find out the most recent one by parsing maven-metadata.xml, if it exists
val url = repoUrl + metadataPath
try {
val doc = parseXml(url)
val ts = timestamp.evaluate(doc, XPathConstants.STRING)
val bn = buildNumber.evaluate(doc, XPathConstants.STRING)
if (! ts.toString().isEmpty() && ! bn.toString().isEmpty()) {
return Version(snapshotVersion, ts.toString() + "-" + bn.toString())
} else {
val lastUpdated = XPATH.compile("/metadata/versioning/lastUpdated")
if (! lastUpdated.toString().isEmpty()) {
return Version.of(lastUpdated.toString())
}
}
} catch(ex: Exception) {
log(2, "Couldn't find metadata at $url")
}
return null
}
}

View file

@ -4,10 +4,16 @@ import com.beust.kobalt.KobaltException
import com.beust.kobalt.api.IClasspathDependency
import com.beust.kobalt.api.Kobalt
import com.beust.kobalt.internal.KobaltSettings
import com.beust.kobalt.internal.KobaltSettingsXml
import com.beust.kobalt.maven.CompletedFuture
import com.beust.kobalt.maven.MavenId
import com.beust.kobalt.misc.KobaltLogger
import com.beust.kobalt.misc.Versions
import com.beust.kobalt.misc.log
import com.beust.kobalt.misc.warn
import com.google.common.cache.CacheBuilder
import com.google.common.cache.CacheLoader
import com.google.common.cache.LoadingCache
import com.google.inject.Inject
import org.eclipse.aether.artifact.Artifact
import org.eclipse.aether.artifact.DefaultArtifact
@ -30,27 +36,51 @@ class DependencyResult(val dependency: IClasspathDependency, val repoUrl: String
class KobaltAether @Inject constructor (val settings: KobaltSettings) {
val localRepo: File get() = File(settings.localRepo)
class MaybeArtifact(val result: DependencyResult?, val error: String?)
private val CACHE : LoadingCache<String, MaybeArtifact> = CacheBuilder.newBuilder()
.build(object : CacheLoader<String, MaybeArtifact>() {
override fun load(id: String): MaybeArtifact {
return doResolve(id)
}
})
/**
* Don't call this method directly, use `DepFactory` instead.
* Create an IClasspathDependency from a Kobalt id.
*/
fun create(id: String): IClasspathDependency {
val aether = Aether(localRepo)
val cr = aether.transitiveDependencies(DefaultArtifact(id))
val cr = aether.transitiveDependencies(DefaultArtifact(MavenId.toKobaltId(id)))
return if (cr != null) AetherDependency(cr.root.artifact)
else throw KobaltException("Couldn't resolve $id")
}
/**
* @return the latest artifact for the given group and artifactId.
*/
fun latestArtifact(group: String, artifactId: String, extension: String = "jar") : DependencyResult
= Aether(localRepo).latestArtifact(group, artifactId, extension).let {
DependencyResult(AetherDependency(it.artifact), it.repository.toString())
}
fun resolve(id: String): DependencyResult {
val results = Aether(localRepo).resolve(DefaultArtifact(id))
val result = CACHE.get(id)
if (result.result != null) return result.result
else throw KobaltException("Couldn't resolve $id")
}
/**
* Resolve the given Kobalt id.
*/
private fun doResolve(id: String): MaybeArtifact {
log(1, "Resolving $id")
val results = Aether(localRepo).resolve(DefaultArtifact(MavenId.toKobaltId(id)))
if (results != null && results.size > 0) {
return DependencyResult(AetherDependency(results[0].artifact), results[0].repository.toString())
return MaybeArtifact(
DependencyResult(AetherDependency(results[0].artifact), results[0].repository.toString()),
null)
} else {
throw KobaltException("Couldn't resolve $id")
return MaybeArtifact(null, "Couldn't locate $id")
}
}
}
@ -219,10 +249,13 @@ class AetherDependency(val artifact: Artifact): IClasspathDependency, Comparable
override fun toString() = id
}
//fun main(argv: Array<String>) {
// KobaltLogger.LOG_LEVEL = 2
// val aether = Aether()
// val latestResult = aether.latestArtifact("org.testng", "testng")
// val latest = latestResult.artifact
// println("Latest: " + latest.version + " " + latest.file)
//}
fun main(argv: Array<String>) {
KobaltLogger.LOG_LEVEL = 1
val id = "org.testng:testng:6.9.11"
val aether = KobaltAether(KobaltSettings(KobaltSettingsXml()))
val r = aether.resolve(id)
val r2 = aether.resolve(id)
val d = org.eclipse.aether.artifact.DefaultArtifact("org.testng:testng:6.9")
println("Artifact: " + d)
}

View file

@ -3,7 +3,9 @@ package com.beust.kobalt.app
import com.beust.kobalt.Args
import com.beust.kobalt.internal.KobaltSettings
import com.beust.kobalt.internal.PluginInfo
import com.beust.kobalt.maven.*
import com.beust.kobalt.maven.LocalRepo
import com.beust.kobalt.maven.Pom
import com.beust.kobalt.maven.PomGenerator
import com.beust.kobalt.misc.DependencyExecutor
import com.beust.kobalt.misc.KobaltExecutors
import com.beust.kobalt.plugin.publish.BintrayApi
@ -29,9 +31,7 @@ public open class MainModule(val args: Args, val settings: KobaltSettings) : Abs
PomGenerator.IFactory::class.java,
BintrayApi.IFactory::class.java,
Pom.IFactory::class.java,
BuildFileCompiler.IFactory::class.java,
ArtifactFetcher.IFactory::class.java,
RepoFinderCallable.IFactory::class.java)
BuildFileCompiler.IFactory::class.java)
.forEach {
install(builder.build(it))
}

View file

@ -164,16 +164,5 @@ class DownloadTest @Inject constructor(
val d = closure.filter { it.id.contains("eclipse-collections-api")}
Assert.assertEquals(d.size, 1)
}
@Test
fun containerPom() {
val repoResult = RepoFinderCallable("org.jetbrains.kotlin:kotlin-project:1.0.0",
HostConfig("http://repo1.maven.org/maven2/"),
localRepo, pomFactory, dependencyManager).call()
val rr = repoResult[0]
Assert.assertTrue(rr.found)
Assert.assertTrue(rr.localPath != null && rr.localPath!!.startsWith("junit/junit"))
Assert.assertEquals(rr.version.toString(), "4.12")
}
}

View file

@ -9,13 +9,13 @@ class MavenIdTest {
@DataProvider
fun dp() : Array<Array<out Any?>> {
return arrayOf(
arrayOf("javax.inject:javax.inject:", "javax.inject", "javax.inject", null, null, null),
arrayOf("javax.inject:javax.inject:", "javax.inject", "javax.inject", "(0,]", "jar", null),
arrayOf("org.jetbrains.kotlin:kotlin-compiler-embeddable:1.0.0-beta-1038",
"org.jetbrains.kotlin", "kotlin-compiler-embeddable", "1.0.0-beta-1038",
null, null),
arrayOf("com.google.inject:guice:4.0:no_aop",
"com.google.inject", "guice", "4.0", null, "no_aop"),
arrayOf("com.android.support:appcompat-v7:22.2.1@aar",
"jar", null),
arrayOf("com.google.inject:guice::no_aop:4.0",
"com.google.inject", "guice", "4.0", "jar", "no_aop"),
arrayOf("com.android.support:appcompat-v7:aar:22.2.1",
"com.android.support", "appcompat-v7", "22.2.1", "aar", null)
)
}

View file

@ -1,42 +0,0 @@
package com.beust.kobalt.maven
import com.beust.kobalt.TestModule
import org.testng.Assert
import org.testng.annotations.Test
import javax.inject.Inject
@Test
@org.testng.annotations.Guice(modules = arrayOf(TestModule::class))
class RemoteRepoTest @Inject constructor(val repoFinder: RepoFinder, val dependencyManager: DependencyManager){
@Test
fun mavenMetadata() {
val dep = dependencyManager.create("org.codehaus.groovy:groovy-all:")
// Note: this test might fail if a new version of Groovy gets uploaded, need
// to find a stable (i.e. abandoned) package
with(dep.id.split(":")[2]) {
Assert.assertTrue(this == "2.4.5" || this == "2.4.6")
}
}
@Test(enabled = false)
fun metadataForSnapshots() {
val jar = dependencyManager.create("org.apache.maven.wagon:wagon-provider-test:2.10-SNAPSHOT")
Assert.assertTrue(jar.jarFile.get().exists())
}
fun resolveAarWithVersion() {
val repoResult = repoFinder.findCorrectRepo("com.jakewharton.timber:timber:4.1.0")
with(repoResult) {
Assert.assertEquals(path, "com/jakewharton/timber/timber/4.1.0/timber-4.1.0.aar")
}
}
@Test(groups = arrayOf("broken"), enabled = false)
fun resolveAarWithoutVersion() {
val repoResult = repoFinder.findCorrectRepo("com.jakewharton.timber:timber:")
with(repoResult) {
Assert.assertEquals(path, "com/jakewharton/timber/timber/4.1.0/timber-4.1.0.aar")
}
}
}