rife2/bld
rife2/bld
2
0
Fork 0
mirror of https://github.com/ethauvin/bld.git synced 2025-04-25 16:27:11 -07:00
Code Activity

More changes to cache format, now the dependency tree generation is cached

Browse source
This commit is contained in:
Geert Bevin 2024-07-20 11:30:35 -04:00
parent d24e9d223c
commit 3ec2cc68ce
5 changed files with 257 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/main/java/rife/bld/BaseProject.java
View file

@ -1593,14 +1593,15 @@ public class BaseProject extends BuildExecutor {
private void performAutoDownloadPurge() {
var resolution = new VersionResolution(properties());
var cache = new BldCache(libBldDirectory(), resolution);
cache.fingerprintDependencies(repositories(), dependencies(), downloadSources(), downloadJavadoc());
if (cache.isDependenciesHashValid()) {
cache.fingerprintDependencies(repositories(), dependencies());
if (cache.isDependenciesCacheValid(downloadSources(), downloadJavadoc())) {
return;
}
try {
executeAutoDownloadPurge();
cache.cacheDependenciesDownloads(downloadSources(), downloadJavadoc());
cache.writeCache();
} catch (Exception e) {
throw new RuntimeException(e);

164
src/main/java/rife/bld/BldCache.java
View file

@ -33,6 +33,9 @@ public class BldCache {
private static final String PROPERTY_SUFFIX_HASH = ".hash";
private static final String PROPERTY_SUFFIX_LOCAL = ".local";
private static final String PROPERTY_SUFFIX_DOWNLOAD_SOURCES = ".download.sources";
private static final String PROPERTY_SUFFIX_DOWNLOAD_JAVADOC = ".download.javadoc";
private static final String PROPERTY_SUFFIX_DEPENDENCY_TREE = ".dependency.tree";
private static final String WRAPPER_PROPERTIES_HASH = Wrapper.WRAPPER_PROPERTIES + PROPERTY_SUFFIX_HASH;
private static final String BLD_BUILD_HASH = "bld-build" + PROPERTY_SUFFIX_HASH;
@ -40,14 +43,32 @@ public class BldCache {
private static final String PROPERTY_EXTENSIONS_PREFIX = "bld.extensions";
private static final String PROPERTY_EXTENSIONS_HASH = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_HASH;
private static final String PROPERTY_EXTENSIONS_LOCAL = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_LOCAL;
private static final String PROPERTY_EXTENSIONS_DOWNLOAD_SOURCES = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_DOWNLOAD_SOURCES;
private static final String PROPERTY_EXTENSIONS_DOWNLOAD_JAVADOC = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_DOWNLOAD_JAVADOC;
private static final String PROPERTY_EXTENSIONS_DEPENDENCY_TREE = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_DEPENDENCY_TREE;
private static final String PROPERTY_DEPENDENCIES_PREFIX = "bld.dependencies";
private static final String PROPERTY_DEPENDENCIES_HASH = PROPERTY_DEPENDENCIES_PREFIX + PROPERTY_SUFFIX_HASH;
private static final String PROPERTY_DEPENDENCIES_DOWNLOAD_SOURCES = PROPERTY_DEPENDENCIES_PREFIX + PROPERTY_SUFFIX_DOWNLOAD_SOURCES;
private static final String PROPERTY_DEPENDENCIES_DOWNLOAD_JAVADOC = PROPERTY_DEPENDENCIES_PREFIX + PROPERTY_SUFFIX_DOWNLOAD_JAVADOC;
private static final String PROPERTY_DEPENDENCIES_COMPILE_DEPENDENCY_TREE = PROPERTY_DEPENDENCIES_PREFIX + ".compile" + PROPERTY_SUFFIX_DEPENDENCY_TREE;
private static final String PROPERTY_DEPENDENCIES_PROVIDED_DEPENDENCY_TREE = PROPERTY_DEPENDENCIES_PREFIX + ".provided" + PROPERTY_SUFFIX_DEPENDENCY_TREE;
private static final String PROPERTY_DEPENDENCIES_RUNTIME_DEPENDENCY_TREE = PROPERTY_DEPENDENCIES_PREFIX + ".runtime" + PROPERTY_SUFFIX_DEPENDENCY_TREE;
private static final String PROPERTY_DEPENDENCIES_TEST_DEPENDENCY_TREE = PROPERTY_DEPENDENCIES_PREFIX + ".test" + PROPERTY_SUFFIX_DEPENDENCY_TREE;
private final File bldLibDir_;
private final VersionResolution resolution_;
private String extensionsHash_;
private Boolean extensionsDownloadSources_;
private Boolean extensionsDownloadJavadocs_;
private String extensionsDependencyTree_;
private String dependenciesHash_;
private Boolean dependenciesDownloadSources_;
private Boolean dependenciesDownloadJavadocs_;
private String dependenciesCompileDependencyTree_;
private String dependenciesProvidedDependencyTree_;
private String dependenciesRuntimeDependencyTree_;
private String dependenciesTestDependencyTree_;
public BldCache(File bldLibDir, VersionResolution resolution) {
bldLibDir_ = bldLibDir;
@ -57,12 +78,12 @@ public class BldCache {
new File(bldLibDir, BLD_BUILD_HASH).delete();
}
public void fingerprintExtensions(Collection<String> repositories, Collection<String> extensions, boolean downloadSources, boolean downloadJavadoc) {
public void fingerprintExtensions(Collection<String> repositories, Collection<String> extensions) {
try {
var overrides_fp = String.join("\n", resolution_.versionOverrides().entrySet().stream().map(e -> e.getKey() + ":" + e.getValue()).toList());
var repositories_fp = String.join("\n", repositories);
var extensions_fp = String.join("\n", extensions);
var fingerprint = overrides_fp + "\n" + repositories_fp + "\n" + extensions_fp + "\n" + downloadSources + "\n" + downloadJavadoc;
var fingerprint = overrides_fp + "\n" + repositories_fp + "\n" + extensions_fp + "\n";
var digest = MessageDigest.getInstance("SHA-1");
digest.update(fingerprint.getBytes(StandardCharsets.UTF_8));
@ -73,7 +94,20 @@ public class BldCache {
}
}
public void fingerprintDependencies(List<Repository> repositories, DependencyScopes dependencies, boolean downloadSources, boolean downloadJavadoc) {
public void cacheExtensionsDownloads(boolean downloadSources, boolean downloadJavadoc) {
extensionsDownloadSources_ = downloadSources;
extensionsDownloadJavadocs_ = downloadJavadoc;
}
public void cacheExtensionsDependencyTree(String dependencyTree) {
extensionsDependencyTree_ = dependencyTree;
}
public String getCachedExtensionsDependencyTree() {
return hashProperties().getProperty(PROPERTY_EXTENSIONS_DEPENDENCY_TREE);
}
public void fingerprintDependencies(List<Repository> repositories, DependencyScopes dependencies) {
var finger_print = new StringBuilder();
finger_print.append(String.join("\n", resolution_.versionOverrides().entrySet().stream().map(e -> e.getKey() + ":" + e.getValue()).toList()));
for (var repository : repositories) {
@ -90,10 +124,6 @@ public class BldCache {
}
}
}
finger_print.append(downloadSources)
.append('\n')
.append(downloadJavadoc)
.append('\n');
try {
var digest = MessageDigest.getInstance("SHA-1");
@ -105,10 +135,64 @@ public class BldCache {
}
}
public void cacheDependenciesDownloads(boolean downloadSources, boolean downloadJavadoc) {
dependenciesDownloadSources_ = downloadSources;
dependenciesDownloadJavadocs_ = downloadJavadoc;
}
public void cacheDependenciesCompileDependencyTree(String compileTree) {
dependenciesCompileDependencyTree_ = compileTree;
}
public String getCachedDependenciesCompileDependencyTree() {
return hashProperties().getProperty(PROPERTY_DEPENDENCIES_COMPILE_DEPENDENCY_TREE);
}
public void cacheDependenciesProvidedDependencyTree(String providedTree) {
dependenciesProvidedDependencyTree_ = providedTree;
}
public String getCachedDependenciesProvidedDependencyTree() {
return hashProperties().getProperty(PROPERTY_DEPENDENCIES_PROVIDED_DEPENDENCY_TREE);
}
public void cacheDependenciesRuntimeDependencyTree(String runtimeTree) {
dependenciesRuntimeDependencyTree_ = runtimeTree;
}
public String getCachedDependenciesRuntimeDependencyTree() {
return hashProperties().getProperty(PROPERTY_DEPENDENCIES_RUNTIME_DEPENDENCY_TREE);
}
public void cacheDependenciesTestDependencyTree(String testTree) {
dependenciesTestDependencyTree_ = testTree;
}
public String getCachedDependenciesTestDependencyTree() {
return hashProperties().getProperty(PROPERTY_DEPENDENCIES_TEST_DEPENDENCY_TREE);
}
public boolean isExtensionHashValid() {
return validateExtensionsHash(extensionsHash_);
}
public boolean isExtensionsCacheValid(boolean downloadSources, boolean downloadJavadoc) {
var properties = hashProperties();
if (properties.isEmpty()) {
return false;
}
if (downloadSources != Boolean.parseBoolean(properties.getProperty(PROPERTY_EXTENSIONS_DOWNLOAD_SOURCES))) {
return false;
}
if (downloadJavadoc != Boolean.parseBoolean(properties.getProperty(PROPERTY_EXTENSIONS_DOWNLOAD_JAVADOC))) {
return false;
}
return validateExtensionsHash(extensionsHash_);
}
private File getCacheFile() {
return new File(bldLibDir_, BLD_CACHE);
}
@ -170,6 +254,23 @@ public class BldCache {
return validateDependenciesHash(dependenciesHash_);
}
public boolean isDependenciesCacheValid(boolean downloadSources, boolean downloadJavadoc) {
var properties = hashProperties();
if (properties.isEmpty()) {
return false;
}
if (downloadSources != Boolean.parseBoolean(properties.getProperty(PROPERTY_DEPENDENCIES_DOWNLOAD_SOURCES))) {
return false;
}
if (downloadJavadoc != Boolean.parseBoolean(properties.getProperty(PROPERTY_DEPENDENCIES_DOWNLOAD_JAVADOC))) {
return false;
}
return validateDependenciesHash(dependenciesHash_);
}
private boolean validateDependenciesHash(String hash) {
var properties = hashProperties();
if (properties.isEmpty()) {
@ -188,7 +289,22 @@ public class BldCache {
try {
if (extensionsHash_ != null) {
properties.put(PROPERTY_EXTENSIONS_HASH, extensionsHash_);
if (!extensionsHash_.equals(properties.get(PROPERTY_EXTENSIONS_HASH))) {
properties.put(PROPERTY_EXTENSIONS_HASH, extensionsHash_);
properties.remove(PROPERTY_EXTENSIONS_DEPENDENCY_TREE);
}
if (extensionsDependencyTree_ != null) {
properties.put(PROPERTY_EXTENSIONS_DEPENDENCY_TREE, extensionsDependencyTree_);
}
}
if (extensionsDownloadSources_ != null) {
properties.put(PROPERTY_EXTENSIONS_DOWNLOAD_SOURCES, String.valueOf(extensionsDownloadSources_));
}
if (extensionsDownloadJavadocs_ != null) {
properties.put(PROPERTY_EXTENSIONS_DOWNLOAD_JAVADOC, String.valueOf(extensionsDownloadJavadocs_));
}
if (extensionsLocalArtifacts != null) {
@ -205,7 +321,37 @@ public class BldCache {
}
if (dependenciesHash_ != null) {
properties.put(PROPERTY_DEPENDENCIES_HASH, dependenciesHash_);
if (!dependenciesHash_.equals(properties.get(PROPERTY_DEPENDENCIES_HASH))) {
properties.put(PROPERTY_DEPENDENCIES_HASH, dependenciesHash_);
properties.remove(PROPERTY_DEPENDENCIES_COMPILE_DEPENDENCY_TREE);
properties.remove(PROPERTY_DEPENDENCIES_PROVIDED_DEPENDENCY_TREE);
properties.remove(PROPERTY_DEPENDENCIES_RUNTIME_DEPENDENCY_TREE);
properties.remove(PROPERTY_DEPENDENCIES_TEST_DEPENDENCY_TREE);
}
if (dependenciesCompileDependencyTree_ != null) {
properties.put(PROPERTY_DEPENDENCIES_COMPILE_DEPENDENCY_TREE, dependenciesCompileDependencyTree_);
}
if (dependenciesProvidedDependencyTree_ != null) {
properties.put(PROPERTY_DEPENDENCIES_PROVIDED_DEPENDENCY_TREE, dependenciesProvidedDependencyTree_);
}
if (dependenciesRuntimeDependencyTree_ != null) {
properties.put(PROPERTY_DEPENDENCIES_RUNTIME_DEPENDENCY_TREE, dependenciesRuntimeDependencyTree_);
}
if (dependenciesTestDependencyTree_ != null) {
properties.put(PROPERTY_DEPENDENCIES_TEST_DEPENDENCY_TREE, dependenciesTestDependencyTree_);
}
}
if (dependenciesDownloadSources_ != null) {
properties.put(PROPERTY_DEPENDENCIES_DOWNLOAD_SOURCES, String.valueOf(dependenciesDownloadSources_));
}
if (dependenciesDownloadJavadocs_ != null) {
properties.put(PROPERTY_DEPENDENCIES_DOWNLOAD_JAVADOC, String.valueOf(dependenciesDownloadJavadocs_));
}
bldLibDir_.mkdirs();

101
src/main/java/rife/bld/operations/DependencyTreeOperation.java
View file

@ -5,12 +5,14 @@
package rife.bld.operations;
import rife.bld.BaseProject;
import rife.bld.BldCache;
import rife.bld.BldVersion;
import rife.bld.BuildExecutor;
import rife.bld.dependencies.*;
import rife.bld.wrapper.Wrapper;
import rife.ioc.HierarchicalProperties;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@ -34,6 +36,7 @@ public class DependencyTreeOperation extends AbstractOperation<DependencyTreeOpe
private final DependencyScopes extensionDependencies_ = new DependencyScopes();
private final StringBuilder dependencyTree_ = new StringBuilder();
private File libBldDir_ = null;
/**
* Performs the dependency tree operation.
@ -46,11 +49,97 @@ public class DependencyTreeOperation extends AbstractOperation<DependencyTreeOpe
return;
}
var extensions_tree = executeGenerateExtensionsDependencies();
var compile_tree = executeGenerateCompileDependencies();
var provided_tree = executeGenerateProvidedDependencies();
var runtime_tree = executeGenerateRuntimeDependencies();
var test_tree = executeGenerateTestDependencies();
// calculate the dependency tree of the extensions, using the cache if possible
String extensions_tree = null;
BldCache extensions_cache = null;
if (libBldDir_ != null) {
extensions_cache = new BldCache(libBldDir_, new VersionResolution(extensionProperties()));
extensions_cache.fingerprintExtensions(
extensionRepositories().stream().map(Repository::toString).toList(),
extensionDependencies().scope(compile).stream().map(Dependency::toString).toList());
if (extensions_cache.isExtensionHashValid()) {
var cached_tree = extensions_cache.getCachedExtensionsDependencyTree();
if (cached_tree != null) {
extensions_tree = cached_tree;
}
}
}
if (extensions_tree == null) {
extensions_tree = executeGenerateExtensionsDependencies();
if (extensions_cache != null) {
extensions_cache.cacheExtensionsDependencyTree(extensions_tree);
extensions_cache.writeCache();
}
}
// calculate the dependency tree of the dependencies, using the cache if possible
String compile_tree = null;
String provided_tree = null;
String runtime_tree = null;
String test_tree = null;
BldCache dependencies_cache = null;
if (libBldDir_ != null) {
dependencies_cache = new BldCache(libBldDir_, new VersionResolution(properties()));
dependencies_cache.fingerprintDependencies(repositories(), dependencies());
if (dependencies_cache.isDependenciesHashValid()) {
var cached_compile_tree = dependencies_cache.getCachedDependenciesCompileDependencyTree();
if (cached_compile_tree != null) {
compile_tree = cached_compile_tree;
}
var cached_provided_tree = dependencies_cache.getCachedDependenciesProvidedDependencyTree();
if (cached_provided_tree != null) {
provided_tree = cached_provided_tree;
}
var cached_runtime_tree = dependencies_cache.getCachedDependenciesRuntimeDependencyTree();
if (cached_runtime_tree != null) {
runtime_tree = cached_runtime_tree;
}
var cached_test_tree = dependencies_cache.getCachedDependenciesTestDependencyTree();
if (cached_test_tree != null) {
test_tree = cached_test_tree;
}
}
}
var write_dependencies_cache = false;
if (compile_tree == null) {
compile_tree = executeGenerateCompileDependencies();
if (dependencies_cache != null) {
dependencies_cache.cacheDependenciesCompileDependencyTree(compile_tree);
write_dependencies_cache = true;
}
}
if (provided_tree == null) {
provided_tree = executeGenerateProvidedDependencies();
if (dependencies_cache != null) {
dependencies_cache.cacheDependenciesProvidedDependencyTree(provided_tree);
write_dependencies_cache = true;
}
}
if (runtime_tree == null) {
runtime_tree = executeGenerateRuntimeDependencies();
if (dependencies_cache != null) {
dependencies_cache.cacheDependenciesRuntimeDependencyTree(runtime_tree);
write_dependencies_cache = true;
}
}
if (test_tree == null) {
test_tree = executeGenerateTestDependencies();
if (dependencies_cache != null) {
dependencies_cache.cacheDependenciesTestDependencyTree(test_tree);
write_dependencies_cache = true;
}
}
if (write_dependencies_cache) {
dependencies_cache.writeCache();
}
// output the dependency trees
dependencyTree_.setLength(0);
dependencyTree_.append(extensions_tree);
dependencyTree_.append(System.lineSeparator());
@ -143,6 +232,8 @@ public class DependencyTreeOperation extends AbstractOperation<DependencyTreeOpe
* @since 1.5.21
*/
public DependencyTreeOperation fromProject(BaseProject project) {
libBldDir_ = project.libBldDirectory();
// add the repositories and dependencies from the extensions
var wrapper = new Wrapper();
wrapper.currentDir(project.workDirectory());

6
src/main/java/rife/bld/wrapper/WrapperExtensionResolver.java
View file

@ -64,14 +64,13 @@ public class WrapperExtensionResolver {
downloadJavadoc_ = downloadJavadoc;
cache_.fingerprintExtensions(
repositories_.stream().map(Objects::toString).toList(),
dependencies_.stream().map(Objects::toString).toList(),
downloadSources, downloadJavadoc);
dependencies_.stream().map(Objects::toString).toList());
}
public void updateExtensions() {
// verify and update the fingerprint hash file,
// don't update the extensions if the hash is identical
if (cache_.isExtensionHashValid()) {
if (cache_.isExtensionsCacheValid(downloadSources_, downloadJavadoc_)) {
return;
}
@ -81,6 +80,7 @@ public class WrapperExtensionResolver {
// purge the files that are not part of the latest extensions anymore
purgeExtensionDependencies(filenames);
cache_.cacheExtensionsDownloads(downloadSources_, downloadJavadoc_);
cache_.writeCache(localArtifacts_);
if (headerPrinted_) {