From 3ec2cc68cea0ff49a9b19669ba2bd879eee32be6 Mon Sep 17 00:00:00 2001
From: Geert Bevin <gbevin@uwyn.com>
Date: Sat, 20 Jul 2024 11:30:35 -0400
Subject: [PATCH] More changes to cache format, now the dependency tree
 generation is cached

---
 lib/bld/bld-wrapper.jar                       | Bin 29778 -> 29778 bytes
 src/main/java/rife/bld/BaseProject.java       |   5 +-
 src/main/java/rife/bld/BldCache.java          | 164 +++++++++++++++++-
 .../operations/DependencyTreeOperation.java   | 101 ++++++++++-
 .../bld/wrapper/WrapperExtensionResolver.java |   6 +-
 5 files changed, 257 insertions(+), 19 deletions(-)

diff --git a/lib/bld/bld-wrapper.jar b/lib/bld/bld-wrapper.jar
index 66637530ca1c10e6bf225ea0580cd389c4a1bacf..f92a21ef0bc68bc25566e39227c4e2d9cc5398fe 100644
GIT binary patch
delta 140
zcmccgg7MM|MxFp~W)=|!4h{~6?a>o?>Y27jZ|q%c38W_fveBBnph$3Yu2nY^kiYqt
zO%Isvu>Y+K;<x5rU<A?O`9>h|$=!vXAnID73y6{_@&ZvAMWG;SKZFu1b_4O^isQk8
L#Q`8nu*4An@5VP^

delta 140
zcmccgg7MM|MxFp~W)=|!4h{~6g(4Gq>X{abZ0ucZ38W_fveBBnph$3Yu2nY^kiYqt
zO%Isvu>Y+K;<x5rU<A?O`9>h|$=!vXAnID73y6{_@&ZvAMWG;SKZFu1b_4O^isQk8
L#Q`8nu*4Anj;}G0

diff --git a/src/main/java/rife/bld/BaseProject.java b/src/main/java/rife/bld/BaseProject.java
index 6833a6d..688c137 100644
--- a/src/main/java/rife/bld/BaseProject.java
+++ b/src/main/java/rife/bld/BaseProject.java
@@ -1593,14 +1593,15 @@ public class BaseProject extends BuildExecutor {
     private void performAutoDownloadPurge() {
         var resolution = new VersionResolution(properties());
         var cache = new BldCache(libBldDirectory(), resolution);
-        cache.fingerprintDependencies(repositories(), dependencies(), downloadSources(), downloadJavadoc());
-        if (cache.isDependenciesHashValid()) {
+        cache.fingerprintDependencies(repositories(), dependencies());
+        if (cache.isDependenciesCacheValid(downloadSources(), downloadJavadoc())) {
             return;
         }
 
         try {
             executeAutoDownloadPurge();
 
+            cache.cacheDependenciesDownloads(downloadSources(), downloadJavadoc());
             cache.writeCache();
         } catch (Exception e) {
             throw new RuntimeException(e);
diff --git a/src/main/java/rife/bld/BldCache.java b/src/main/java/rife/bld/BldCache.java
index dbe1764..1b312ae 100644
--- a/src/main/java/rife/bld/BldCache.java
+++ b/src/main/java/rife/bld/BldCache.java
@@ -33,6 +33,9 @@ public class BldCache {
 
     private static final String PROPERTY_SUFFIX_HASH = ".hash";
     private static final String PROPERTY_SUFFIX_LOCAL = ".local";
+    private static final String PROPERTY_SUFFIX_DOWNLOAD_SOURCES = ".download.sources";
+    private static final String PROPERTY_SUFFIX_DOWNLOAD_JAVADOC = ".download.javadoc";
+    private static final String PROPERTY_SUFFIX_DEPENDENCY_TREE = ".dependency.tree";
 
     private static final String WRAPPER_PROPERTIES_HASH = Wrapper.WRAPPER_PROPERTIES + PROPERTY_SUFFIX_HASH;
     private static final String BLD_BUILD_HASH = "bld-build" + PROPERTY_SUFFIX_HASH;
@@ -40,14 +43,32 @@ public class BldCache {
     private static final String PROPERTY_EXTENSIONS_PREFIX = "bld.extensions";
     private static final String PROPERTY_EXTENSIONS_HASH = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_HASH;
     private static final String PROPERTY_EXTENSIONS_LOCAL = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_LOCAL;
+    private static final String PROPERTY_EXTENSIONS_DOWNLOAD_SOURCES = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_DOWNLOAD_SOURCES;
+    private static final String PROPERTY_EXTENSIONS_DOWNLOAD_JAVADOC = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_DOWNLOAD_JAVADOC;
+    private static final String PROPERTY_EXTENSIONS_DEPENDENCY_TREE = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_DEPENDENCY_TREE;
 
     private static final String PROPERTY_DEPENDENCIES_PREFIX = "bld.dependencies";
     private static final String PROPERTY_DEPENDENCIES_HASH = PROPERTY_DEPENDENCIES_PREFIX + PROPERTY_SUFFIX_HASH;
+    private static final String PROPERTY_DEPENDENCIES_DOWNLOAD_SOURCES = PROPERTY_DEPENDENCIES_PREFIX + PROPERTY_SUFFIX_DOWNLOAD_SOURCES;
+    private static final String PROPERTY_DEPENDENCIES_DOWNLOAD_JAVADOC = PROPERTY_DEPENDENCIES_PREFIX + PROPERTY_SUFFIX_DOWNLOAD_JAVADOC;
+    private static final String PROPERTY_DEPENDENCIES_COMPILE_DEPENDENCY_TREE = PROPERTY_DEPENDENCIES_PREFIX + ".compile" + PROPERTY_SUFFIX_DEPENDENCY_TREE;
+    private static final String PROPERTY_DEPENDENCIES_PROVIDED_DEPENDENCY_TREE = PROPERTY_DEPENDENCIES_PREFIX + ".provided" + PROPERTY_SUFFIX_DEPENDENCY_TREE;
+    private static final String PROPERTY_DEPENDENCIES_RUNTIME_DEPENDENCY_TREE = PROPERTY_DEPENDENCIES_PREFIX + ".runtime" + PROPERTY_SUFFIX_DEPENDENCY_TREE;
+    private static final String PROPERTY_DEPENDENCIES_TEST_DEPENDENCY_TREE = PROPERTY_DEPENDENCIES_PREFIX + ".test" + PROPERTY_SUFFIX_DEPENDENCY_TREE;
 
     private final File bldLibDir_;
     private final VersionResolution resolution_;
     private String extensionsHash_;
+    private Boolean extensionsDownloadSources_;
+    private Boolean extensionsDownloadJavadocs_;
+    private String extensionsDependencyTree_;
     private String dependenciesHash_;
+    private Boolean dependenciesDownloadSources_;
+    private Boolean dependenciesDownloadJavadocs_;
+    private String dependenciesCompileDependencyTree_;
+    private String dependenciesProvidedDependencyTree_;
+    private String dependenciesRuntimeDependencyTree_;
+    private String dependenciesTestDependencyTree_;
 
     public BldCache(File bldLibDir, VersionResolution resolution) {
         bldLibDir_ = bldLibDir;
@@ -57,12 +78,12 @@ public class BldCache {
         new File(bldLibDir, BLD_BUILD_HASH).delete();
     }
 
-    public void fingerprintExtensions(Collection<String> repositories, Collection<String> extensions, boolean downloadSources, boolean downloadJavadoc) {
+    public void fingerprintExtensions(Collection<String> repositories, Collection<String> extensions) {
         try {
             var overrides_fp = String.join("\n", resolution_.versionOverrides().entrySet().stream().map(e -> e.getKey() + ":" + e.getValue()).toList());
             var repositories_fp = String.join("\n", repositories);
             var extensions_fp = String.join("\n", extensions);
-            var fingerprint = overrides_fp + "\n" + repositories_fp + "\n" + extensions_fp + "\n" + downloadSources + "\n" + downloadJavadoc;
+            var fingerprint = overrides_fp + "\n" + repositories_fp + "\n" + extensions_fp + "\n";
             var digest = MessageDigest.getInstance("SHA-1");
             digest.update(fingerprint.getBytes(StandardCharsets.UTF_8));
 
@@ -73,7 +94,20 @@ public class BldCache {
         }
     }
 
-    public void fingerprintDependencies(List<Repository> repositories, DependencyScopes dependencies, boolean downloadSources, boolean downloadJavadoc) {
+    public void cacheExtensionsDownloads(boolean downloadSources, boolean downloadJavadoc) {
+        extensionsDownloadSources_ = downloadSources;
+        extensionsDownloadJavadocs_ = downloadJavadoc;
+    }
+
+    public void cacheExtensionsDependencyTree(String dependencyTree) {
+        extensionsDependencyTree_ = dependencyTree;
+    }
+
+    public String getCachedExtensionsDependencyTree() {
+        return hashProperties().getProperty(PROPERTY_EXTENSIONS_DEPENDENCY_TREE);
+    }
+
+    public void fingerprintDependencies(List<Repository> repositories, DependencyScopes dependencies) {
         var finger_print = new StringBuilder();
         finger_print.append(String.join("\n", resolution_.versionOverrides().entrySet().stream().map(e -> e.getKey() + ":" + e.getValue()).toList()));
         for (var repository : repositories) {
@@ -90,10 +124,6 @@ public class BldCache {
                 }
             }
         }
-        finger_print.append(downloadSources)
-            .append('\n')
-            .append(downloadJavadoc)
-            .append('\n');
 
         try {
             var digest = MessageDigest.getInstance("SHA-1");
@@ -105,10 +135,64 @@ public class BldCache {
         }
     }
 
+    public void cacheDependenciesDownloads(boolean downloadSources, boolean downloadJavadoc) {
+        dependenciesDownloadSources_ = downloadSources;
+        dependenciesDownloadJavadocs_ = downloadJavadoc;
+    }
+
+    public void cacheDependenciesCompileDependencyTree(String compileTree) {
+        dependenciesCompileDependencyTree_ = compileTree;
+    }
+
+    public String getCachedDependenciesCompileDependencyTree() {
+        return hashProperties().getProperty(PROPERTY_DEPENDENCIES_COMPILE_DEPENDENCY_TREE);
+    }
+
+    public void cacheDependenciesProvidedDependencyTree(String providedTree) {
+        dependenciesProvidedDependencyTree_ = providedTree;
+    }
+
+    public String getCachedDependenciesProvidedDependencyTree() {
+        return hashProperties().getProperty(PROPERTY_DEPENDENCIES_PROVIDED_DEPENDENCY_TREE);
+    }
+
+    public void cacheDependenciesRuntimeDependencyTree(String runtimeTree) {
+        dependenciesRuntimeDependencyTree_ = runtimeTree;
+    }
+
+    public String getCachedDependenciesRuntimeDependencyTree() {
+        return hashProperties().getProperty(PROPERTY_DEPENDENCIES_RUNTIME_DEPENDENCY_TREE);
+    }
+
+    public void cacheDependenciesTestDependencyTree(String testTree) {
+        dependenciesTestDependencyTree_ = testTree;
+    }
+
+    public String getCachedDependenciesTestDependencyTree() {
+        return hashProperties().getProperty(PROPERTY_DEPENDENCIES_TEST_DEPENDENCY_TREE);
+    }
+
     public boolean isExtensionHashValid() {
         return validateExtensionsHash(extensionsHash_);
     }
 
+    public boolean isExtensionsCacheValid(boolean downloadSources, boolean downloadJavadoc) {
+        var properties = hashProperties();
+        if (properties.isEmpty()) {
+            return false;
+        }
+
+        if (downloadSources != Boolean.parseBoolean(properties.getProperty(PROPERTY_EXTENSIONS_DOWNLOAD_SOURCES))) {
+            return false;
+        }
+
+        if (downloadJavadoc != Boolean.parseBoolean(properties.getProperty(PROPERTY_EXTENSIONS_DOWNLOAD_JAVADOC))) {
+            return false;
+        }
+
+        return validateExtensionsHash(extensionsHash_);
+    }
+
     private File getCacheFile() {
         return new File(bldLibDir_, BLD_CACHE);
     }
@@ -170,6 +254,23 @@ public class BldCache {
         return validateDependenciesHash(dependenciesHash_);
     }
 
+    public boolean isDependenciesCacheValid(boolean downloadSources, boolean downloadJavadoc) {
+        var properties = hashProperties();
+        if (properties.isEmpty()) {
+            return false;
+        }
+
+        if (downloadSources != Boolean.parseBoolean(properties.getProperty(PROPERTY_DEPENDENCIES_DOWNLOAD_SOURCES))) {
+            return false;
+        }
+
+        if (downloadJavadoc != Boolean.parseBoolean(properties.getProperty(PROPERTY_DEPENDENCIES_DOWNLOAD_JAVADOC))) {
+            return false;
+        }
+
+        return validateDependenciesHash(dependenciesHash_);
+    }
+
     private boolean validateDependenciesHash(String hash) {
         var properties = hashProperties();
         if (properties.isEmpty()) {
@@ -188,7 +289,22 @@ public class BldCache {
 
         try {
             if (extensionsHash_ != null) {
-                properties.put(PROPERTY_EXTENSIONS_HASH, extensionsHash_);
+                if (!extensionsHash_.equals(properties.get(PROPERTY_EXTENSIONS_HASH))) {
+                    properties.put(PROPERTY_EXTENSIONS_HASH, extensionsHash_);
+                    properties.remove(PROPERTY_EXTENSIONS_DEPENDENCY_TREE);
+                }
+
+                if (extensionsDependencyTree_ != null) {
+                    properties.put(PROPERTY_EXTENSIONS_DEPENDENCY_TREE, extensionsDependencyTree_);
+                }
+            }
+
+            if (extensionsDownloadSources_ != null) {
+                properties.put(PROPERTY_EXTENSIONS_DOWNLOAD_SOURCES, String.valueOf(extensionsDownloadSources_));
+            }
+
+            if (extensionsDownloadJavadocs_ != null) {
+                properties.put(PROPERTY_EXTENSIONS_DOWNLOAD_JAVADOC, String.valueOf(extensionsDownloadJavadocs_));
             }
 
             if (extensionsLocalArtifacts != null) {
@@ -205,7 +321,37 @@ public class BldCache {
             }
 
             if (dependenciesHash_ != null) {
-                properties.put(PROPERTY_DEPENDENCIES_HASH, dependenciesHash_);
+                if (!dependenciesHash_.equals(properties.get(PROPERTY_DEPENDENCIES_HASH))) {
+                    properties.put(PROPERTY_DEPENDENCIES_HASH, dependenciesHash_);
+                    properties.remove(PROPERTY_DEPENDENCIES_COMPILE_DEPENDENCY_TREE);
+                    properties.remove(PROPERTY_DEPENDENCIES_PROVIDED_DEPENDENCY_TREE);
+                    properties.remove(PROPERTY_DEPENDENCIES_RUNTIME_DEPENDENCY_TREE);
+                    properties.remove(PROPERTY_DEPENDENCIES_TEST_DEPENDENCY_TREE);
+                }
+
+                if (dependenciesCompileDependencyTree_ != null) {
+                    properties.put(PROPERTY_DEPENDENCIES_COMPILE_DEPENDENCY_TREE, dependenciesCompileDependencyTree_);
+                }
+
+                if (dependenciesProvidedDependencyTree_ != null) {
+                    properties.put(PROPERTY_DEPENDENCIES_PROVIDED_DEPENDENCY_TREE, dependenciesProvidedDependencyTree_);
+                }
+
+                if (dependenciesRuntimeDependencyTree_ != null) {
+                    properties.put(PROPERTY_DEPENDENCIES_RUNTIME_DEPENDENCY_TREE, dependenciesRuntimeDependencyTree_);
+                }
+
+                if (dependenciesTestDependencyTree_ != null) {
+                    properties.put(PROPERTY_DEPENDENCIES_TEST_DEPENDENCY_TREE, dependenciesTestDependencyTree_);
+                }
+            }
+
+            if (dependenciesDownloadSources_ != null) {
+                properties.put(PROPERTY_DEPENDENCIES_DOWNLOAD_SOURCES, String.valueOf(dependenciesDownloadSources_));
+            }
+
+            if (dependenciesDownloadJavadocs_ != null) {
+                properties.put(PROPERTY_DEPENDENCIES_DOWNLOAD_JAVADOC, String.valueOf(dependenciesDownloadJavadocs_));
             }
 
             bldLibDir_.mkdirs();
diff --git a/src/main/java/rife/bld/operations/DependencyTreeOperation.java b/src/main/java/rife/bld/operations/DependencyTreeOperation.java
index 00c961d..4a535cc 100644
--- a/src/main/java/rife/bld/operations/DependencyTreeOperation.java
+++ b/src/main/java/rife/bld/operations/DependencyTreeOperation.java
@@ -5,12 +5,14 @@
 package rife.bld.operations;
 
 import rife.bld.BaseProject;
+import rife.bld.BldCache;
 import rife.bld.BldVersion;
 import rife.bld.BuildExecutor;
 import rife.bld.dependencies.*;
 import rife.bld.wrapper.Wrapper;
 import rife.ioc.HierarchicalProperties;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -34,6 +36,7 @@ public class DependencyTreeOperation extends AbstractOperation<DependencyTreeOpe
     private final DependencyScopes extensionDependencies_ = new DependencyScopes();
 
     private final StringBuilder dependencyTree_ = new StringBuilder();
+    private File libBldDir_ = null;
 
     /**
      * Performs the dependency tree operation.
@@ -46,11 +49,97 @@ public class DependencyTreeOperation extends AbstractOperation<DependencyTreeOpe
             return;
         }
 
-        var extensions_tree = executeGenerateExtensionsDependencies();
-        var compile_tree = executeGenerateCompileDependencies();
-        var provided_tree = executeGenerateProvidedDependencies();
-        var runtime_tree = executeGenerateRuntimeDependencies();
-        var test_tree = executeGenerateTestDependencies();
+        // calculate the dependency tree of the extensions, using the cache if possible
+
+        String extensions_tree = null;
+        BldCache extensions_cache = null;
+        if (libBldDir_ != null) {
+            extensions_cache =  new BldCache(libBldDir_, new VersionResolution(extensionProperties()));
+            extensions_cache.fingerprintExtensions(
+                extensionRepositories().stream().map(Repository::toString).toList(),
+                extensionDependencies().scope(compile).stream().map(Dependency::toString).toList());
+            if (extensions_cache.isExtensionHashValid()) {
+                var cached_tree = extensions_cache.getCachedExtensionsDependencyTree();
+                if (cached_tree != null) {
+                    extensions_tree = cached_tree;
+                }
+            }
+        }
+
+        if (extensions_tree == null) {
+            extensions_tree = executeGenerateExtensionsDependencies();
+            if (extensions_cache != null) {
+                extensions_cache.cacheExtensionsDependencyTree(extensions_tree);
+                extensions_cache.writeCache();
+            }
+        }
+
+        // calculate the dependency tree of the dependencies, using the cache if possible
+
+        String compile_tree = null;
+        String provided_tree = null;
+        String runtime_tree = null;
+        String test_tree = null;
+        BldCache dependencies_cache = null;
+        if (libBldDir_ != null) {
+            dependencies_cache =  new BldCache(libBldDir_, new VersionResolution(properties()));
+            dependencies_cache.fingerprintDependencies(repositories(), dependencies());
+            if (dependencies_cache.isDependenciesHashValid()) {
+                var cached_compile_tree = dependencies_cache.getCachedDependenciesCompileDependencyTree();
+                if (cached_compile_tree != null) {
+                    compile_tree = cached_compile_tree;
+                }
+                var cached_provided_tree = dependencies_cache.getCachedDependenciesProvidedDependencyTree();
+                if (cached_provided_tree != null) {
+                    provided_tree = cached_provided_tree;
+                }
+                var cached_runtime_tree = dependencies_cache.getCachedDependenciesRuntimeDependencyTree();
+                if (cached_runtime_tree != null) {
+                    runtime_tree = cached_runtime_tree;
+                }
+                var cached_test_tree = dependencies_cache.getCachedDependenciesTestDependencyTree();
+                if (cached_test_tree != null) {
+                    test_tree = cached_test_tree;
+                }
+            }
+        }
+        
+        var write_dependencies_cache = false;
+        if (compile_tree == null) {
+            compile_tree = executeGenerateCompileDependencies();
+            if (dependencies_cache != null) {
+                dependencies_cache.cacheDependenciesCompileDependencyTree(compile_tree);
+                write_dependencies_cache = true;
+            }
+        }
+        if (provided_tree == null) {
+            provided_tree = executeGenerateProvidedDependencies();
+            if (dependencies_cache != null) {
+                dependencies_cache.cacheDependenciesProvidedDependencyTree(provided_tree);
+                write_dependencies_cache = true;
+            }
+        }
+        if (runtime_tree == null) {
+            runtime_tree = executeGenerateRuntimeDependencies();
+            if (dependencies_cache != null) {
+                dependencies_cache.cacheDependenciesRuntimeDependencyTree(runtime_tree);
+                write_dependencies_cache = true;
+            }
+        }
+        if (test_tree == null) {
+            test_tree = executeGenerateTestDependencies();
+            if (dependencies_cache != null) {
+                dependencies_cache.cacheDependenciesTestDependencyTree(test_tree);
+                write_dependencies_cache = true;
+            }
+        }
+
+        if (write_dependencies_cache) {
+            dependencies_cache.writeCache();
+        }
+
+        // output the dependency trees
+
         dependencyTree_.setLength(0);
         dependencyTree_.append(extensions_tree);
         dependencyTree_.append(System.lineSeparator());
@@ -143,6 +232,8 @@ public class DependencyTreeOperation extends AbstractOperation<DependencyTreeOpe
      * @since 1.5.21
      */
     public DependencyTreeOperation fromProject(BaseProject project) {
+        libBldDir_ = project.libBldDirectory();
+
         // add the repositories and dependencies from the extensions
         var wrapper = new Wrapper();
         wrapper.currentDir(project.workDirectory());
diff --git a/src/main/java/rife/bld/wrapper/WrapperExtensionResolver.java b/src/main/java/rife/bld/wrapper/WrapperExtensionResolver.java
index 817fa74..ccacc0e 100644
--- a/src/main/java/rife/bld/wrapper/WrapperExtensionResolver.java
+++ b/src/main/java/rife/bld/wrapper/WrapperExtensionResolver.java
@@ -64,14 +64,13 @@ public class WrapperExtensionResolver {
         downloadJavadoc_ = downloadJavadoc;
         cache_.fingerprintExtensions(
             repositories_.stream().map(Objects::toString).toList(),
-            dependencies_.stream().map(Objects::toString).toList(),
-            downloadSources, downloadJavadoc);
+            dependencies_.stream().map(Objects::toString).toList());
     }
 
     public void updateExtensions() {
         // verify and update the fingerprint hash file,
         // don't update the extensions if the hash is identical
-        if (cache_.isExtensionHashValid()) {
+        if (cache_.isExtensionsCacheValid(downloadSources_, downloadJavadoc_)) {
             return;
         }
 
@@ -81,6 +80,7 @@ public class WrapperExtensionResolver {
         // purge the files that are not part of the latest extensions anymore
         purgeExtensionDependencies(filenames);
 
+        cache_.cacheExtensionsDownloads(downloadSources_, downloadJavadoc_);
         cache_.writeCache(localArtifacts_);
 
         if (headerPrinted_) {