hash based deps processing

build-caching/pom.xml

@@ -37,6 +37,11 @@
             <version>3.28.0-GA</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.github.javaparser</groupId>
+            <artifactId>javaparser-core</artifactId>
+            <version>3.24.4</version>
+        </dependency>
 
         <!-- watch service -->
         <dependency>

build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java

@@ -1,6 +1,9 @@
 package com.vertispan.j2cl.build;
 
 import com.vertispan.j2cl.build.impl.CollectedTaskInputs;
+import com.vertispan.j2cl.build.incremental.BuildMap;
+import com.vertispan.j2cl.build.incremental.BuildMapHashReader;
+import com.vertispan.j2cl.build.incremental.JavaFileHashReader;
 import com.vertispan.j2cl.build.task.OutputTypes;
 import com.vertispan.j2cl.build.task.TaskFactory;
 import org.apache.commons.io.FileUtils;
@@ -23,8 +26,6 @@
 import java.util.stream.Stream;
 
 public class BuildService {
-
-    private Map<Project, Path> strippedSources = new HashMap<>();
     private final TaskRegistry taskRegistry;
     private final TaskScheduler taskScheduler;
     private final DiskCache diskCache;
@@ -35,6 +36,9 @@ public class BuildService {
 
     // hashes of each file in each project, updated under lock
     private final Map<Project, Map<Path, DiskCache.CacheEntry>> currentProjectSourceHash = new HashMap<>();
+    private final AtomicReference<Map<Project, Map<Path, DiskCache.CacheEntry>>> createdFilesRef = new AtomicReference<>();
+    private final AtomicReference<Map<Project, Map<Path, DiskCache.CacheEntry>>> changedFilesRef = new AtomicReference<>();
+    private final AtomicReference<Map<Project, Map<Path, DiskCache.CacheEntry>>> deletedFilesRef = new AtomicReference<>();
 
     private BlockingBuildListener prevBuild;
 
@@ -48,10 +52,6 @@ public BuildService(TaskRegistry taskRegistry, TaskScheduler taskScheduler, Disk
         this.incremental = incremental;
     }
 
-    public void addStrippedSourcesPath(Project project, Path path) {
-        this.strippedSources.put(project, path);
-    }
-
     public Map<Project, BuildMap> getBuildMaps() {
         return buildMaps;
     }
@@ -69,6 +69,10 @@ public DiskCache getDiskCache() {
     public void assignProject(Project project, String finalTask, PropertyTrackingConfig.ConfigValueProvider config) {
         // find the tasks and their upstream tasks
         collectTasksFromProject(finalTask, project, config, inputs);
+        // TODO this is a bit of a hack, but it's the only way to get the project's .class files
+        if(incremental) {
+            collectTasksFromProject(OutputTypes.BYTECODE, project, config, inputs);
+        }
     }
 
     private void collectTasksFromProject(String taskName, Project project, PropertyTrackingConfig.ConfigValueProvider config, Map<Input, CollectedTaskInputs> collectedSoFar) {
@@ -89,7 +93,7 @@ private void collectTasksFromProject(String taskName, Project project, PropertyT
                 throw new NullPointerException("Missing task factory: " + taskName);
             }
             assert taskFactory.inputs.isEmpty();
-            TaskFactory.Task task = taskFactory.resolve(project, propertyTrackingConfig, this);
+            TaskFactory.Task task = taskFactory.resolve(project, propertyTrackingConfig);
             collectedInputs.setTask(task);
             collectedInputs.setInputs(new ArrayList<>(taskFactory.inputs));
             taskFactory.inputs.clear();
@@ -184,10 +188,6 @@ public synchronized void initialHashes() {
                 });
     }
 
-    AtomicReference<Map<Project, Map<Path, DiskCache.CacheEntry>>> createdFilesRef = new AtomicReference<>();
-    AtomicReference<Map<Project, Map<Path, DiskCache.CacheEntry>>> changedFilesRef = new AtomicReference<>();
-    AtomicReference<Map<Project, Map<Path, DiskCache.CacheEntry>>> deletedFilesRef = new AtomicReference<>();
-
     /**
      * Marks that a file has been created, deleted, or modified in the given project.
      */
@@ -238,7 +238,6 @@ public synchronized Cancelable requestBuild(BuildListener buildListener) throws
             prevBuild.blockUntilFinished();
         }
 
-
         if(incremental) {
             Map<Project, Map<Path, DiskCache.CacheEntry>> createdFiles = createdFilesRef.getAndSet(new HashMap<>());
             Map<Project, Map<Path, DiskCache.CacheEntry>> changedFiles = changedFilesRef.getAndSet(new HashMap<>());
@@ -325,8 +324,20 @@ private BuildMap createBuildMap(Project project, Path dir, Map<Project, Map<Path
         }
 
         if (changedFilesMap != null) {
+            for (DiskCache.CacheEntry value : changedFilesMap.values()) {
+                if (value.getSourcePath().toString().endsWith(".java")) {
+                    String fileName = value.getSourcePath().toString()
+                            .substring(0, value.getSourcePath().toString().lastIndexOf(".java"));
+                    String buildMapFileName = fileName + ".build.map";
+                    Path buildMapFile = dir.resolve("results").resolve(buildMapFileName);
+                    String hash = new BuildMapHashReader(buildMapFile).hash();
+                    boolean propagate = new JavaFileHashReader(value.getAbsolutePath()).hash().equals(hash);
+                    //read hash from .java and compare to propagate deps
+                    value.setPropagate(!propagate);
+                }
+            }
             changedFilesMap.values().forEach(e -> dirToProjectFiles.get(e.getAbsoluteParent().toString())
-                    .getUpdated().add(e.getSourcePath().toString()));
+                    .getUpdated().put(e.getSourcePath().toString(), e.isPropagate()));
         }
 
         if (deletedFilesMap != null) {
@@ -341,7 +352,7 @@ private BuildMap createBuildMap(Project project, Path dir, Map<Project, Map<Path
             // we don't know what this module will link to, so all classes need to be cloned.
             for (com.vertispan.j2cl.build.task.Dependency dep : project.getDependencies()) {
                 BuildMap depBuildMap = buildMaps.get(dep.getProject());
-                Input input = new Input((Project) dep.getProject(), OutputTypes.TRANSPILED_JS);
+                Input input = new Input((Project) dep.getProject(), OutputTypes.BYTECODE);
                 Path path = diskCache.getLastSuccessfulDirectory(input);
                 if (depBuildMap == null && path != null) {
                     depBuildMap = safeCreateBuildMap((Project) dep.getProject(), path,
@@ -353,7 +364,10 @@ private BuildMap createBuildMap(Project project, Path dir, Map<Project, Map<Path
                     depBuildMap.cloneToTargetBuildMap(buildMap);
                 }
             }
-            buildMap.build(strippedSources.get(project));
+
+            Input input = new Input(project, OutputTypes.BYTECODE);
+            Path path = diskCache.getLastSuccessfulDirectory(input);
+            buildMap.build(path);
 
         } catch (Exception e) {
             e.printStackTrace();
@@ -401,7 +415,7 @@ private void buildRequested(Map<Project, Map<Path, DiskCache.CacheEntry>> curren
                 // null check avoids re-entrance, as createBuildMap is recursive
                 // and the same dep can be revisited.
                 if (buildMaps.get(p) == null) {
-                    Input input = new Input(p, OutputTypes.TRANSPILED_JS);
+                    Input input = new Input(p, OutputTypes.BYTECODE);
                     if (diskCache.lastSuccessfulTaskDir.containsKey(input)) {
                         safeCreateBuildMap(p, diskCache.getLastSuccessfulDirectory(input),
                                 createdFiles, changedFiles, deletedFiles);
@@ -430,10 +444,6 @@ public void copyAndDeleteFiles(Project project, String outputType, Path path) {
                             String binaryTypeName = firstPart.replace('/', '.');
 
                             if (visited.add(binaryTypeName)) {
-                                Path javaPath = path.resolve(firstPart + ".java");
-                                boolean b1 = Files.deleteIfExists(javaPath);
-                                System.out.println("Delete: " + javaPath + ":" + b1);
-
                                 if (outputType.equals(OutputTypes.STRIPPED_SOURCES)) {
                                     Path jsNativePath = path.resolve(firstPart + ".native.js");
                                     Files.deleteIfExists(jsNativePath);

build-caching/src/main/java/com/vertispan/j2cl/build/ChangedAcceptor.java

@@ -2,6 +2,7 @@
 
 import java.util.function.Predicate;
 
+import com.vertispan.j2cl.build.incremental.BuildMap;
 import com.vertispan.j2cl.build.task.CachedPath;
 import io.methvin.watcher.hashing.FileHash;
 

build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java

@@ -229,6 +229,8 @@ public static class CacheEntry implements Comparable<CacheEntry>, CachedPath {
 
         private long time;
 
+        private boolean propagate;
+
         public CacheEntry(Path sourcePath, Path absoluteParent, FileHash hash) {
             if (sourcePath.isAbsolute()) {
                 this.sourcePath = absoluteParent.relativize(sourcePath);
@@ -306,6 +308,14 @@ public String toString() {
                     ", hash=" + hash +
                     '}';
         }
+
+        public boolean isPropagate() {
+            return propagate;
+        }
+
+        public void setPropagate(boolean propagate) {
+            this.propagate = propagate;
+        }
     }
 
     /**
@@ -583,4 +593,9 @@ public void markFailed(CacheResult failedResult) {
         }
     }
 
+
+    public BuildService buildService() {
+        return buildService;
+    }
+
 }

build-caching/src/main/java/com/vertispan/j2cl/build/InputSourceTaskFactory.java

@@ -25,7 +25,7 @@ public String getVersion() {
     }
 
     @Override
-    public Task resolve(Project project, Config config, BuildService buildService) {
+    public Task resolve(Project project, Config config) {
         return (ignore) -> {};
     }
 }

build-caching/src/main/java/com/vertispan/j2cl/build/ProjectFiles.java

@@ -1,15 +1,12 @@
 package com.vertispan.j2cl.build;
 
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 
 public class ProjectFiles {
     private String dir;
 
     private List<String>        removed        = new ArrayList<>(); // files
-    private Set<String>         updated        = new HashSet<>(); // files
+    private Map<String, Boolean>         updated        = new HashMap<>(); // files
     private Set<String>         added          = new HashSet<>(); // files
     private Set<String>         all; // files
 
@@ -26,7 +23,7 @@ public List<String> getRemoved() {
         return removed;
     }
 
-    public Set<String> getUpdated() {
+    public Map<String, Boolean> getUpdated() {
         return updated;
     }
 
@@ -41,7 +38,7 @@ public Set<String> getAll() {
     @Override public String toString() {
         return "ProjectFiles{" +
                "removed=" + removed +
-               ", updated=" + updated +
+               ", updated=" + updated.keySet() +
                ", added=" + added +
                ", all=" + all +
                '}';

build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java

@@ -7,7 +7,6 @@
 import com.vertispan.j2cl.build.task.TaskContext;
 
 import java.io.FileNotFoundException;
-import java.nio.file.Path;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -211,7 +210,7 @@ private void executeTask(CollectedTaskInputs taskDetails, DiskCache.CacheResult
                     }
                     try {
                         long start = System.currentTimeMillis();
-                        taskDetails.getTask().execute(new TaskContext(result.outputDir(), log));
+                        taskDetails.getTask().execute(new TaskContext(result.outputDir(), log, diskCache.buildService()));
                         if (Thread.currentThread().isInterrupted()) {
                             // Tried and failed to be canceled, so even though we were successful, some files might
                             // have been deleted. Continue deleting contents
@@ -349,7 +348,7 @@ private boolean executeFinalTask(CollectedTaskInputs taskDetails, DiskCache.Cach
                     try {
                         //TODO Make sure that we want to write this to _only_ the current log, and not also to any file
                         //TODO Also be sure to write a prefix automatically
-                        ((TaskFactory.FinalOutputTask) taskDetails.getTask()).finish(new TaskContext(cacheResult.outputDir(), buildLog));
+                        ((TaskFactory.FinalOutputTask) taskDetails.getTask()).finish(new TaskContext(cacheResult.outputDir(), buildLog, diskCache.buildService()));
                         buildLog.info("Finished final task " + taskDetails.getDebugName() + " in " + (System.currentTimeMillis() - start) + "ms");
                     } catch (Throwable t) {
                         buildLog.error("FAILED   " + taskDetails.getDebugName() + " in " + (System.currentTimeMillis() - start) + "ms",t);

build-caching/src/main/java/com/vertispan/j2cl/build/UnpackJarTaskFactory.java

@@ -29,7 +29,7 @@ public String getVersion() {
     }
 
     @Override
-    public Task resolve(Project project, Config config, BuildService buildService) {
+    public Task resolve(Project project, Config config) {
         // we don't have any proper inputs or configs
 
         // given the first (only) entry in the project's sources, unpack them

build-caching/src/main/java/com/vertispan/j2cl/build/impl/CollectedTaskInputs.java

@@ -52,7 +52,7 @@ public static CollectedTaskInputs jar(Project project) {
 
         CollectedTaskInputs t = new CollectedTaskInputs(project);
         t.setTaskFactory(new UnpackJarTaskFactory());
-        t.setTask(t.getTaskFactory().resolve(project, null, null));
+        t.setTask(t.getTaskFactory().resolve(project, null));
         // create a fake input and give it a hash so that this unpack only runs if the jar changes
         Input jarInput = new Input(project, "jar");
         try {