Incremental draft

build-caching/pom.xml

@@ -16,11 +16,38 @@
 
 
     <dependencies>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.jspecify</groupId>
+            <artifactId>jspecify</artifactId>
+            <version>0.2.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.javassist</groupId>
+            <artifactId>javassist</artifactId>
+            <version>3.28.0-GA</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.github.javaparser</groupId>
+            <artifactId>javaparser-core</artifactId>
+            <version>3.24.4</version>
+        </dependency>
+
         <!-- watch service -->
         <dependency>
             <groupId>io.methvin</groupId>
             <artifactId>directory-watcher</artifactId>
-            <version>0.15.0</version>
+            <version>0.15.1</version>
         </dependency>
 
         <!-- Test -->

build-caching/src/main/java/com/vertispan/j2cl/build/ChangedAcceptor.java

@@ -0,0 +1,36 @@
+package com.vertispan.j2cl.build;
+
+import java.util.function.Predicate;
+
+import com.vertispan.j2cl.build.incremental.BuildMap;
+import com.vertispan.j2cl.build.task.CachedPath;
+import io.methvin.watcher.hashing.FileHash;
+
+public class ChangedAcceptor implements Predicate<CachedPath> {
+    private BuildMap buildMap;
+
+    public ChangedAcceptor(Project project, BuildService buildService) {
+
+
+        buildMap = buildService != null ? buildService.getBuildMaps().get(project) : null;
+    }
+
+    @Override public boolean test(CachedPath cachedPath) {
+        boolean found = true;
+        if (buildMap != null) {
+            String tested = cachedPath.getSourcePath().toString();
+            if(tested.endsWith(".native.js")) {
+                String candidate = tested.replace(".native.js",".java");
+                if(buildMap.getChangedFiles().contains(candidate)) {
+                    return true;
+                }
+            }
+
+            found = cachedPath.getHash().equals(FileHash.DIRECTORY) ||
+                    buildMap.getChangedFiles().contains(cachedPath.getSourcePath().toString());
+        }
+
+        return found;
+    }
+
+}

build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java

@@ -2,15 +2,19 @@
 
 import com.vertispan.j2cl.build.impl.CollectedTaskInputs;
 import com.vertispan.j2cl.build.task.CachedPath;
+import com.vertispan.j2cl.build.task.OutputTypes;
 import io.methvin.watcher.PathUtils;
 import io.methvin.watcher.hashing.FileHash;
 import io.methvin.watcher.hashing.FileHasher;
 import io.methvin.watchservice.MacOSXListeningWatchService;
 import io.methvin.watchservice.WatchablePath;
+import org.apache.commons.io.FileUtils;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.io.Writer;
+import java.nio.charset.Charset;
 import java.nio.file.WatchService;
 import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
@@ -54,8 +58,8 @@ public TaskOutput output() {
             return taskOutput;
         }
 
-        public void markSuccess() {
-            markFinished(this);
+        public void markSuccess(Input input) {
+            markFinished(input, this);
             runningTasks.remove(taskDir);
         }
         public void markFailure() {
@@ -79,6 +83,11 @@ public void cancel() {
 
     protected final File cacheDir;
     private final Executor executor;
+    protected BuildService buildService;
+
+    public void setBuildService(BuildService buildService) {
+        this.buildService = buildService;
+    }
     /**
      * A single watch service to monitor all changes to the cache dir, under the assumption that
      * the entire cache directory is on a single filesystem.
@@ -90,6 +99,7 @@ public void cancel() {
     private final Thread watchThread = new Thread(this::checkForWork, "DiskCacheThread");
     private Map<Path, TaskOutput> knownOutputs = new ConcurrentHashMap<>();
     private Map<Input, TaskOutput> lastSuccessfulOutputs = new ConcurrentHashMap<>();
+    protected Map<Input, Path> lastSuccessfulTaskDir = new ConcurrentHashMap<>();
 
     private final Map<Path, Path> knownMarkers = new ConcurrentHashMap<>();
     private final Map<Path, Set<PendingCacheResult>> taskFutures = new ConcurrentHashMap<>();
@@ -115,6 +125,10 @@ public DiskCache(File cacheDir, Executor executor) throws IOException {
         livenessThread.start();
     }
 
+    public Path getLastSuccessfulDirectory(Input input) {
+        return this.lastSuccessfulTaskDir.get(input);
+    }
+
     private void checkForWork() {
         try {
             WatchKey key;
@@ -213,6 +227,10 @@ public static class CacheEntry implements Comparable<CacheEntry>, CachedPath {
         /** Hash of the file, so we can notice changes, or hash the tree.  */
         private final FileHash hash;
 
+        private long time;
+
+        private boolean propagate;
+
         public CacheEntry(Path sourcePath, Path absoluteParent, FileHash hash) {
             if (sourcePath.isAbsolute()) {
                 this.sourcePath = absoluteParent.relativize(sourcePath);
@@ -221,8 +239,18 @@ public CacheEntry(Path sourcePath, Path absoluteParent, FileHash hash) {
             }
             this.absoluteParent = absoluteParent;
             this.hash = hash;
+            try {
+                Path path  = absoluteParent.resolve(sourcePath);
+                if ( Files.exists(path)) {
+                    time = Files.getLastModifiedTime(path).toMillis();
+                }
+            } catch (IOException e) {
+                throw new RuntimeException("Unable to read Timestamp: for " + this + "\n"  + e);
+            }
         }
 
+
+
         @Override
         public Path getSourcePath() {
             return sourcePath;
@@ -242,6 +270,11 @@ public FileHash getHash() {
             return hash;
         }
 
+        @Override
+        public long getLastModifiedTime() {
+            return time;
+        }
+
         @Override
         public int compareTo(CacheEntry cacheEntry) {
             return sourcePath.compareTo(cacheEntry.sourcePath);
@@ -275,6 +308,14 @@ public String toString() {
                     ", hash=" + hash +
                     '}';
         }
+
+        public boolean isPropagate() {
+            return propagate;
+        }
+
+        public void setPropagate(boolean propagate) {
+            this.propagate = propagate;
+        }
     }
 
     /**
@@ -449,6 +490,7 @@ public void waitForTask(CollectedTaskInputs taskDetails, Listener listener) {
                 // caller can begin work right away
                 Files.createDirectory(outputDir);
                 Files.createFile(logFile(taskDir));
+                Files.createDirectory(taskDir.resolve("status"));
                 cancelable.ready();
                 return;
             }
@@ -526,9 +568,14 @@ private WatchKey registerWatchCreate(Path taskDir) throws IOException {
         return watchable.register(this.service, StandardWatchEventKinds.ENTRY_CREATE);
     }
 
-    public void markFinished(CacheResult successfulResult) {
+    public void markFinished(Input input, CacheResult successfulResult) {
         try {
-            this.knownOutputs.put(successfulResult.taskDir, makeOutput(successfulResult.taskDir));
+            TaskOutput output = makeOutput(successfulResult.taskDir);
+            this.lastSuccessfulTaskDir.put(input, successfulResult.taskDir);
+            this.knownOutputs.put(successfulResult.taskDir, output);
+            if (input.getOutputType().equals(OutputTypes.TRANSPILED_JS)) {
+                buildService.writeFilesDat(input.getProject());
+            }
             Files.createFile(successMarker(successfulResult.taskDir));
         } catch (IOException ioException) {
             //TODO need to basically stop everything if we can't write files to cache
@@ -546,4 +593,9 @@ public void markFailed(CacheResult failedResult) {
         }
     }
 
+
+    public BuildService buildService() {
+        return buildService;
+    }
+
 }

build-caching/src/main/java/com/vertispan/j2cl/build/InputSourceTaskFactory.java

@@ -26,6 +26,6 @@ public String getVersion() {
 
     @Override
     public Task resolve(Project project, Config config) {
-        return ignore -> {};
+        return (ignore) -> {};
     }
 }

build-caching/src/main/java/com/vertispan/j2cl/build/ProjectFiles.java

@@ -0,0 +1,46 @@
+package com.vertispan.j2cl.build;
+
+import java.util.*;
+
+public class ProjectFiles {
+    private String dir;
+
+    private List<String>        removed        = new ArrayList<>(); // files
+    private Map<String, Boolean>         updated        = new HashMap<>(); // files
+    private Set<String>         added          = new HashSet<>(); // files
+    private Set<String>         all; // files
+
+    public ProjectFiles(String dir, Set<String> all) {
+        this.dir = dir;
+        this.all = all;
+    }
+
+    public String getDir() {
+        return dir;
+    }
+
+    public List<String> getRemoved() {
+        return removed;
+    }
+
+    public Map<String, Boolean> getUpdated() {
+        return updated;
+    }
+
+    public Set<String> getAdded() {
+        return added;
+    }
+
+    public Set<String> getAll() {
+        return all;
+    }
+
+    @Override public String toString() {
+        return "ProjectFiles{" +
+               "removed=" + removed +
+               ", updated=" + updated.keySet() +
+               ", added=" + added +
+               ", all=" + all +
+               '}';
+    }
+}

build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java

@@ -214,4 +214,9 @@ public Path getWebappDirectory() {
         }
         return Paths.get(s);
     }
+
+    @Override
+    public boolean getIncremental() {
+        return Boolean.parseBoolean(getString("incremental"));
+    }
 }

build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java

@@ -7,7 +7,6 @@
 import com.vertispan.j2cl.build.task.TaskContext;
 
 import java.io.FileNotFoundException;
-import java.nio.file.Path;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -211,7 +210,7 @@ private void executeTask(CollectedTaskInputs taskDetails, DiskCache.CacheResult
                     }
                     try {
                         long start = System.currentTimeMillis();
-                        taskDetails.getTask().execute(new TaskContext(result.outputDir(), log));
+                        taskDetails.getTask().execute(new TaskContext(result.outputDir(), log, diskCache.buildService()));
                         if (Thread.currentThread().isInterrupted()) {
                             // Tried and failed to be canceled, so even though we were successful, some files might
                             // have been deleted. Continue deleting contents
@@ -222,7 +221,7 @@ private void executeTask(CollectedTaskInputs taskDetails, DiskCache.CacheResult
                         if (elapsedMillis > 5) {
                             buildLog.info("Finished " + taskDetails.getDebugName() + " in " + elapsedMillis + "ms");
                         }
-                        result.markSuccess();
+                        result.markSuccess(taskDetails.getAsInput());
 
                     } catch (Throwable exception) {
                         if (Thread.currentThread().isInterrupted()) {
@@ -349,7 +348,7 @@ private boolean executeFinalTask(CollectedTaskInputs taskDetails, DiskCache.Cach
                     try {
                         //TODO Make sure that we want to write this to _only_ the current log, and not also to any file
                         //TODO Also be sure to write a prefix automatically
-                        ((TaskFactory.FinalOutputTask) taskDetails.getTask()).finish(new TaskContext(cacheResult.outputDir(), buildLog));
+                        ((TaskFactory.FinalOutputTask) taskDetails.getTask()).finish(new TaskContext(cacheResult.outputDir(), buildLog, diskCache.buildService()));
                         buildLog.info("Finished final task " + taskDetails.getDebugName() + " in " + (System.currentTimeMillis() - start) + "ms");
                     } catch (Throwable t) {
                         buildLog.error("FAILED   " + taskDetails.getDebugName() + " in " + (System.currentTimeMillis() - start) + "ms",t);