diff --git a/build-caching/pom.xml b/build-caching/pom.xml index c6beb5cb..24bc900d 100644 --- a/build-caching/pom.xml +++ b/build-caching/pom.xml @@ -37,6 +37,11 @@ 3.28.0-GA + + com.github.javaparser + javaparser-core + 3.24.4 + diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java b/build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java index 1f97298c..047ee681 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java @@ -1,6 +1,9 @@ package com.vertispan.j2cl.build; import com.vertispan.j2cl.build.impl.CollectedTaskInputs; +import com.vertispan.j2cl.build.incremental.BuildMap; +import com.vertispan.j2cl.build.incremental.BuildMapHashReader; +import com.vertispan.j2cl.build.incremental.JavaFileHashReader; import com.vertispan.j2cl.build.task.OutputTypes; import com.vertispan.j2cl.build.task.TaskFactory; import org.apache.commons.io.FileUtils; @@ -23,8 +26,6 @@ import java.util.stream.Stream; public class BuildService { - - private Map strippedSources = new HashMap<>(); private final TaskRegistry taskRegistry; private final TaskScheduler taskScheduler; private final DiskCache diskCache; @@ -35,6 +36,9 @@ public class BuildService { // hashes of each file in each project, updated under lock private final Map> currentProjectSourceHash = new HashMap<>(); + private final AtomicReference>> createdFilesRef = new AtomicReference<>(); + private final AtomicReference>> changedFilesRef = new AtomicReference<>(); + private final AtomicReference>> deletedFilesRef = new AtomicReference<>(); private BlockingBuildListener prevBuild; @@ -48,10 +52,6 @@ public BuildService(TaskRegistry taskRegistry, TaskScheduler taskScheduler, Disk this.incremental = incremental; } - public void addStrippedSourcesPath(Project project, Path path) { - this.strippedSources.put(project, path); - } - public Map getBuildMaps() { return buildMaps; } @@ -69,6 +69,10 @@ public DiskCache getDiskCache() { public void assignProject(Project project, String finalTask, PropertyTrackingConfig.ConfigValueProvider config) { // find the tasks and their upstream tasks collectTasksFromProject(finalTask, project, config, inputs); + // TODO this is a bit of a hack, but it's the only way to get the project's .class files + if(incremental) { + collectTasksFromProject(OutputTypes.BYTECODE, project, config, inputs); + } } private void collectTasksFromProject(String taskName, Project project, PropertyTrackingConfig.ConfigValueProvider config, Map collectedSoFar) { @@ -89,7 +93,7 @@ private void collectTasksFromProject(String taskName, Project project, PropertyT throw new NullPointerException("Missing task factory: " + taskName); } assert taskFactory.inputs.isEmpty(); - TaskFactory.Task task = taskFactory.resolve(project, propertyTrackingConfig, this); + TaskFactory.Task task = taskFactory.resolve(project, propertyTrackingConfig); collectedInputs.setTask(task); collectedInputs.setInputs(new ArrayList<>(taskFactory.inputs)); taskFactory.inputs.clear(); @@ -184,10 +188,6 @@ public synchronized void initialHashes() { }); } - AtomicReference>> createdFilesRef = new AtomicReference<>(); - AtomicReference>> changedFilesRef = new AtomicReference<>(); - AtomicReference>> deletedFilesRef = new AtomicReference<>(); - /** * Marks that a file has been created, deleted, or modified in the given project. */ @@ -238,7 +238,6 @@ public synchronized Cancelable requestBuild(BuildListener buildListener) throws prevBuild.blockUntilFinished(); } - if(incremental) { Map> createdFiles = createdFilesRef.getAndSet(new HashMap<>()); Map> changedFiles = changedFilesRef.getAndSet(new HashMap<>()); @@ -325,8 +324,20 @@ private BuildMap createBuildMap(Project project, Path dir, Map dirToProjectFiles.get(e.getAbsoluteParent().toString()) - .getUpdated().add(e.getSourcePath().toString())); + .getUpdated().put(e.getSourcePath().toString(), e.isPropagate())); } if (deletedFilesMap != null) { @@ -341,7 +352,7 @@ private BuildMap createBuildMap(Project project, Path dir, Map> curren // null check avoids re-entrance, as createBuildMap is recursive // and the same dep can be revisited. if (buildMaps.get(p) == null) { - Input input = new Input(p, OutputTypes.TRANSPILED_JS); + Input input = new Input(p, OutputTypes.BYTECODE); if (diskCache.lastSuccessfulTaskDir.containsKey(input)) { safeCreateBuildMap(p, diskCache.getLastSuccessfulDirectory(input), createdFiles, changedFiles, deletedFiles); @@ -430,10 +444,6 @@ public void copyAndDeleteFiles(Project project, String outputType, Path path) { String binaryTypeName = firstPart.replace('/', '.'); if (visited.add(binaryTypeName)) { - Path javaPath = path.resolve(firstPart + ".java"); - boolean b1 = Files.deleteIfExists(javaPath); - System.out.println("Delete: " + javaPath + ":" + b1); - if (outputType.equals(OutputTypes.STRIPPED_SOURCES)) { Path jsNativePath = path.resolve(firstPart + ".native.js"); Files.deleteIfExists(jsNativePath); diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedAcceptor.java b/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedAcceptor.java index 5824fb4b..c5fac4a8 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedAcceptor.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedAcceptor.java @@ -2,6 +2,7 @@ import java.util.function.Predicate; +import com.vertispan.j2cl.build.incremental.BuildMap; import com.vertispan.j2cl.build.task.CachedPath; import io.methvin.watcher.hashing.FileHash; diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java index d5dca26c..72b023fd 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java @@ -229,6 +229,8 @@ public static class CacheEntry implements Comparable, CachedPath { private long time; + private boolean propagate; + public CacheEntry(Path sourcePath, Path absoluteParent, FileHash hash) { if (sourcePath.isAbsolute()) { this.sourcePath = absoluteParent.relativize(sourcePath); @@ -306,6 +308,14 @@ public String toString() { ", hash=" + hash + '}'; } + + public boolean isPropagate() { + return propagate; + } + + public void setPropagate(boolean propagate) { + this.propagate = propagate; + } } /** @@ -583,4 +593,9 @@ public void markFailed(CacheResult failedResult) { } } + + public BuildService buildService() { + return buildService; + } + } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/InputSourceTaskFactory.java b/build-caching/src/main/java/com/vertispan/j2cl/build/InputSourceTaskFactory.java index 2c74ef17..0d3f528f 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/InputSourceTaskFactory.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/InputSourceTaskFactory.java @@ -25,7 +25,7 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService buildService) { + public Task resolve(Project project, Config config) { return (ignore) -> {}; } } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/ProjectFiles.java b/build-caching/src/main/java/com/vertispan/j2cl/build/ProjectFiles.java index 679f2c70..5ce47da5 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/ProjectFiles.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/ProjectFiles.java @@ -1,15 +1,12 @@ package com.vertispan.j2cl.build; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; public class ProjectFiles { private String dir; private List removed = new ArrayList<>(); // files - private Set updated = new HashSet<>(); // files + private Map updated = new HashMap<>(); // files private Set added = new HashSet<>(); // files private Set all; // files @@ -26,7 +23,7 @@ public List getRemoved() { return removed; } - public Set getUpdated() { + public Map getUpdated() { return updated; } @@ -41,7 +38,7 @@ public Set getAll() { @Override public String toString() { return "ProjectFiles{" + "removed=" + removed + - ", updated=" + updated + + ", updated=" + updated.keySet() + ", added=" + added + ", all=" + all + '}'; diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java index f80b9499..21402168 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java @@ -7,7 +7,6 @@ import com.vertispan.j2cl.build.task.TaskContext; import java.io.FileNotFoundException; -import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -211,7 +210,7 @@ private void executeTask(CollectedTaskInputs taskDetails, DiskCache.CacheResult } try { long start = System.currentTimeMillis(); - taskDetails.getTask().execute(new TaskContext(result.outputDir(), log)); + taskDetails.getTask().execute(new TaskContext(result.outputDir(), log, diskCache.buildService())); if (Thread.currentThread().isInterrupted()) { // Tried and failed to be canceled, so even though we were successful, some files might // have been deleted. Continue deleting contents @@ -349,7 +348,7 @@ private boolean executeFinalTask(CollectedTaskInputs taskDetails, DiskCache.Cach try { //TODO Make sure that we want to write this to _only_ the current log, and not also to any file //TODO Also be sure to write a prefix automatically - ((TaskFactory.FinalOutputTask) taskDetails.getTask()).finish(new TaskContext(cacheResult.outputDir(), buildLog)); + ((TaskFactory.FinalOutputTask) taskDetails.getTask()).finish(new TaskContext(cacheResult.outputDir(), buildLog, diskCache.buildService())); buildLog.info("Finished final task " + taskDetails.getDebugName() + " in " + (System.currentTimeMillis() - start) + "ms"); } catch (Throwable t) { buildLog.error("FAILED " + taskDetails.getDebugName() + " in " + (System.currentTimeMillis() - start) + "ms",t); diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/UnpackJarTaskFactory.java b/build-caching/src/main/java/com/vertispan/j2cl/build/UnpackJarTaskFactory.java index 37744511..aa07551d 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/UnpackJarTaskFactory.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/UnpackJarTaskFactory.java @@ -29,7 +29,7 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService buildService) { + public Task resolve(Project project, Config config) { // we don't have any proper inputs or configs // given the first (only) entry in the project's sources, unpack them diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/impl/CollectedTaskInputs.java b/build-caching/src/main/java/com/vertispan/j2cl/build/impl/CollectedTaskInputs.java index 4962be13..324b79e4 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/impl/CollectedTaskInputs.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/impl/CollectedTaskInputs.java @@ -52,7 +52,7 @@ public static CollectedTaskInputs jar(Project project) { CollectedTaskInputs t = new CollectedTaskInputs(project); t.setTaskFactory(new UnpackJarTaskFactory()); - t.setTask(t.getTaskFactory().resolve(project, null, null)); + t.setTask(t.getTaskFactory().resolve(project, null)); // create a fake input and give it a hash so that this unpack only runs if the jar changes Input jarInput = new Input(project, "jar"); try { diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/BuildMap.java b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/BuildMap.java similarity index 84% rename from build-caching/src/main/java/com/vertispan/j2cl/build/BuildMap.java rename to build-caching/src/main/java/com/vertispan/j2cl/build/incremental/BuildMap.java index b56dc48c..6f152636 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/BuildMap.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/BuildMap.java @@ -1,6 +1,8 @@ -package com.vertispan.j2cl.build; +package com.vertispan.j2cl.build.incremental; import com.google.common.collect.Streams; +import com.vertispan.j2cl.build.Project; +import com.vertispan.j2cl.build.ProjectFiles; import java.io.IOException; import java.nio.file.Files; @@ -24,10 +26,8 @@ public class BuildMap { private Map qualifiedSourceNameToPath = new HashMap<>(); private List childrenChangedFiles = new ArrayList<>(); - ; private Set changedFiles = new HashSet<>(); - ; private Set expandedFiles = new HashSet<>(); @@ -65,10 +65,9 @@ public void build(Path dir) { } private void populateFilesToDelete() { - // Merge all except added - which by it's nature has nothing needed deleting for (ProjectFiles p : dirToprojectFiles.values()) { filesToDelete.addAll(p.getRemoved()); - filesToDelete.addAll(p.getUpdated()); + filesToDelete.addAll(p.getUpdated().keySet()); } filesToDelete.addAll(getExpandedFiles()); } @@ -80,7 +79,7 @@ private void buildAndProcessChangedFiles(Path dir) { // Populate the complete list of potentially changed files for (ProjectFiles projectFiles : dirToprojectFiles.values()) { - changedFiles.addAll(projectFiles.getUpdated()); + changedFiles.addAll(projectFiles.getUpdated().keySet()); changedFiles.addAll(projectFiles.getAdded()); } changedFiles.addAll(expandedFiles); @@ -92,7 +91,11 @@ public List getFilesToDelete() { public void expandChangedFiles() { for (ProjectFiles projectFiles : dirToprojectFiles.values()) { - expandChangedFiles(projectFiles.getUpdated(), expandedFiles); + expandChangedFiles(projectFiles.getUpdated() + .entrySet() + .stream() + .filter(e -> e.getValue()).map(e -> e.getKey()) + .collect(Collectors.toSet()), expandedFiles); } expandChangedFiles(childrenChangedFiles, expandedFiles); } @@ -126,10 +129,7 @@ private void expandChangedFiles(TypeInfo typeInfo, Set changedFiles) { expandChangedFiles(dep.outgoing, changedFiles); } - // Now add all the dependencies - - for (TypeDependency dep : typeInfo.getMethodFieldIn()) { maybeAddNativeFile(dep.outgoing); changedFiles.add(qualifiedSourceNameToPath.get(dep.outgoing.getQualifiedSourceName())); @@ -216,13 +216,11 @@ private Map readBuildMapDescrForAllFiles(Path dir, Map typeInfoDescrs, Path dir) { - - if (javaFileName.endsWith(".java")) { String fileName = javaFileName.substring(0, javaFileName.lastIndexOf(".java")); String buildMapFileName = fileName + ".build.map"; - Path buildMapPath = dir.resolve(buildMapFileName); - + Path buildMapPath = dir.resolve("results") + .resolve(buildMapFileName); if (Files.notExists(buildMapPath)) { throw new RuntimeException("build.map files must exist for all changed .java files"); } @@ -250,7 +248,7 @@ TypeInfoDescr readBuildMapAsDescrs(Path buildMapPath, String javaFileName, Map innerTypes) { } } - public static class LineReader { - private int lineNbr; - List lines; - - public LineReader(List lines) { - this.lines = lines; - } - - String getAndInc() { - String line = lines.get(lineNbr++); - - - // skip any empty lines - while (lineNbr < lines.size() && - lines.get(lineNbr).trim().isEmpty()) { - lineNbr++; - } - - return line; - } - - String peekNext() { - return lines.get(lineNbr); - } - - boolean hasNext() { - return lineNbr < lines.size(); - } - - } - - static class TypeInfoDescr { - private String qualifiedSourceName; - private String superTypeName; // optional - private String qualifiedBinaryName; // optional if it's different to qualifiedSourceName, i.e. nested classes - private String nativePathName; // optional - private String enclosingType; - private List innerTypes; - private List interfaces; - private List dependencies; // dependencies that are not one of the above. - - public TypeInfoDescr(String qualifiedSourceName, String qualifiedBinaryName, String nativePathName) { - this.qualifiedSourceName = qualifiedSourceName; - this.qualifiedBinaryName = qualifiedBinaryName != null && !qualifiedBinaryName.trim().isEmpty() ? qualifiedBinaryName : qualifiedSourceName; - this.nativePathName = nativePathName; - - this.innerTypes = new ArrayList<>(); - this.interfaces = new ArrayList<>(); - this.dependencies = new ArrayList<>(); - } - - public List dependencies() { - return dependencies; - } - - @Override - public String toString() { - return "TypeInfoDescr{" + - "qualifiedSourceName='" + qualifiedSourceName + '\'' + - ", qualifiedBinaryName='" + qualifiedBinaryName + '\'' + - ", superTypeName='" + superTypeName + '\'' + - ", nativePathName='" + nativePathName + '\'' + - ", enclosingType='" + enclosingType + '\'' + - ", innerTypes=" + innerTypes + - ", interfaces=" + interfaces + - ", dependencies=" + dependencies + - '}'; - } - } - TypeInfoDescr readBuildMapSources(LineReader reader, Map typeInfoDescrs) { String[] segments = reader.getAndInc().split(":", -1); if (segments.length != 2) { @@ -460,6 +393,10 @@ void readBuildMapDependencies(LineReader reader, TypeInfoDescr typeInfoDescr) { } } + private void readBuildMapHash(LineReader reader, TypeInfoDescr typeInfoDescr) { + typeInfoDescr.hash = reader.getAndInc(); + } + private boolean isNotEmpty(int i, String[] x) { return x[i] != null && x[i].length() > 0; } @@ -491,7 +428,12 @@ public void cloneToTargetBuildMap(BuildMap target) { target.qualifiedSourceNameToPath.putAll(qualifiedSourceNameToPath); for (ProjectFiles projectFiles : dirToprojectFiles.values()) { - target.childrenChangedFiles.addAll(projectFiles.getUpdated()); + target.childrenChangedFiles.addAll(projectFiles.getUpdated() + .entrySet() + .stream() + .filter(e -> e.getValue()) + .map(e -> e.getKey()) + .collect(Collectors.toSet())); } target.childrenChangedFiles.addAll(expandedFiles); diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/BuildMapBuilder.java b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/BuildMapBuilder.java index e6e0b709..89b94b45 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/BuildMapBuilder.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/BuildMapBuilder.java @@ -15,8 +15,10 @@ */ package com.vertispan.j2cl.build.incremental; +import com.google.common.hash.HashCode; import javassist.CtClass; import javassist.CtMethod; +import javassist.Modifier; import javassist.NotFoundException; import javassist.bytecode.BadBytecode; import javassist.bytecode.SignatureAttribute; @@ -26,9 +28,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; +import java.util.*; +import java.util.function.BiPredicate; +import java.util.function.Predicate; +import java.util.stream.Collectors; /** * This is a clone of LibraryInfoBuilder, with all pruning removed. @@ -46,7 +49,9 @@ public BuildMapBuilder(Path outputPath) { this.outputPath = outputPath; } - public void addClass(CtClass clazz) throws NotFoundException { + public void addClass(CtClass clazz, Path javaFilePath) throws NotFoundException { + JavaFileHashReader javaFileHashReader = new JavaFileHashReader(javaFilePath); + StringBuffer sb = new StringBuffer("- sources"); sb.append(LINE_SEPARATOR); sb.append(clazz.getName()); @@ -57,7 +62,7 @@ public void addClass(CtClass clazz) throws NotFoundException { CtClass superClass = clazz.getSuperclass(); - if (!superClass.getName().equals("java.lang.Object")) { + if (superClass != null) { sb.append(superClass.getName()); sb.append(":"); sb.append(LINE_SEPARATOR); @@ -82,41 +87,20 @@ public void addClass(CtClass clazz) throws NotFoundException { sb.append("- dependencies"); sb.append(LINE_SEPARATOR); - Set types = new HashSet<>(); - clazz.getRefClasses() + .stream() + .filter(ref -> maybeAddType.test(clazz.getName(), ref)) .forEach( - c -> maybeAddType(c, types)); - - Arrays.stream(clazz.getFields()) - .forEach( - field -> { - try { - if(field.getGenericSignature() != null) { - SignatureAttribute.Type typeSignature = SignatureAttribute.toTypeSignature(field.getGenericSignature()); - if(typeSignature instanceof javassist.bytecode.SignatureAttribute.ClassType) { - javassist.bytecode.SignatureAttribute.ClassType classType = - (javassist.bytecode.SignatureAttribute.ClassType) typeSignature; - for (SignatureAttribute.TypeArgument typeArgument : classType.getTypeArguments()) { - maybeAddType(typeArgument.toString(), types); - } - } - } - maybeAddType(field.getType().getName(), types); - } catch (NotFoundException e) { - throw new RuntimeException(e); - } catch (BadBytecode e) { - throw new RuntimeException(e); - } + ref -> { + sb.append(ref); + sb.append(LINE_SEPARATOR); }); - types.forEach(type -> { - if (!clazz.getName().equals(type)) { - sb.append(type); - sb.append(LINE_SEPARATOR); - } - }); + sb.append("- hash"); + sb.append(LINE_SEPARATOR); + sb.append(javaFileHashReader.hash()); + sb.append(LINE_SEPARATOR); String pkg = clazz.getPackageName().replaceAll("\\.", System.getProperty("file.separator")); File output = new File(outputPath.toFile(), pkg + "/" + clazz.getSimpleName() + ".build.map"); @@ -131,9 +115,6 @@ public void addClass(CtClass clazz) throws NotFoundException { } - void maybeAddType(String clazz, Set types) { - if(!(clazz.startsWith("java."))) { - types.add(clazz); - } - } + private BiPredicate maybeAddType = (clazz, ref) -> !(ref.startsWith("java.") || clazz.equals(ref)); + } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/BuildMapHashReader.java b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/BuildMapHashReader.java new file mode 100644 index 00000000..98cb69f5 --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/BuildMapHashReader.java @@ -0,0 +1,33 @@ +package com.vertispan.j2cl.build.incremental; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +public class BuildMapHashReader { + + private final Path path; + + public BuildMapHashReader(Path buildMapPath) { + this.path = buildMapPath; + } + + public String hash() { + List lines; + try { + lines = Files.readAllLines(path); + } catch (IOException e) { + throw new RuntimeException(e); + } + + for (LineReader reader = new LineReader(lines); reader.hasNext(); ) { + String line = reader.getAndInc(); + if(line.startsWith("- hash")) { + return reader.getAndInc(); + } + } + + throw new RuntimeException("Unable to find hash in " + path); + } +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/HashBuilder.java b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/HashBuilder.java new file mode 100644 index 00000000..7b06a40e --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/HashBuilder.java @@ -0,0 +1,23 @@ +package com.vertispan.j2cl.build.incremental; + +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +public class HashBuilder { + + private Set hashes = new TreeSet<>(); + + public void addField(String name, String type) { + hashes.add(Objects.hash(type, name)); + } + + public void addMethod(String name, String type, String params) { + hashes.add(Objects.hash(type, name, params)); + } + + @Override + public String toString() { + return Objects.hash(hashes.toArray(new Integer[hashes.size()])) + ""; + } +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/JavaFileHashReader.java b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/JavaFileHashReader.java new file mode 100644 index 00000000..4b9c48ba --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/JavaFileHashReader.java @@ -0,0 +1,48 @@ +package com.vertispan.j2cl.build.incremental; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.Modifier; +import com.github.javaparser.ast.body.FieldDeclaration; +import com.github.javaparser.ast.body.MethodDeclaration; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Collectors; + +public class JavaFileHashReader { + + private final Path path; + + public JavaFileHashReader(Path path) { + this.path = path; + } + + public String hash() { + try(InputStream in = Files.newInputStream(path)) { + CompilationUnit cu = new JavaParser().parse(in).getResult().get(); + HashBuilder builder = new HashBuilder(); + cu.findAll(FieldDeclaration.class).stream() + .filter(field -> !field.getModifiers().contains(Modifier.privateModifier())) + .forEach(field -> field.getVariables().forEach(variable -> { + builder.addField(variable.getName().toString(), variable.getType().toString()); + })); + + cu.findAll(MethodDeclaration.class).stream() + .filter(method -> !method.getModifiers().contains(Modifier.privateModifier())) + .forEach(method -> { + String params = method.getParameters().stream() + .map(param -> param.getType().toString() + " " + param.getName()) + .collect(Collectors.joining(", ")); + builder.addMethod(method.getName().toString(), method.getType().toString(), params); + }); + + return builder.toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/LibraryInfoBuilder.java b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/LibraryInfoBuilder.java deleted file mode 100644 index fb905f13..00000000 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/LibraryInfoBuilder.java +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Copyright 2018 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.vertispan.j2cl.build.incremental; - -import com.google.common.base.Predicates; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -/*import com.google.j2cl.common.Problems; -import com.google.j2cl.common.Problems.FatalError; -import com.google.j2cl.common.SourcePosition; -import com.google.j2cl.transpiler.ast.AbstractVisitor; -import com.google.j2cl.transpiler.ast.DeclaredTypeDescriptor; -import com.google.j2cl.transpiler.ast.FieldAccess; -import com.google.j2cl.transpiler.ast.FieldDescriptor; -import com.google.j2cl.transpiler.ast.Invocation; -import com.google.j2cl.transpiler.ast.JavaScriptConstructorReference; -import com.google.j2cl.transpiler.ast.Member; -import com.google.j2cl.transpiler.ast.MemberDescriptor; -import com.google.j2cl.transpiler.ast.MethodDescriptor; -import com.google.j2cl.transpiler.ast.Type; -import com.google.j2cl.transpiler.ast.TypeDescriptors; -import com.google.protobuf.util.JsonFormat;*/ - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -/** Traverse types and gather execution flow information for building call graph. */ -public final class LibraryInfoBuilder { - - /* public static final int NULL_TYPE = 0; - private final LibraryInfo.Builder libraryInfo = LibraryInfo.newBuilder(); - private final Map types = new HashMap<>(); - - public void addType( - Type type, - String headerFilePath, - String implFilePath, - Map outputSourceInfoByMember) { - - if (!isPrunableType(type.getTypeDescriptor())) { - return; - } - - TypeInfo.Builder typeInfoBuilder = - TypeInfo.newBuilder() - .setTypeId(getTypeId(type.getTypeDescriptor())) - .setHeaderSourceFilePath(headerFilePath) - .setImplSourceFilePath(implFilePath) - .setJstypeInterface(type.isInterface() && type.getTypeDescriptor().isJsType()); - - DeclaredTypeDescriptor superTypeDescriptor = type.getSuperTypeDescriptor(); - if (superTypeDescriptor != null - && !superTypeDescriptor.isNative() - && !TypeDescriptors.isJavaLangObject(superTypeDescriptor)) { - typeInfoBuilder.setExtendsType(getTypeId(superTypeDescriptor)); - } - - for (DeclaredTypeDescriptor superInterfaceType : type.getSuperInterfaceTypeDescriptors()) { - if (!superInterfaceType.isNative() && !superInterfaceType.isJsFunctionInterface()) { - typeInfoBuilder.addImplementsType(getTypeId(superInterfaceType)); - } - } - - // Collect references to getter and setter for the same field under the same key to - // create only one MemberInfo instance that combines all the references appearing in their - // bodies (see #getMemberId). - Map memberInfoBuilders = - Maps.newLinkedHashMapWithExpectedSize(type.getMembers().size()); - - boolean hasConstantEntryPoint = false; - for (Member member : type.getMembers()) { - MemberDescriptor memberDescriptor = member.getDescriptor(); - - if (memberDescriptor.hasJsNamespace()) { - // Members with an explicit namespace members don't really belong to the type. Skip them - // here, otherwise they would be an entry point for this type, and the type might be - // unnecessarily retained by rta. - continue; - } - - if (memberDescriptor.getOrigin().isInstanceOfSupportMember()) { - // InstanceOf support members should not be considered methods that are prunable if there - // are no references, since the references are hidden by the runtime. In the end - // InstanceOf support members are live whenever the type is live. - continue; - } - - if (memberDescriptor.isField() && !mayTriggerClinit(memberDescriptor)) { - if (memberDescriptor.isCompileTimeConstant() && isJsAccessible(memberDescriptor)) { - hasConstantEntryPoint = true; - } - // We don't need to record fields, there is not much value on pruning them. However there is - // slight complication for fields that may trigger clinit which may (or may not) generated - // as getter, so we need to record their usage (hence their data here as well). - continue; - } - - MemberInfo.Builder builder = - memberInfoBuilders.computeIfAbsent( - getMemberId(memberDescriptor), - m -> createMemberInfo(memberDescriptor, outputSourceInfoByMember)); - - collectReferencedTypesAndMethodInvocations(member, builder); - } - - if (hasConstantEntryPoint) { - // Ensure the type is not pruned by RTA. - memberInfoBuilders.put( - "$js_entry$", - MemberInfo.newBuilder().setName("$js_entry$").setStatic(true).setJsAccessible(true)); - } - - libraryInfo.addType( - typeInfoBuilder.addAllMember( - memberInfoBuilders.values().stream() - .map(MemberInfo.Builder::build) - .collect(Collectors.toList()))); - } - - private static MemberInfo.Builder createMemberInfo( - MemberDescriptor memberDescriptor, - Map outputSourceInfoByMember) { - MemberInfo.Builder memberInfoBuilder = - MemberInfo.newBuilder() - .setName(getMemberId(memberDescriptor)) - .setStatic(memberDescriptor.isStatic()) - .setJsAccessible(isJsAccessible(memberDescriptor)); - - // See the limitations of member removal in b/177365417. - SourcePosition position = outputSourceInfoByMember.get(memberDescriptor); - if (position != null) { - memberInfoBuilder.setPosition( - com.google.j2cl.transpiler.backend.libraryinfo.SourcePosition.newBuilder() - .setStart(position.getStartFilePosition().getLine()) - // For the minifier, end position is exclusive. - .setEnd(position.getEndFilePosition().getLine() + 1) - .build()); - } - - return memberInfoBuilder; - } - - private void collectReferencedTypesAndMethodInvocations( - Member member, MemberInfo.Builder memberInfoBuilder) { - Set invokedMethods = - new LinkedHashSet<>(memberInfoBuilder.getInvokedMethodsList()); - - // The set of types that are explicitly referenced in this member; these come from - // JavaScriptConstructorReferences that appear in the AST from type literals, casts, - // instanceofs and ALSO also the qualifier in every static member reference. - // References to static members already include the enclosing class, so in order to avoid - // redundancy in library info these types are tracked separately and removed. - Set explicitlyReferencedTypes = - new LinkedHashSet<>(memberInfoBuilder.getReferencedTypesList()); - - // The set of types that are implicitly referenced in this member; these come from static - // Invocations in the AST, e.g. the enclosing class of a static method call. These types will be - // preserved by RTA when seeing the reference to the static member so there is no need to - // separately record them as referenced types. - Set typesReferencedViaStaticMemberReferences = new HashSet<>(); - - member.accept( - new AbstractVisitor() { - @Override - public void exitJavaScriptConstructorReference(JavaScriptConstructorReference node) { - DeclaredTypeDescriptor referencedType = - node.getReferencedTypeDeclaration().toRawTypeDescriptor(); - - if (!isPrunableType(referencedType)) { - return; - } - - if (isJsAccessible(referencedType)) { - return; - } - - // No need to record references to parent or itself since they will be live regardless. - if (member.getDescriptor().getEnclosingTypeDescriptor().isSubtypeOf(referencedType)) { - return; - } - - // In Javascript a Class is statically referenced by using it's constructor function. - explicitlyReferencedTypes.add(getTypeId(referencedType)); - } - - @Override - public void exitFieldAccess(FieldAccess node) { - FieldDescriptor target = node.getTarget(); - - if (!isPrunableType(target.getEnclosingTypeDescriptor())) { - return; - } - - if (isJsAccessible(target)) { - // We don't record access to js accessible fields since they are never pruned. - return; - } - - if (!mayTriggerClinit(target)) { - return; - } - - // Register static FieldAccess as getter/setter invocations. We are conservative here - // because getter and setter functions have the same name: i.e. the name of the field. - // If a field is accessed, we visit both getter and setter. - addInvokedMethod(target, node.getQualifier() != null ? (DeclaredTypeDescriptor) node.getQualifier().getTypeDescriptor() : null); - } - - @Override - public void exitInvocation(Invocation node) { - MethodDescriptor target = node.getTarget(); - - if (!isPrunableType(target.getEnclosingTypeDescriptor())) { - return; - } - - if (isJsAccessible(target)) { - // We don't record call to js accessible methods since they are never pruned. - return; - } - - // Only record a $clinit call if it is from a clinit itself. All other clinit calls are - // from the entry points of the class and doesn't need recording since RapidTypeAnalyser - // will make the clinit alive when it arrives to an entry point. - if (target.getName().equals("$clinit") - && !member.getDescriptor().getName().equals("$clinit")) { - return; - } - - // TODO(b/34928687): Remove after $loadmodule moved the AST. - if (target.getName().equals("$loadModules")) { - return; - } - - addInvokedMethod(target, node.getQualifier() != null ? (DeclaredTypeDescriptor) node.getQualifier().getTypeDescriptor() : null); - } - - private void addInvokedMethod(MemberDescriptor target, DeclaredTypeDescriptor nodeEnclosingType) { - invokedMethods.add(createMethodInvocation(target, nodeEnclosingType)); - if (!target.isInstanceMember()) { - typesReferencedViaStaticMemberReferences.add( - getTypeId(target.getEnclosingTypeDescriptor())); - } - } - }); - - memberInfoBuilder - .clearReferencedTypes() - .clearInvokedMethods() - .addAllInvokedMethods(invokedMethods) - // Record only the explicit type references without the implicit ones which are redundant. - .addAllReferencedTypes( - Sets.difference(explicitlyReferencedTypes, typesReferencedViaStaticMemberReferences)); - } - - private MethodInvocation createMethodInvocation(MemberDescriptor memberDescriptor, DeclaredTypeDescriptor nodeEnclosingType) { - return MethodInvocation.newBuilder() - .setMethod(getMemberId(memberDescriptor)) - .setTargetEnclosingType(getTypeId(memberDescriptor.getEnclosingTypeDescriptor())) - .setNodeEnclosingType((nodeEnclosingType != null) ? getTypeId(nodeEnclosingType) : -1) - .build(); - } - - private int getTypeId(DeclaredTypeDescriptor typeDescriptor) { - // Note that the IDs start from '1' to reserve '0' for NULL_TYPE. - return types.computeIfAbsent(typeDescriptor.getQualifiedJsName(), x -> types.size() + 1); - } - - private LibraryInfo build() { - libraryInfo.clearTypeMap(); - String[] typeMap = new String[types.size() + 1]; - typeMap[NULL_TYPE] = ""; - types.forEach((name, i) -> typeMap[i] = name); - libraryInfo.addAllTypeMap(Arrays.asList(typeMap)); - return libraryInfo.build(); - } - - *//** Serialize a LibraryInfo object into a JSON string. *//* - public String toJson(Problems problems) { - try { - return JsonFormat.printer().print(build()); - } catch (IOException e) { - problems.fatal(FatalError.CANNOT_WRITE_FILE, e.toString()); - return null; - } - } - - public byte[] toByteArray() { - return build().toByteArray(); - } - - private static String getMemberId(MemberDescriptor memberDescriptor) { - // TODO(b/158014657): remove this once the bug is fixed. - String mangledName = - isPropertyAccessor(memberDescriptor) - ? memberDescriptor.computePropertyMangledName() - : memberDescriptor.getMangledName(); - - // Avoid unintented collissions by using the seperate namespace for static and non-static. - return memberDescriptor.isInstanceMember() ? mangledName + "_$i" : mangledName; - } - - private static boolean mayTriggerClinit(MemberDescriptor memberDescriptor) { - return memberDescriptor.isStatic() && !memberDescriptor.isCompileTimeConstant(); - } - - private static boolean isPropertyAccessor(MemberDescriptor memberDescriptor) { - if (!memberDescriptor.isMethod()) { - return false; - } - MethodDescriptor methodDescriptor = (MethodDescriptor) memberDescriptor; - return methodDescriptor.isPropertyGetter() || methodDescriptor.isPropertySetter(); - } - - private static boolean isPrunableType(DeclaredTypeDescriptor typeDescriptor) { - return !typeDescriptor.isNative() && !typeDescriptor.isJsEnum(); - } - - private static boolean isJsAccessible(MemberDescriptor memberDescriptor) { - // Note that we are not collecting bootstrap classes in references so we need to force them in. - // There are 2 reasons for that: - // - We are inconsistent in marking their references wrt. being native or not. - // - They are frequently used and never pruned; recording them is wasteful. - return isInBootstrap(memberDescriptor.getEnclosingTypeDescriptor()) - || memberDescriptor.canBeReferencedExternally(); - } - - private static boolean isJsAccessible(DeclaredTypeDescriptor typeDescriptor) { - return isInBootstrap(typeDescriptor) - || typeDescriptor.getDeclaredMemberDescriptors().stream() - .filter(Predicates.not(MemberDescriptor::isInstanceMember)) - .anyMatch(MemberDescriptor::isJsMember); - } - - private static boolean isInBootstrap(DeclaredTypeDescriptor typeDescriptor) { - return TypeDescriptors.isBootstrapNamespace(typeDescriptor) - || isAccesssedFromJ2clBootstrapJsFiles(typeDescriptor); - } - - // There are references to non JsMember members of these types from JavaScript in the J2CL - // runtime. For now we will consider and all their members accessible by JavaScript code. - // TODO(b/29509857): remove once the refactoring of the code in nativebootstrap and vmbootstrap - // is completed and the references removed. - private static final ImmutableSet TYPES_ACCESSED_FROM_J2CL_BOOTSTRAP_JS = - ImmutableSet.of( - "javaemul.internal.InternalPreconditions", - "java.lang.Object", - "java.lang.Double", - "java.lang.Boolean", - "java.lang.Number", - "java.lang.CharSequence", - "java.lang.String", - "java.lang.Class", - "java.lang.Comparable", - "java.lang.Integer"); - - private static boolean isAccesssedFromJ2clBootstrapJsFiles( - DeclaredTypeDescriptor typeDescriptor) { - return TYPES_ACCESSED_FROM_J2CL_BOOTSTRAP_JS.contains(typeDescriptor.getQualifiedSourceName()); - }*/ -} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/LineReader.java b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/LineReader.java new file mode 100644 index 00000000..571e8287 --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/LineReader.java @@ -0,0 +1,34 @@ +package com.vertispan.j2cl.build.incremental; + +import java.util.List; + +public class LineReader { + int lineNbr; + List lines; + + public LineReader(List lines) { + this.lines = lines; + } + + String getAndInc() { + String line = lines.get(lineNbr++); + + + // skip any empty lines + while (lineNbr < lines.size() && + lines.get(lineNbr).trim().isEmpty()) { + lineNbr++; + } + + return line; + } + + String peekNext() { + return lines.get(lineNbr); + } + + boolean hasNext() { + return lineNbr < lines.size(); + } + +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/TypeDependency.java b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/TypeDependency.java similarity index 95% rename from build-caching/src/main/java/com/vertispan/j2cl/build/TypeDependency.java rename to build-caching/src/main/java/com/vertispan/j2cl/build/incremental/TypeDependency.java index 5a252a10..a218079d 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/TypeDependency.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/TypeDependency.java @@ -1,4 +1,4 @@ -package com.vertispan.j2cl.build; +package com.vertispan.j2cl.build.incremental; public class TypeDependency { TypeInfo incoming; diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/TypeInfo.java b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/TypeInfo.java similarity index 99% rename from build-caching/src/main/java/com/vertispan/j2cl/build/TypeInfo.java rename to build-caching/src/main/java/com/vertispan/j2cl/build/incremental/TypeInfo.java index bc77d34b..73ff0c97 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/TypeInfo.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/TypeInfo.java @@ -1,4 +1,4 @@ -package com.vertispan.j2cl.build; +package com.vertispan.j2cl.build.incremental; import java.util.ArrayList; import java.util.List; diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/TypeInfoDescr.java b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/TypeInfoDescr.java new file mode 100644 index 00000000..d2157e37 --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/TypeInfoDescr.java @@ -0,0 +1,46 @@ +package com.vertispan.j2cl.build.incremental; + +import java.util.ArrayList; +import java.util.List; + +public class TypeInfoDescr { + String qualifiedSourceName; + String superTypeName; // optional + String qualifiedBinaryName; // optional if it's different to qualifiedSourceName, i.e. nested classes + String nativePathName; // optional + String enclosingType; + List innerTypes; + List interfaces; + List dependencies; // dependencies that are not one of the above. + + String hash; + + public TypeInfoDescr(String qualifiedSourceName, String qualifiedBinaryName, String nativePathName) { + this.qualifiedSourceName = qualifiedSourceName; + this.qualifiedBinaryName = qualifiedBinaryName != null && !qualifiedBinaryName.trim().isEmpty() ? qualifiedBinaryName : qualifiedSourceName; + this.nativePathName = nativePathName; + + this.innerTypes = new ArrayList<>(); + this.interfaces = new ArrayList<>(); + this.dependencies = new ArrayList<>(); + } + + public List dependencies() { + return dependencies; + } + + @Override + public String toString() { + return "TypeInfoDescr{" + + "qualifiedSourceName='" + qualifiedSourceName + '\'' + + ", qualifiedBinaryName='" + qualifiedBinaryName + '\'' + + ", superTypeName='" + superTypeName + '\'' + + ", nativePathName='" + nativePathName + '\'' + + ", enclosingType='" + enclosingType + '\'' + + ", innerTypes=" + innerTypes + + ", interfaces=" + interfaces + + ", dependencies=" + dependencies + + ", hash=" + hash + + '}'; + } +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/library_info.proto b/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/library_info.proto deleted file mode 100644 index 1a6133b0..00000000 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/incremental/library_info.proto +++ /dev/null @@ -1,41 +0,0 @@ -syntax = "proto3"; - -package j2cl; - -option java_multiple_files = true; -option java_package = "com.google.j2cl.transpiler.backend.libraryinfo"; - -message LibraryInfo { - repeated string type_map = 1; - repeated TypeInfo type = 2; -} - -message TypeInfo { - int32 type_id = 1; - int32 extends_type = 2; - repeated int32 implements_type = 3; - repeated MemberInfo member = 4; - string header_source_file_path = 5; - string impl_source_file_path = 6; - bool jstype_interface = 7; -} - -message MemberInfo { - string name = 1; - bool static = 3; - bool js_accessible = 4; - repeated MethodInvocation invoked_methods = 5; - repeated int32 referenced_types = 6; - SourcePosition position = 7; -} - -message MethodInvocation { - string method = 1; - int32 target_enclosing_type = 2; - int32 node_enclosing_type = 3; -} - -message SourcePosition { - int32 start = 1; - int32 end = 2; -} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskContext.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskContext.java index 59575933..2b918667 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskContext.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskContext.java @@ -1,14 +1,19 @@ package com.vertispan.j2cl.build.task; +import com.vertispan.j2cl.build.BuildService; + import java.nio.file.Path; public class TaskContext implements BuildLog { private final Path path; private final BuildLog log; - public TaskContext(Path path, BuildLog log) { + private final BuildService buildService; + + public TaskContext(Path path, BuildLog log, BuildService buildService) { this.path = path; this.log = log; + this.buildService = buildService; } public Path outputPath() { @@ -58,4 +63,8 @@ public void error(String msg, Throwable t) { public void error(Throwable t) { log.error(t); } + + public BuildService getBuildService() { + return buildService; + } } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskFactory.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskFactory.java index bfb46e82..93f0ef5a 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskFactory.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskFactory.java @@ -112,6 +112,6 @@ public interface FinalOutputTask extends Task { * @return a task that will be executed each time the given project * needs to be built, which should use created inputs and configs */ - public abstract Task resolve(Project project, Config config, BuildService buildService); + public abstract Task resolve(Project project, Config config); } diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java index 190b91d2..2680673d 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java @@ -3,7 +3,6 @@ import com.google.auto.service.AutoService; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.vertispan.j2cl.build.BuildService; import com.vertispan.j2cl.build.task.*; import com.vertispan.j2cl.tools.Closure; import org.apache.commons.io.FileUtils; @@ -45,7 +44,7 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService buildService) { + public Task resolve(Project project, Config config) { List jsSources = Stream .concat( scope(project.getDependencies(), Dependency.Scope.RUNTIME) diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BytecodeTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BytecodeTask.java index c39703ae..3100fed8 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BytecodeTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BytecodeTask.java @@ -3,16 +3,23 @@ import com.google.auto.service.AutoService; import com.google.j2cl.common.SourceUtils; import com.vertispan.j2cl.build.BuildService; +import com.vertispan.j2cl.build.DiskCache; +import com.vertispan.j2cl.build.incremental.BuildMapBuilder; import com.vertispan.j2cl.build.task.*; import com.vertispan.j2cl.tools.Javac; +import javassist.ClassPool; +import javassist.NotFoundException; import javax.annotation.Nullable; import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.PathMatcher; +import java.nio.file.Paths; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -46,7 +53,7 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService service) { + public Task resolve(Project project, Config config) { boolean incremental = config.getIncremental(); if (!project.hasSourcesMapped()) { @@ -86,15 +93,6 @@ public Task resolve(Project project, Config config, BuildService service) { extraClasspath.stream() ).collect(Collectors.toList()); - if (incremental) { - Path bytecodePath = service.getDiskCache().getLastSuccessfulDirectory(new com.vertispan.j2cl.build.Input((com.vertispan.j2cl.build.Project) project, - OutputTypes.BYTECODE)); - - if (bytecodePath != null) { - classpathDirs.add(bytecodePath.resolve("results").toFile()); - } - } - List sourcePaths = inputDirs.getParentPaths().stream().map(Path::toFile).collect(Collectors.toList()); File generatedClassesDir = getGeneratedClassesDir(context); File classOutputDir = context.outputPath().toFile(); @@ -115,6 +113,41 @@ public Task resolve(Project project, Config config, BuildService service) { exception.printStackTrace(); throw exception; } + + if (incremental) { + + Path bytecodePath = context.getBuildService().getDiskCache().getLastSuccessfulDirectory(new com.vertispan.j2cl.build.Input((com.vertispan.j2cl.build.Project) project, + OutputTypes.BYTECODE)); + + BuildMapBuilder builder = new BuildMapBuilder(context.outputPath()); + + ClassPool pool = new ClassPool(null); + pool.appendSystemPath(); + pool.appendClassPath(classOutputDir.getAbsolutePath()); + + Files.walk(classOutputDir.toPath()) + .filter(Files::isRegularFile) + .filter(JAVA_BYTECODE::matches) + .forEach(p -> { + String className = classOutputDir.toPath() + .relativize(p) + .toString() + .replace(".class", "") + .replace("/", "."); + try { + String javaFile = classOutputDir.toPath().relativize(p).toString().replaceAll("\\.class", ".java"); + Path file = inputSources.getFilesAndHashes() + .stream() + .filter(f -> f.getSourcePath().toString().equals(javaFile)) + .findFirst() + .orElseThrow(() -> new RuntimeException("No source file found for " + javaFile)) + .getAbsolutePath(); + builder.addClass(pool.get(className), file); + } catch (NotFoundException e) { + throw new RuntimeException(e); + } + }); + } } // Copy all resources, even .java files, so that this output is the source of truth as if this diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java index ab49199a..2058f93f 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java @@ -48,7 +48,7 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService buildService) { + public Task resolve(Project project, Config config) { final List js; if (project.isJsZip()) { js = Collections.singletonList(input(project, OutputTypes.BYTECODE).filter(ClosureTask.PLAIN_JS_SOURCES)); diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureTask.java index 39db65d1..11daad44 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureTask.java @@ -4,7 +4,6 @@ import com.google.javascript.jscomp.CompilationLevel; import com.google.javascript.jscomp.CompilerOptions; import com.google.javascript.jscomp.DependencyOptions; -import com.vertispan.j2cl.build.BuildService; import com.vertispan.j2cl.build.task.*; import com.vertispan.j2cl.tools.Closure; import org.apache.commons.io.FileUtils; @@ -134,7 +133,7 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService service) { + public Task resolve(Project project, Config config) { // collect current project JS sources and runtime deps JS sources // TODO filter to just JS and sourcemaps? probably not required unless we also get sources // from the actual input source instead of copying it along each step diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/IJarTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/IJarTask.java index 244565d7..e7b42f89 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/IJarTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/IJarTask.java @@ -1,7 +1,6 @@ package com.vertispan.j2cl.build.provided; import com.google.auto.service.AutoService; -import com.vertispan.j2cl.build.BuildService; import com.vertispan.j2cl.build.task.*; import java.nio.file.Files; @@ -30,7 +29,7 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService buildService) { + public Task resolve(Project project, Config config) { Input myStrippedBytecode = input(project, OutputTypes.STRIPPED_BYTECODE); return context -> { diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/IJsTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/IJsTask.java index 14154ae2..315ea7c2 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/IJsTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/IJsTask.java @@ -1,7 +1,6 @@ package com.vertispan.j2cl.build.provided; import com.google.auto.service.AutoService; -import com.vertispan.j2cl.build.BuildService; import com.vertispan.j2cl.build.task.*; /** @@ -27,7 +26,7 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService buildService) { + public Task resolve(Project project, Config config) { Input js = input(project, OutputTypes.TRANSPILED_JS); return context -> { if (js.getFilesAndHashes().isEmpty()) { diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java index bc83a669..e522e47e 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java @@ -43,7 +43,7 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService buildService) { + public Task resolve(Project project, Config config) { boolean incremental = config.getIncremental(); // J2CL is only interested in .java and .native.js files in our own sources Input ownJavaSources = input(project, OutputTypes.STRIPPED_SOURCES).filter(JAVA_SOURCES, NATIVE_JS_SOURCES); @@ -62,7 +62,7 @@ public Task resolve(Project project, Config config, BuildService buildService) { return (context) -> { List javaFiles = ownJavaSources.getFilesAndHashes() .stream() - .filter( new ChangedAcceptor((com.vertispan.j2cl.build.Project) project, buildService)) + .filter( new ChangedAcceptor((com.vertispan.j2cl.build.Project) project, context.getBuildService())) .collect(Collectors.toList()); if (javaFiles.isEmpty()) { @@ -74,8 +74,8 @@ public Task resolve(Project project, Config config, BuildService buildService) { ) .collect(Collectors.toList()); - //ownJavaSources only one file if (incremental) { + BuildService buildService = context.getBuildService(); Path bytecodePath = buildService.getDiskCache().getLastSuccessfulDirectory(new com.vertispan.j2cl.build.Input((com.vertispan.j2cl.build.Project) project, OutputTypes.BYTECODE)); @@ -101,10 +101,14 @@ public Task resolve(Project project, Config config, BuildService buildService) { .map(p -> SourceUtils.FileInfo.create(p.getAbsolutePath().toString(), p.getSourcePath().toString())) .collect(Collectors.toList()); + javaSources.forEach(f -> { + context.debug("Compiling " + f.sourcePath()); + }); + List nativeSources = ownNativeJsSources.stream().flatMap(i -> i.getFilesAndHashes() .stream()) - .filter( new ChangedAcceptor((com.vertispan.j2cl.build.Project) project, buildService)) + .filter( new ChangedAcceptor((com.vertispan.j2cl.build.Project) project, context.getBuildService())) .filter(e -> NATIVE_JS_SOURCES.matches(e.getSourcePath())) .map(p -> SourceUtils.FileInfo.create(p.getAbsolutePath().toString(), p.getSourcePath().toString())) .collect(Collectors.toList()); diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/JavacTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/JavacTask.java index e89a3e35..45149208 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/JavacTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/JavacTask.java @@ -37,7 +37,7 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService buildService) { + public Task resolve(Project project, Config config) { boolean incremental = config.getIncremental(); // emits only stripped bytecode, so we're not worried about anything other than .java files to compile and .class on the classpath Input ownSources = input(project, OutputTypes.STRIPPED_SOURCES).filter(JAVA_SOURCES); @@ -54,7 +54,7 @@ public Task resolve(Project project, Config config, BuildService buildService) { return (context) -> { List files = ownSources.getFilesAndHashes() .stream() - .filter( new ChangedAcceptor((com.vertispan.j2cl.build.Project) project, buildService)).collect(Collectors.toList()); + .filter( new ChangedAcceptor((com.vertispan.j2cl.build.Project) project, context.getBuildService())).collect(Collectors.toList()); if (files.isEmpty()) { return;// no work to do @@ -65,7 +65,7 @@ public Task resolve(Project project, Config config, BuildService buildService) { List sourcePaths = ownSources.getParentPaths().stream().map(Path::toFile).collect(Collectors.toList()); if (incremental) { - Path bytecodePath = buildService.getDiskCache().getLastSuccessfulDirectory(new com.vertispan.j2cl.build.Input((com.vertispan.j2cl.build.Project) project, + Path bytecodePath = context.getBuildService().getDiskCache().getLastSuccessfulDirectory(new com.vertispan.j2cl.build.Input((com.vertispan.j2cl.build.Project) project, OutputTypes.STRIPPED_BYTECODE)); if (bytecodePath != null) { diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java index 66fc812d..036b4b24 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java @@ -2,7 +2,6 @@ import com.google.auto.service.AutoService; import com.google.j2cl.common.SourceUtils; -import com.vertispan.j2cl.build.BuildService; import com.vertispan.j2cl.build.task.*; import com.vertispan.j2cl.tools.GwtIncompatiblePreprocessor; @@ -30,7 +29,7 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService buildService) { + public Task resolve(Project project, Config config) { Input inputSources = input(project, OutputTypes.BYTECODE).filter(JAVA_SOURCES); return context -> { diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/TestCollectionTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/TestCollectionTask.java index b573fcaa..f1c43cb6 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/TestCollectionTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/TestCollectionTask.java @@ -1,7 +1,6 @@ package com.vertispan.j2cl.build.provided; import com.google.auto.service.AutoService; -import com.vertispan.j2cl.build.BuildService; import com.vertispan.j2cl.build.task.*; import org.apache.commons.io.FileUtils; @@ -35,7 +34,7 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService buildService) { + public Task resolve(Project project, Config config) { // gather possible inputs so we can get the test summary file // we assume here that the user will correctly depend on the junit apt, might revise this later Input apt = input(project, OutputTypes.BYTECODE).filter(TEST_SUMMARY_JSON, TEST_SUITE); diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/TurbineTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/TurbineTask.java index fb2344f2..24ac553f 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/TurbineTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/TurbineTask.java @@ -9,6 +9,7 @@ import com.vertispan.j2cl.build.BuildService; import com.vertispan.j2cl.build.incremental.BuildMapBuilder; import com.vertispan.j2cl.build.task.*; +import javassist.ClassPath; import javassist.ClassPool; import javassist.CtClass; import javassist.NotFoundException; @@ -47,10 +48,10 @@ public String getVersion() { } @Override - public Task resolve(Project project, Config config, BuildService buildService) { + public Task resolve(Project project, Config config) { int version = SourceVersion.latestSupported().ordinal(); if(version == 8) { - return super.resolve(project, config, buildService); + return super.resolve(project, config); } // emits only stripped bytecode, so we're not worried about anything other than .java files to compile and .class on the classpath @@ -58,7 +59,6 @@ public Task resolve(Project project, Config config, BuildService buildService) { List extraClasspath = config.getExtraClasspath(); - boolean incremental = config.getIncremental(); List compileClasspath = scope(project.getDependencies(), Dependency.Scope.COMPILE).stream() .map(p -> input(p, OutputTypes.STRIPPED_BYTECODE_HEADERS)) .map(input -> input.filter(JAVA_BYTECODE)) @@ -97,35 +97,6 @@ public Task resolve(Project project, Config config, BuildService buildService) { // usually it means, it's an apt that can't be processed, log it context.info(e.getMessage()); } - - if(incremental) { - - buildService.addStrippedSourcesPath((com.vertispan.j2cl.build.Project) project, context.outputPath()); - - BuildMapBuilder builder = new BuildMapBuilder(context.outputPath()); - String jarFileName = output.toString(); - - JarFile jar = new JarFile(output); - ClassPool pool = ClassPool.getDefault(); - try{ - pool.insertClassPath(jarFileName); - - jar.stream().map(JarEntry::getName).forEach(name -> { - if(name.endsWith(".class") && !name.startsWith("META-INF")) { - String className = name.replace(".class", "").replace("/", "."); - try { - CtClass clazz = pool.get(className); - builder.addClass(clazz); - } catch (NotFoundException e) { - throw new RuntimeException(e); - } - } - }); - - } catch (NotFoundException e) { - context.info(e.getMessage()); - } - } }; } diff --git a/j2cl-tools/src/main/java/com/vertispan/j2cl/build/provided/BytecodeTask.java b/j2cl-tools/src/main/java/com/vertispan/j2cl/build/provided/BytecodeTask.java deleted file mode 100644 index fc1eaa71..00000000 --- a/j2cl-tools/src/main/java/com/vertispan/j2cl/build/provided/BytecodeTask.java +++ /dev/null @@ -1,129 +0,0 @@ -package com.vertispan.j2cl.build.provided; - -import com.google.auto.service.AutoService; -import com.google.j2cl.common.SourceUtils; -import com.vertispan.j2cl.build.BuildService; -import com.vertispan.j2cl.build.ChangedAcceptor; -import com.vertispan.j2cl.build.DiskCache; -import com.vertispan.j2cl.build.task.*; -import com.vertispan.j2cl.tools.Javac; - -import java.io.File; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.PathMatcher; -import java.util.Collection; -import java.util.List; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * This implementation and {@link AptTask} are wired together, if you replace - * one you may need to replace the other at the same time (the SkipAptTask is an - * exception to this). - * - * The assumption is that since these are generated at the same time by a single - * invocation of javac, we want to generate the bytecode first for downstream - * projects so they can also generate their own sources. With this though, - * the AptTask should be a no-op, so it shouldn't really matter. - */ -@AutoService(TaskFactory.class) -public class BytecodeTask extends TaskFactory { - - public static final PathMatcher JAVA_SOURCES = FileSystems.getDefault().getPathMatcher("glob:**/*.java"); - public static final PathMatcher JAVA_BYTECODE = FileSystems.getDefault().getPathMatcher("glob:**/*.class"); - - @Override - public String getOutputType() { - return OutputTypes.BYTECODE; - } - - @Override - public String getTaskName() { - return "default"; - } - - @Override - public String getVersion() { - return "0"; - } - - @Override - public Task resolve(Project project, Config config, BuildService buildService) { - boolean incremental = true; - if (!project.hasSourcesMapped()) { - // instead copy the bytecode out of the jar so it can be used by downtream bytecode/apt tasks - Input existingUnpackedBytecode = input(project, OutputTypes.INPUT_SOURCES, buildService);//.filter(JAVA_BYTECODE); - return (output) -> { - for (CachedPath entry : existingUnpackedBytecode.getFilesAndHashes()) { - Files.createDirectories(output.path().resolve(entry.getSourcePath()).getParent()); - Files.copy(entry.getAbsolutePath(), output.path().resolve(entry.getSourcePath())); - } - }; - } - - // TODO just use one input for both of these - // track the dirs (with all file changes) so that APT can see things it wants - Input inputDirs = input(project, OutputTypes.INPUT_SOURCES, buildService); - // track just java files (so we can just compile them) - Input inputSources = input(project, OutputTypes.INPUT_SOURCES, buildService).filter(JAVA_SOURCES); - - List bytecodeClasspath = scope(project.getDependencies(), com.vertispan.j2cl.build.task.Dependency.Scope.COMPILE) - .stream() - .map(inputs(OutputTypes.BYTECODE, buildService)) - .collect(Collectors.toList()); - - File bootstrapClasspath = config.getBootstrapClasspath(); - List extraClasspath = config.getExtraClasspath(); - return (output) -> { - List files = inputSources.getFilesAndHashes() - .stream() - .filter( new ChangedAcceptor((com.vertispan.j2cl.build.Project) project, buildService)).collect(Collectors.toList()); - - if (files.isEmpty()) { - return;// no work to do - } - - bytecodeClasspath.stream().map(Input::getParentPaths).flatMap(Collection::stream).map(Path::toFile).forEach( - f -> System.out.println("bytecodepath: " + f.getAbsolutePath().toString())); - - List classpathDirs = Stream.of( - bytecodeClasspath.stream().map(Input::getParentPaths).flatMap(Collection::stream).map(Path::toFile), - extraClasspath.stream() //, -// inputDirs.getParentPaths().stream().map(Path::toFile) - ) - .flatMap(Function.identity()) - .collect(Collectors.toList()); - - if (incremental) { - Path bytecodePath = buildService.getDiskCache().getLastSuccessfulDirectory(new com.vertispan.j2cl.build.Input((com.vertispan.j2cl.build.Project) project, - OutputTypes.BYTECODE)); - - if (bytecodePath != null) { - classpathDirs.add(bytecodePath.resolve("results").toFile()); - } - } - - // TODO don't dump APT to the same dir? - Javac javac = new Javac(output.path().toFile(), classpathDirs, output.path().toFile(), bootstrapClasspath); - - // TODO convention for mapping to original file paths, provide FileInfo out of Inputs instead of Paths, - // automatically relativized? - List sources = files.stream() - .map(p -> SourceUtils.FileInfo.create(p.getAbsolutePath().toString(), p.getSourcePath().toString())) - .collect(Collectors.toList()); - - try { - if (!javac.compile(sources)) { - throw new RuntimeException("Failed to complete bytecode task, check log"); - } - } catch (Exception exception) { - exception.printStackTrace(); - throw exception; - } - - }; - } -}