Rough draft of bundling sourcemaps and original sources

This will cost extra CPU time, but save on IO time and disk space, which
is likely a net win for most builds. This is a first draft, which
appears to work as expected (sourcemaps function correctly, vastly
smaller output in BUNDLE_JAR), but doesn't yet fully reduce the number
of copies we can make, and doesn't avoid copying sourcemap bundles (in
cases where js bundles are not copied). There may also be caching
issues, where sourcemaps are incorrect not generated or are regenerated.

Author: niloc132

j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java

@@ -5,13 +5,10 @@
 import com.google.gson.GsonBuilder;
 import com.google.javascript.jscomp.deps.ClosureBundler;
 import com.vertispan.j2cl.build.task.*;
-import com.vertispan.j2cl.tools.Closure;
-import org.apache.commons.io.FileUtils;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.PathMatcher;
@@ -53,7 +50,7 @@ public Task resolve(Project project, Config config) {
                                 .map(inputs(OutputTypes.BUNDLED_JS)),
                         Stream.of(input(project, OutputTypes.BUNDLED_JS))
                 )
-                .map(i -> i.filter(BUNDLE_JS))
+                .map(i -> i.filter(BUNDLE_JS, withSuffix(".bundle.js.map")))
                 .collect(Collectors.toUnmodifiableList());
 
         // Sort the projects, to try to include them in order. We can't be sure that all project
@@ -113,19 +110,20 @@ public void finish(TaskContext taskContext) throws IOException {
                     Files.copy(bundle.getAbsolutePath(), targetFile, StandardCopyOption.REPLACE_EXISTING);
                 }
 
-                File destSourcesDir = outputDir.toPath().resolve(Closure.SOURCES_DIRECTORY_NAME).toFile();
-                destSourcesDir.mkdirs();
-                for (Path dir : jsSources.stream().map(Input::getParentPaths).flatMap(Collection::stream).map(p -> p.resolve(Closure
-                        .SOURCES_DIRECTORY_NAME)).collect(Collectors.toSet())) {
-                    FileUtils.copyDirectory(dir.toFile(), destSourcesDir);
-                }
+//                File destSourcesDir = outputDir.toPath().resolve(Closure.SOURCES_DIRECTORY_NAME).toFile();
+//                destSourcesDir.mkdirs();
+//                for (Path dir : jsSources.stream().map(Input::getParentPaths).flatMap(Collection::stream).map(p -> p.resolve(Closure
+//                        .SOURCES_DIRECTORY_NAME)).collect(Collectors.toSet())) {
+//                    FileUtils.copyDirectory(dir.toFile(), destSourcesDir);
+//                }
 
                 try {
                     Gson gson = new GsonBuilder().setPrettyPrinting().create();
                     String scriptsArray = gson.toJson(sourceOrder.stream()
                             .flatMap(i -> i.getFilesAndHashes().stream())
                             .map(CachedPath::getSourcePath)
                             .map(Path::toString)
+                            .filter(s -> s.endsWith(".js"))
                             .collect(Collectors.toUnmodifiableList())
                     );
                     // unconditionally set this to false, so that our dependency order works, since we're always in BUNDLE now

j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java

@@ -4,6 +4,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.reflect.TypeToken;
+import com.google.debugging.sourcemap.SourceMapConsumerV3;
+import com.google.debugging.sourcemap.SourceMapGeneratorV3;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.javascript.jscomp.Compiler;
@@ -32,11 +34,13 @@
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FilterWriter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
+import java.io.Writer;
 import java.lang.reflect.Type;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -121,6 +125,7 @@ public Task resolve(Project project, Config config) {
             List<DependencyInfoAndSource> dependencyInfos = new ArrayList<>();
             Compiler jsCompiler = new Compiler(System.err);//TODO before merge, write this to the log
 
+            Path sourcesPath = context.outputPath().resolve(Closure.SOURCES_DIRECTORY_NAME);
             if (incrementalEnabled && context.lastSuccessfulOutput().isPresent()) {
                 // collect any dep info from disk for existing files
                 final Map<String, DependencyInfoAndSource> depInfoMap;
@@ -145,7 +150,7 @@ public Task resolve(Project project, Config config) {
                         } else {
                             // ADD or MODIFY
                             CompilerInput input = new CompilerInput(SourceFile.builder()
-                                    .withPath(context.outputPath().resolve(Closure.SOURCES_DIRECTORY_NAME).resolve(change.getSourcePath()))
+                                    .withPath(sourcesPath.resolve(change.getSourcePath()))
                                     .withOriginalPath(change.getSourcePath().toString())
                                     .build());
                             input.setCompiler(jsCompiler);
@@ -166,7 +171,7 @@ public Task resolve(Project project, Config config) {
                 for (Input jsInput : js) {
                     for (CachedPath path : jsInput.getFilesAndHashes()) {
                         CompilerInput input = new CompilerInput(SourceFile.builder()
-                                .withPath(context.outputPath().resolve(Closure.SOURCES_DIRECTORY_NAME).resolve(path.getSourcePath()))
+                                .withPath(sourcesPath.resolve(path.getSourcePath()))
                                 .withOriginalPath(path.getSourcePath().toString())
                                 .build());
                         input.setCompiler(jsCompiler);
@@ -182,6 +187,8 @@ public Task resolve(Project project, Config config) {
 
             // TODO optional/stretch-goal find first change in the list, so we can keep old prefix of bundle output
 
+            SourceMapGeneratorV3 sourceMapGenerator = new SourceMapGeneratorV3();
+
             // rebundle all (optional: remaining) files using this already handled sort
             ClosureBundler bundler = new ClosureBundler(Transpiler.NULL, new BaseTranspiler(
                     new BaseTranspiler.CompilerSupplier(
@@ -195,27 +202,52 @@ public Task resolve(Project project, Config config) {
                             ImmutableMap.of()
                     ),
                     ""
-            )).useEval(true);
+            )).useEval(false);
+
+            String sourcemapOutFileName = fileNameKey + ".bundle.js.map";
 
-            try (OutputStream outputStream = Files.newOutputStream(Paths.get(outputFile));
-                 BufferedWriter bundleOut = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) {
+            try (OutputStream outputStream = Files.newOutputStream(outputFilePath);
+                 BufferedWriter bundleOut = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8));
+                 LineCountingWriter writer = new LineCountingWriter(bundleOut)) {
                 for (DependencyInfoAndSource info : sorter.getSortedList()) {
                     String code = info.getSource();
                     String name = info.getName();
+                    String sourcemapContents = info.loadSourcemap(sourcesPath);
 
                     //TODO do we actually need this?
                     if (Compiler.isFillFileName(name) && code.isEmpty()) {
                         continue;
                     }
 
-                    // append this file and a comment where it came from
-                    bundleOut.append("//").append(name).append("\n");
-                    bundler.withPath(name).withSourceUrl(Closure.SOURCES_DIRECTORY_NAME + "/" + name).appendTo(bundleOut, info, code);
-                    bundleOut.append("\n");
+                    writer.append("//").append(name).append("\n");
+
+                    if (sourcemapContents != null) {
+                        sourceMapGenerator.setStartingPosition(writer.getLine(), 0);
+                        SourceMapConsumerV3 section = new SourceMapConsumerV3();
+                        section.parse(sourcemapContents);
+                        section.visitMappings((sourceName, symbolName, sourceStartPosition, startPosition, endPosition) -> sourceMapGenerator.addMapping(Paths.get(name).resolveSibling(sourceName).toString(), symbolName, sourceStartPosition, startPosition, endPosition));
+                        for (String source : section.getOriginalSources()) {
+                            String content = Files.readString(sourcesPath.resolve(name).resolveSibling(source));
+                            sourceMapGenerator.addSourcesContent(Paths.get(name).resolveSibling(source).toString(), content);
+                        }
+                    }
 
+                    // append this file and a comment where it came from
+                    bundler.withPath(name).appendTo(writer, info, code);
+                    writer.append("\n");
                 }
 
+                // write a reference to our new sourcemaps
+//                writer.append("// " + writer.getLine()).append("\n");
+                writer.append("//# sourceMappingURL=").append(sourcemapOutFileName).append('\n');
             }
+
+            // TODO hash in the name
+            try (OutputStream outputStream = Files.newOutputStream(outputFilePath.resolveSibling(sourcemapOutFileName));
+                 BufferedWriter smOut = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) {
+                sourceMapGenerator.appendTo(smOut, fileNameKey);
+            }
+
             // append dependency info to deserialize on some incremental rebuild
             try (OutputStream outputStream = Files.newOutputStream(context.outputPath().resolve("depInfo.json"));
                  BufferedWriter jsonOut = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) {
@@ -238,6 +270,56 @@ public Task resolve(Project project, Config config) {
         };
     }
 
+
+    public static class LineCountingWriter extends FilterWriter {
+        private int line;
+        protected LineCountingWriter(Writer out) {
+            super(out);
+        }
+
+        public int getLine() {
+            return line;
+        }
+
+        @Override
+        public void write(int c) throws IOException {
+            if (c == '\n') {
+                line++;
+            }
+            super.write(c);
+        }
+
+        @Override
+        public void write(char[] cbuf, int off, int len) throws IOException {
+            for (char c : cbuf) {
+                if (c == '\n') {
+                    line++;
+                }
+            }
+            super.write(cbuf, off, len);
+        }
+
+        @Override
+        public void write(String str, int off, int len) throws IOException {
+            str.chars().skip(off).limit(len).forEach(c -> {
+                if (c == '\n') {
+                    line++;
+                }
+            });
+            super.write(str, off, len);
+        }
+
+        @Override
+        public void write(char[] cbuf) throws IOException {
+            for (char c : cbuf) {
+                if (c == '\n') {
+                    line++;
+                }
+            }
+            super.write(cbuf);
+        }
+    }
+
     public interface SourceSupplier {
         String get() throws IOException;
     }
@@ -309,6 +391,17 @@ public boolean getHasExternsAnnotation() {
         public boolean getHasNoCompileAnnotation() {
             return delegate.getHasNoCompileAnnotation();
         }
+
+        public String loadSourcemap(Path outPath) throws IOException {
+            String sourceMappingUrlMarker = "//# sourceMappingURL=";
+            int offset = getSource().lastIndexOf(sourceMappingUrlMarker);
+            if (offset == -1) {
+                return null;
+            }
+            int urlPos = offset + sourceMappingUrlMarker.length();
+            String sourcemapName = getSource().substring(urlPos).split("\\s")[0];
+            return Files.readString(outPath.resolve(getName()).resolveSibling(sourcemapName));
+        }
     }
 
     public static class DependencyInfoFormat implements DependencyInfo {