From 1b57157fa6b65dbd2b19b3083bfd332b0f2fdd60 Mon Sep 17 00:00:00 2001 From: Thomas Pakh Date: Sat, 27 Jul 2019 19:38:05 -0400 Subject: [PATCH] Make ClientFileCollector parallel --- build.gradle | 4 +-- .../launcher/builder/ClientFileCollector.java | 36 ++++++++++++++++++- .../launcher/builder/DirectoryWalker.java | 14 ++++++-- .../launcher/model/modpack/Manifest.java | 3 +- 4 files changed, 51 insertions(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index 8565ca426..13a1cc44a 100644 --- a/build.gradle +++ b/build.gradle @@ -29,8 +29,8 @@ subprojects { group = 'com.skcraft' version = '4.3-SNAPSHOT' - sourceCompatibility = 1.6 - targetCompatibility = 1.6 + sourceCompatibility = 1.8 + targetCompatibility = 1.8 repositories { mavenCentral() diff --git a/launcher-builder/src/main/java/com/skcraft/launcher/builder/ClientFileCollector.java b/launcher-builder/src/main/java/com/skcraft/launcher/builder/ClientFileCollector.java index 02805a341..af947d7e6 100644 --- a/launcher-builder/src/main/java/com/skcraft/launcher/builder/ClientFileCollector.java +++ b/launcher-builder/src/main/java/com/skcraft/launcher/builder/ClientFileCollector.java @@ -12,12 +12,18 @@ import com.skcraft.launcher.model.modpack.FileInstall; import com.skcraft.launcher.model.modpack.Manifest; import lombok.NonNull; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; import lombok.extern.java.Log; import org.apache.commons.io.FilenameUtils; import java.io.File; import java.io.IOException; import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.logging.Level; /** * Walks a path and adds hashed path versions to the given @@ -51,13 +57,41 @@ public ClientFileCollector(@NonNull Manifest manifest, @NonNull PropertiesApplic protected DirectoryBehavior getBehavior(@NonNull String name) { return getDirectoryBehavior(name); } + + @AllArgsConstructor + @EqualsAndHashCode + private static class FileEntry { + final File file; + final String relPath; + } + + private final ArrayList fileEntries = new ArrayList<>(); @Override protected void onFile(File file, String relPath) throws IOException { if (file.getName().endsWith(FileInfoScanner.FILE_SUFFIX) || file.getName().endsWith(URL_FILE_SUFFIX)) { return; } - + fileEntries.add(new FileEntry(file, relPath)); + } + + @Override + protected void onWalkComplete() { + long start = System.currentTimeMillis(); + fileEntries.parallelStream().forEach(fileEntry -> { + try { + addFile(fileEntry.file, fileEntry.relPath); + } catch (IOException e) { + log.log(Level.SEVERE, "Error processing file.", e); + throw new RuntimeException(e); + } + }); + long stop = System.currentTimeMillis(); + log.info("Finished processing " + fileEntries.size() + " files in " + (stop - start) + "ms."); + fileEntries.clear(); + } + + private void addFile(File file, String relPath) throws IOException { FileInstall entry = new FileInstall(); String hash = Files.hash(file, hf).toString(); String to = FilenameUtils.separatorsToUnix(FilenameUtils.normalize(relPath)); diff --git a/launcher-builder/src/main/java/com/skcraft/launcher/builder/DirectoryWalker.java b/launcher-builder/src/main/java/com/skcraft/launcher/builder/DirectoryWalker.java index f92726cdc..442238f81 100644 --- a/launcher-builder/src/main/java/com/skcraft/launcher/builder/DirectoryWalker.java +++ b/launcher-builder/src/main/java/com/skcraft/launcher/builder/DirectoryWalker.java @@ -7,6 +7,7 @@ package com.skcraft.launcher.builder; import lombok.NonNull; +import lombok.extern.java.Log; import java.io.File; import java.io.IOException; @@ -16,6 +17,7 @@ * path (which may be modified by dropping certain directory entries), * and call {@link #onFile(java.io.File, String)} with each file. */ +@Log public abstract class DirectoryWalker { public enum DirectoryBehavior { @@ -40,7 +42,11 @@ public enum DirectoryBehavior { * @throws IOException thrown on I/O error */ public final void walk(@NonNull File dir) throws IOException { - walk(dir, ""); + long start = System.currentTimeMillis(); + walk(dir, ""); + long stop = System.currentTimeMillis(); + log.info("Directory walk complete in " + (stop - start) + "ms."); + this.onWalkComplete(); } /** @@ -80,7 +86,7 @@ private void walk(@NonNull File dir, @NonNull String basePath) throws IOExceptio * Return the behavior for the given directory name. * * @param name the directory name - * @return the behavor + * @return the behavior */ protected DirectoryBehavior getBehavior(String name) { return DirectoryBehavior.CONTINUE; @@ -96,4 +102,8 @@ protected DirectoryBehavior getBehavior(String name) { protected abstract void onFile(File file, String relPath) throws IOException; + /** + * Called after the walk is completed. + */ + protected void onWalkComplete() {}; } diff --git a/launcher/src/main/java/com/skcraft/launcher/model/modpack/Manifest.java b/launcher/src/main/java/com/skcraft/launcher/model/modpack/Manifest.java index cac0f7f88..b3f46e5d6 100644 --- a/launcher/src/main/java/com/skcraft/launcher/model/modpack/Manifest.java +++ b/launcher/src/main/java/com/skcraft/launcher/model/modpack/Manifest.java @@ -22,6 +22,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Collections; import java.util.List; @Data @@ -39,7 +40,7 @@ public class Manifest extends BaseManifest { private LaunchModifier launchModifier; private List features = new ArrayList(); @JsonManagedReference("manifest") - private List tasks = new ArrayList(); + private List tasks = Collections.synchronizedList(new ArrayList()); @Getter @Setter @JsonIgnore private Installer installer; private VersionManifest versionManifest;