diff --git a/blaze-archive/pom.xml b/blaze-archive/pom.xml new file mode 100644 index 00000000..55eded21 --- /dev/null +++ b/blaze-archive/pom.xml @@ -0,0 +1,88 @@ + + + 4.0.0 + blaze-archive + jar + + + com.fizzed + blaze + 1.6.2-SNAPSHOT + + + + com.fizzed.blaze.archive + + + + + + com.fizzed + blaze-core + provided + + + + org.apache.commons + commons-compress + 1.27.1 + + + + org.tukaani + xz + 1.10 + + + + com.github.luben + zstd-jni + 1.5.6-4 + + + + + org.slf4j + jcl-over-slf4j + + + + + + org.hamcrest + hamcrest + test + + + + junit + junit + test + + + + org.mockito + mockito-core + test + + + + com.github.stefanbirkner + system-rules + test + + + + ch.qos.logback + logback-classic + test + + + + com.fizzed + crux-util + test + + + + diff --git a/blaze-archive/src/main/java/com/fizzed/blaze/archive/ArchiveFormat.java b/blaze-archive/src/main/java/com/fizzed/blaze/archive/ArchiveFormat.java new file mode 100644 index 00000000..585a9f2d --- /dev/null +++ b/blaze-archive/src/main/java/com/fizzed/blaze/archive/ArchiveFormat.java @@ -0,0 +1,27 @@ +package com.fizzed.blaze.archive; + +public class ArchiveFormat { + + final private String compressMethod; + final private String archiveMethod; + final private String[] extensions; + + public ArchiveFormat(String archiveMethod, String compressMethod, String... extensions) { + this.compressMethod = compressMethod; + this.archiveMethod = archiveMethod; + this.extensions = extensions; + } + + public String getCompressMethod() { + return compressMethod; + } + + public String getArchiveMethod() { + return archiveMethod; + } + + public String[] getExtensions() { + return extensions; + } + +} \ No newline at end of file diff --git a/blaze-archive/src/main/java/com/fizzed/blaze/archive/ArchiveFormats.java b/blaze-archive/src/main/java/com/fizzed/blaze/archive/ArchiveFormats.java new file mode 100644 index 00000000..1a72bd81 --- /dev/null +++ b/blaze-archive/src/main/java/com/fizzed/blaze/archive/ArchiveFormats.java @@ -0,0 +1,38 @@ +package com.fizzed.blaze.archive; + +import java.util.List; + +import static java.util.Arrays.asList; + +public class ArchiveFormats { + + static public final List ALL = asList( + new ArchiveFormat("zip", null, ".zip"), + new ArchiveFormat("tar", null, ".tar"), + new ArchiveFormat("tar", "gz", ".tar.gz", ".tgz"), + new ArchiveFormat("tar", "bzip2", ".tar.bz2"), + new ArchiveFormat("tar", "xz", ".tar.xz"), + new ArchiveFormat("tar", "zstd", ".tar.zst"), + new ArchiveFormat(null, "gz", ".gz") + ); + + static public ArchiveFormat detectByFileName(String fileName) { + final String n = fileName.toLowerCase(); + + // find the longest matching extension + String matchedExtension = null; + ArchiveFormat matchedFormat = null; + + for (ArchiveFormat af : ALL) { + for (String ext : af.getExtensions()) { + if (n.endsWith(ext) && (matchedExtension == null || ext.length() > matchedExtension.length())) { + matchedExtension = ext; + matchedFormat = af; + } + } + } + + return matchedFormat; + } + +} \ No newline at end of file diff --git a/blaze-archive/src/main/java/com/fizzed/blaze/archive/Unarchive.java b/blaze-archive/src/main/java/com/fizzed/blaze/archive/Unarchive.java new file mode 100644 index 00000000..bd4d2435 --- /dev/null +++ b/blaze-archive/src/main/java/com/fizzed/blaze/archive/Unarchive.java @@ -0,0 +1,209 @@ +/* + * Copyright 2015 Fizzed, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.fizzed.blaze.archive; + +import com.fizzed.blaze.Context; +import com.fizzed.blaze.core.*; +import com.fizzed.blaze.util.ObjectHelper; +import com.fizzed.blaze.util.Timer; +import com.fizzed.blaze.util.VerboseLogger; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.compress.compressors.FileNameUtil; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.*; + +public class Unarchive extends Action implements VerbosityMixin { + + static public class Result extends com.fizzed.blaze.core.Result { + + private int dirsCreated; + private int filesCopied; + private int filesOverwritten; + + Result(Unarchive action, Void value) { + super(action, value); + } + + public int getDirsCreated() { + return dirsCreated; + } + + public int getFilesCopied() { + return filesCopied; + } + + public int getFilesOverwritten() { + return filesOverwritten; + } + + } + + private final VerboseLogger log; + private final Path source; + private Path target; + //private boolean force; + private Verbosity verbosity; + + public Unarchive(Context context, Path source) { + super(context); + this.log = new VerboseLogger(this); + this.source = source; + //this.force = false; + } + + public VerboseLogger getVerboseLogger() { + return this.log; + } + + public Unarchive target(String path) { + ObjectHelper.requireNonNull(path, "path cannot be null"); + return this.target(Paths.get(path)); + } + + public Unarchive target(File path) { + ObjectHelper.requireNonNull(path, "path cannot be null"); + return this.target(path.toPath()); + } + + public Unarchive target(Path path) { + ObjectHelper.requireNonNull(path, "path cannot be null"); + this.target = path; + return this; + } + + /*public Unarchive force() { + this.force = true; + return this; + } + + public Unarchive force(boolean force) { + this.force = force; + return this; + }*/ + + @Override + protected Unarchive.Result doRun() throws BlazeException { + /*if (this.sources.isEmpty() && !this.force) { + throw new BlazeException("Copy requires at least 1 source path (and force is disabled)"); + }*/ + + // the source must all exist (we should check this first before we do anything) + if (!Files.exists(this.source)) { + throw new FileNotFoundException("Source file " + source + " not found"); + } + + // target is current directory by default, or the provided target + final Path destDir = this.target != null ? this.target : Paths.get("."); + + if (!Files.exists(destDir)) { + log.debug("Creating target directory: {}", destDir); + try { + Files.createDirectories(destDir); + } catch (IOException e) { + throw new BlazeException("Unable to create target directory", e); + } + } + + + final Result result = new Result(this, null); + final Timer timer = new Timer(); + + final ArchiveFormat archiveFormat = ArchiveFormats.detectByFileName(this.source.getFileName().toString()); + + if (archiveFormat == null) { + throw new BlazeException("Unable to detect archive format (or its unsupported)"); + } + + log.debug("Unarchiving {} -> {}", this.source, this.target); + log.debug("Detected archive format: archiveMethod={}, compressMethod={}", archiveFormat.getArchiveMethod(), archiveFormat.getCompressMethod()); + + try (InputStream fin = Files.newInputStream(this.source)) { + // we need it buffered so we can auto-detect the format with mark/reset + try (InputStream bin = new BufferedInputStream(fin)) { + // is this file compressed? + InputStream uncompressedIn = bin; + if (archiveFormat.getCompressMethod() != null) { + try { + uncompressedIn = CompressorStreamFactory.getSingleton().createCompressorInputStream(archiveFormat.getCompressMethod(), bin); + } catch (CompressorException e) { + throw new BlazeException("Unable to uncompress source", e); + } + } + + try { + // is this file archived? + if (archiveFormat.getArchiveMethod() != null) { + try { + ArchiveInputStream ais = ArchiveStreamFactory.DEFAULT.createArchiveInputStream(archiveFormat.getArchiveMethod(), uncompressedIn); + + ArchiveEntry entry = ais.getNextEntry(); + while (entry != null) { + final Path file = entry.resolveIn(destDir); + + if (entry.isDirectory()) { + Files.createDirectories(file); + } else { + log.debug(entry.getName()); + Files.createDirectories(file.getParent()); + Files.copy(ais, file, StandardCopyOption.REPLACE_EXISTING); + } + + entry = ais.getNextEntry(); + } + } catch (ArchiveException e) { + throw new BlazeException("Unable to unarchive source", e); + } + } + } finally { + if (uncompressedIn != null) { + uncompressedIn.close(); + } + } + } + + + + /*ArchiveInputStream ain = new ArchiveStreamFactory().createArchiveInputStream(ArchiveStreamFactory.ZIP, is); + ZipArchiveEntry entry = (ZipArchiveEntry) in.getNextEntry(); + OutputStream out = Files.newOutputStream(dir.toPath().resolve(entry.getName())); + IOUtils.copy(in, out); + out.close(); + in.close();final InputStream is = Files.newInputStream(input.toPath()); + ArchiveInputStream in = new ArchiveStreamFactory().createArchiveInputStream(ArchiveStreamFactory.ZIP, is); + ZipArchiveEntry entry = (ZipArchiveEntry) in.getNextEntry(); + OutputStream out = Files.newOutputStream(dir.toPath().resolve(entry.getName())); + IOUtils.copy(in, out); + out.close(); + in.close();*/ + } catch (IOException e) { + throw new BlazeException("Unable to copy", e); + } + + //log.debug("Copied {} files, overwrote {} files, created {} dirs (in {})", result.filesCopied, result.filesOverwritten, result.dirsCreated, timer); + + return new Unarchive.Result(this, null); + } + +} \ No newline at end of file diff --git a/blaze-archive/src/test/java/com/fizzed/blaze/archive/UnarchiveTest.java b/blaze-archive/src/test/java/com/fizzed/blaze/archive/UnarchiveTest.java new file mode 100644 index 00000000..84fc0381 --- /dev/null +++ b/blaze-archive/src/test/java/com/fizzed/blaze/archive/UnarchiveTest.java @@ -0,0 +1,119 @@ +package com.fizzed.blaze.archive; + +import com.fizzed.blaze.Config; +import com.fizzed.blaze.internal.ConfigHelper; +import com.fizzed.blaze.internal.ContextImpl; +import org.apache.commons.io.FileUtils; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import com.fizzed.crux.util.Resources; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.mockito.Mockito.spy; + +public class UnarchiveTest { + + Config config; + ContextImpl context; + Path targetDir; + + @Before + public void setup() throws Exception { + config = ConfigHelper.create(null); + context = spy(new ContextImpl(null, null, Paths.get("blaze.java"), config)); + targetDir = Resources.file("/fixtures/sample-no-root-dir.zip").resolve("../../../../target").toAbsolutePath().normalize(); + } + + protected Path createEmptyTargetDir(String path) throws IOException { + final Path target = targetDir.resolve(path); + FileUtils.deleteQuietly(target.toFile()); + Files.createDirectories(target); + return target; + } + + @Test + public void zipNoRootDir() throws Exception { + final Path file = Resources.file("/fixtures/sample-no-root-dir.zip"); + final Path target = this.createEmptyTargetDir("zipNoRootDir"); + + new Unarchive(this.context, file) + .target(target) + .run(); + + assertThat(Files.exists(target.resolve("a.txt")), is(true)); + assertThat(Files.exists(target.resolve("c/d.txt")), is(true)); + } + + @Test + public void tarNoRootDir() throws Exception { + final Path file = Resources.file("/fixtures/sample-no-root-dir.tar"); + final Path target = this.createEmptyTargetDir("tarNoRootDir"); + + new Unarchive(this.context, file) + .target(target) + .run(); + + assertThat(Files.exists(target.resolve("a.txt")), is(true)); + assertThat(Files.exists(target.resolve("c/d.txt")), is(true)); + } + + @Test + public void tarGzNoRootDir() throws Exception { + final Path file = Resources.file("/fixtures/sample-no-root-dir.tar.gz"); + final Path target = this.createEmptyTargetDir("tarGzNoRootDir"); + + new Unarchive(this.context, file) + .target(target) + .run(); + + assertThat(Files.exists(target.resolve("a.txt")), is(true)); + assertThat(Files.exists(target.resolve("c/d.txt")), is(true)); + } + + @Test + public void tarBz2NoRootDir() throws Exception { + final Path file = Resources.file("/fixtures/sample-no-root-dir.tar.bz2"); + final Path target = this.createEmptyTargetDir("tarBz2NoRootDir"); + + new Unarchive(this.context, file) + .target(target) + .run(); + + assertThat(Files.exists(target.resolve("a.txt")), is(true)); + assertThat(Files.exists(target.resolve("c/d.txt")), is(true)); + } + + @Test + public void tarXzNoRootDir() throws Exception { + final Path file = Resources.file("/fixtures/sample-no-root-dir.tar.xz"); + final Path target = this.createEmptyTargetDir("tarXzNoRootDir"); + + new Unarchive(this.context, file) + .target(target) + .run(); + + assertThat(Files.exists(target.resolve("a.txt")), is(true)); + assertThat(Files.exists(target.resolve("c/d.txt")), is(true)); + } + + @Test + public void tarZstNoRootDir() throws Exception { + final Path file = Resources.file("/fixtures/sample-no-root-dir.tar.zst"); + final Path target = this.createEmptyTargetDir("tarZstNoRootDir"); + + new Unarchive(this.context, file) + .target(target) + .run(); + + assertThat(Files.exists(target.resolve("a.txt")), is(true)); + assertThat(Files.exists(target.resolve("c/d.txt")), is(true)); + } + +} \ No newline at end of file diff --git a/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar b/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar new file mode 100644 index 00000000..5c5e6a52 Binary files /dev/null and b/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar differ diff --git a/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar.bz2 b/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar.bz2 new file mode 100644 index 00000000..eb130c4b Binary files /dev/null and b/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar.bz2 differ diff --git a/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar.gz b/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar.gz new file mode 100644 index 00000000..8e349c3a Binary files /dev/null and b/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar.gz differ diff --git a/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar.xz b/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar.xz new file mode 100644 index 00000000..39726481 Binary files /dev/null and b/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar.xz differ diff --git a/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar.zst b/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar.zst new file mode 100644 index 00000000..f1c7b520 Binary files /dev/null and b/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.tar.zst differ diff --git a/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.zip b/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.zip new file mode 100644 index 00000000..e38e49a5 Binary files /dev/null and b/blaze-archive/src/test/resources/fixtures/sample-no-root-dir.zip differ diff --git a/blaze-archive/src/test/resources/fixtures/sample-with-root-dir-encrypted.zip b/blaze-archive/src/test/resources/fixtures/sample-with-root-dir-encrypted.zip new file mode 100644 index 00000000..5ace666c Binary files /dev/null and b/blaze-archive/src/test/resources/fixtures/sample-with-root-dir-encrypted.zip differ diff --git a/blaze-archive/src/test/resources/fixtures/sample-with-root-dir.tar b/blaze-archive/src/test/resources/fixtures/sample-with-root-dir.tar new file mode 100644 index 00000000..128435af Binary files /dev/null and b/blaze-archive/src/test/resources/fixtures/sample-with-root-dir.tar differ diff --git a/blaze-archive/src/test/resources/fixtures/sample-with-root-dir.tar.bz2 b/blaze-archive/src/test/resources/fixtures/sample-with-root-dir.tar.bz2 new file mode 100644 index 00000000..060f6723 Binary files /dev/null and b/blaze-archive/src/test/resources/fixtures/sample-with-root-dir.tar.bz2 differ diff --git a/blaze-archive/src/test/resources/fixtures/sample-with-root-dir.tar.gz b/blaze-archive/src/test/resources/fixtures/sample-with-root-dir.tar.gz new file mode 100644 index 00000000..93b0bf59 Binary files /dev/null and b/blaze-archive/src/test/resources/fixtures/sample-with-root-dir.tar.gz differ diff --git a/blaze-archive/src/test/resources/fixtures/sample-with-root-dir.zip b/blaze-archive/src/test/resources/fixtures/sample-with-root-dir.zip new file mode 100644 index 00000000..e7cbfc75 Binary files /dev/null and b/blaze-archive/src/test/resources/fixtures/sample-with-root-dir.zip differ diff --git a/pom.xml b/pom.xml index a1936360..12d6daf6 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ 3.2.5 0.0.9 2.0.13 - 1.0.47 + 1.0.48 @@ -35,6 +35,7 @@ blaze-groovy blaze-kotlin blaze-http + blaze-archive blaze-ssh blaze-docker blaze-haproxy