Skip to content

Commit

Permalink
Add option to disable merging blocks
Browse files Browse the repository at this point in the history
  • Loading branch information
@NeRdTheNed
NeRdTheNed committed Nov 12, 2023
1 parent c487330 commit a25bcd5
Show file tree
Hide file tree
Showing 10 changed files with 58 additions and 36 deletions.
7 changes: 6 additions & 1 deletion deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/Deft.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,15 @@ public class Deft {

/** Optimise a raw deflate stream */
public static byte[] optimiseDeflateStream(byte[] original) {
return optimiseDeflateStream(original, true);
}

/** Optimise a raw deflate stream */
public static byte[] optimiseDeflateStream(byte[] original, boolean mergeBlocks) {
final DeflateStream stream = new DeflateStream();

try {
if (stream.parse(original) && (stream.optimise() > 0)) {
if (stream.parse(original) && (stream.optimise(mergeBlocks) > 0)) {
return stream.asBytes();
}
} catch (final IOException e) {
Expand Down
13 changes: 5 additions & 8 deletions deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateStream.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,10 @@ private static DeflateBlock optimiseBlock(DeflateBlock toOptimise, long position
}

public long optimise() {
return optimise(true);
}

public long optimise(boolean mergeBlocks) {
int block = 0;
int pass = 0;
long pos = 0;
Expand Down Expand Up @@ -558,14 +562,7 @@ public long optimise() {
// TODO Try other types of blocks
// TODO Try merging more block types
// TODO Try merging blocks at different passes
final long mergeSaved = mergeBlocks();
saved += mergeSaved;

if (mergeSaved > 0) {
return saved + optimise();
}

return saved;
return mergeBlocks ? saved + mergeBlocks() : saved;
}

public long mergeBlocks() {
Expand Down
14 changes: 8 additions & 6 deletions deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/CMDUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,18 +37,20 @@ public enum RecompressMode {

/** Recompress each deflate stream, and optimise. If the result is smaller, use it. */
private final boolean recompress;
private final boolean mergeBlocks;

private final CompressionUtil compUtil;

CMDUtil(RecompressMode recompressMode, int passes) {
CMDUtil(RecompressMode recompressMode, boolean mergeBlocks, int passes) {
recompress = recompressMode.ordinal() > RecompressMode.NONE.ordinal();
final boolean zopfli = recompressMode.ordinal() >= RecompressMode.ZOPFLI.ordinal();
final Strategy strat = recompressMode.ordinal() >= RecompressMode.ZOPFLI_EXTENSIVE.ordinal() ? Strategy.EXTENSIVE : Strategy.MULTI_CHEAP;
compUtil = recompress ? new CompressionUtil(true, true, recompressMode.ordinal() >= RecompressMode.ZOPFLI_VERY_EXTENSIVE.ordinal(), zopfli, passes, strat, true, true) : null;
compUtil = recompress ? new CompressionUtil(true, true, recompressMode.ordinal() >= RecompressMode.ZOPFLI_VERY_EXTENSIVE.ordinal(), zopfli, passes, strat, true, true, mergeBlocks) : null;
this.mergeBlocks = mergeBlocks;
}

CMDUtil(RecompressMode recompressMode) {
this(recompressMode, 20);
CMDUtil(RecompressMode recompressMode, boolean mergeBlocks) {
this(recompressMode, mergeBlocks, 20);
}

/** Read from the given input stream into the container, optimise, and write to the output stream */
Expand All @@ -65,7 +67,7 @@ private boolean optimise(InputStream is, OutputStream os, DeflateFilesContainer
System.out.println(container.getStreamInfo());
}

final long saved = NO_OPT ? 0 : container.optimise();
final long saved = NO_OPT ? 0 : container.optimise(mergeBlocks);

if (saved != 0) {
System.out.println("Saved " + saved + " bits with optimisation");
Expand All @@ -84,7 +86,7 @@ private boolean optimise(InputStream is, OutputStream os, DeflateFilesContainer
final ByteArrayInputStream bais = new ByteArrayInputStream(recompresed);

if (recompStream.parse(bais)) {
recompStream.optimise();
recompStream.optimise(mergeBlocks);
final long recompSize = recompStream.getSizeBits();
final long originalSize = stream.getSizeBits();
final long streamSaved = originalSize - recompSize;
Expand Down
9 changes: 6 additions & 3 deletions deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/GZipCompress.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ public enum CompressMode {
@Option(names = { "--zopfli-iter", "--iter", "-z" }, defaultValue = "20", description = "Zopfli iterations")
int iter = 20;

@Option(names = { "--merge-blocks", "-b" }, negatable = true, defaultValue = "true", fallbackValue = "true", description = "Try merging deflate blocks. May majorly increase time spent optimising files.")
boolean mergeBlocks = true;

@Option(names = { "--keep-name", "-n" }, negatable = true, defaultValue = "true", fallbackValue = "true", description = "Write filename to GZip header")
boolean name = true;

Expand All @@ -55,10 +58,10 @@ public enum CompressMode {
@Option(names = { "--text", "-T" }, defaultValue = "false", description = "File is ASCII text")
boolean text;

private static CompressionUtil getComp(CompressMode mode, int iter) {
private static CompressionUtil getComp(CompressMode mode, boolean mergeBlocks, int iter) {
final boolean zopfli = mode.ordinal() >= CompressMode.ZOPFLI.ordinal();
final Strategy strat = mode.ordinal() >= CompressMode.ZOPFLI_EXTENSIVE.ordinal() ? Strategy.EXTENSIVE : Strategy.MULTI_CHEAP;
return new CompressionUtil(true, true, mode.ordinal() >= CompressMode.ZOPFLI_VERY_EXTENSIVE.ordinal(), zopfli, iter, strat, true, true);
return new CompressionUtil(true, true, mode.ordinal() >= CompressMode.ZOPFLI_VERY_EXTENSIVE.ordinal(), zopfli, iter, strat, true, true, mergeBlocks);
}

private static int getOS(int fallback) {
Expand Down Expand Up @@ -107,7 +110,7 @@ public Integer call() throws Exception {
return 1;
}

final CompressionUtil compUtil = getComp(recompressMode, iter);
final CompressionUtil compUtil = getComp(recompressMode, mergeBlocks, iter);
final byte[] uncompressed;

try
Expand Down
5 changes: 4 additions & 1 deletion deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/Optimise.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,12 @@ class Optimise implements Callable<Integer> {
@Option(names = { "--zopfli-iter", "--iter", "-I" }, defaultValue = "20", description = "Zopfli iterations. More iterations increases time spent optimising files.")
int recompressZopfliPasses = 20;

@Option(names = { "--merge-blocks", "-b" }, negatable = true, defaultValue = "true", fallbackValue = "true", description = "Try merging deflate blocks. May majorly increase time spent optimising files.")
boolean mergeBlocks = true;

@Override
public Integer call() throws Exception {
final CMDUtil deft = new CMDUtil(recompressMode, recompressZopfliPasses);
final CMDUtil deft = new CMDUtil(recompressMode, mergeBlocks, recompressZopfliPasses);
final boolean didOpt;

try {
Expand Down
5 changes: 4 additions & 1 deletion deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/OptimiseFolder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,12 @@ class OptimiseFolder implements Callable<Integer> {
@Option(names = { "--zopfli-iter", "--iter", "-I" }, defaultValue = "20", description = "Zopfli iterations. More iterations increases time spent optimising files.")
int recompressZopfliPasses = 20;

@Option(names = { "--merge-blocks", "-b" }, negatable = true, defaultValue = "true", fallbackValue = "true", description = "Try merging deflate blocks. May majorly increase time spent optimising files.")
boolean mergeBlocks = true;

@Override
public Integer call() throws Exception {
final CMDUtil deft = new CMDUtil(recompressMode, recompressZopfliPasses);
final CMDUtil deft = new CMDUtil(recompressMode, mergeBlocks, recompressZopfliPasses);
boolean didOpt = true;

try
Expand Down
20 changes: 11 additions & 9 deletions ...compress/src/main/java/com/github/NeRdTheNed/deft4j/util/compression/CompressionUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ public enum Strategy {

private final boolean useDeft;
private final boolean compareDeft;
private final boolean mergeBlocks;

/** Construct the list of compressors for the given settings */
private static Compressor[] getCompressors(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, int iter, Strategy mode, int defaultSplit) {
Expand Down Expand Up @@ -76,22 +77,23 @@ private static Compressor[] getCompressors(boolean java, boolean jzlib, boolean
return compressorsList.toArray(new Compressor[0]);
}

private CompressionUtil(Compressor[] compressors, boolean useDeft, boolean compareDeft) {
private CompressionUtil(Compressor[] compressors, boolean useDeft, boolean compareDeft, boolean mergeBlocks) {
this.compressors = compressors;
this.useDeft = useDeft;
this.compareDeft = compareDeft;
this.mergeBlocks = mergeBlocks;
}

public CompressionUtil(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, int iter, Strategy mode, int defaultSplit, boolean useDeft, boolean compareDeft) {
this(getCompressors(java, jzlib, jzopfli, cafeundzopfli, iter, mode, defaultSplit), useDeft, compareDeft);
public CompressionUtil(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, int iter, Strategy mode, int defaultSplit, boolean useDeft, boolean compareDeft, boolean mergeBlocks) {
this(getCompressors(java, jzlib, jzopfli, cafeundzopfli, iter, mode, defaultSplit), useDeft, compareDeft, mergeBlocks);
}

public CompressionUtil(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, int iter, Strategy mode, boolean useDeft, boolean compareDeft) {
this(java, jzlib, jzopfli, cafeundzopfli, iter, mode, JZOPFLI_DEFAULT_SPLIT, useDeft, compareDeft);
public CompressionUtil(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, int iter, Strategy mode, boolean useDeft, boolean compareDeft, boolean mergeBlocks) {
this(java, jzlib, jzopfli, cafeundzopfli, iter, mode, JZOPFLI_DEFAULT_SPLIT, useDeft, compareDeft, mergeBlocks);
}

public CompressionUtil(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, Strategy mode, boolean useDeft, boolean compareDeft) {
this(java, jzlib, jzopfli, cafeundzopfli, ZOPFLI_ITER, mode, useDeft, compareDeft);
public CompressionUtil(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, Strategy mode, boolean useDeft, boolean compareDeft, boolean mergeBlocks) {
this(java, jzlib, jzopfli, cafeundzopfli, ZOPFLI_ITER, mode, useDeft, compareDeft, mergeBlocks);
}

private Supplier<ExecutorService> execService = () -> {
Expand All @@ -110,7 +112,7 @@ public byte[] compress(byte[] uncompressedData, boolean threaded) throws IOExcep
int tasks = 0;

for (final Compressor compressor : compressors) {
compService.submit(new CompressorTask(compressor, uncompressedData, useDeft));
compService.submit(new CompressorTask(compressor, uncompressedData, useDeft, mergeBlocks));
tasks++;
}

Expand Down Expand Up @@ -149,7 +151,7 @@ public byte[] compress(byte[] uncompressedData, boolean threaded) throws IOExcep

for (byte[] currentResult : currentResultList) {
if (useDeft) {
currentResult = Deft.optimiseDeflateStream(currentResult);
currentResult = Deft.optimiseDeflateStream(currentResult, mergeBlocks);
}

if ((compressedData == null) || (compareDeft ? Deft.getSizeBitsFallback(currentResult) < currentSizeBits : currentResult.length < compressedData.length)) {
Expand Down
6 changes: 4 additions & 2 deletions ...-compress/src/main/java/com/github/NeRdTheNed/deft4j/util/compression/CompressorTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@ class CompressorTask implements Callable<byte[][]> {
private final Compressor compressor;
private final byte[] uncompressedData;
private final boolean optimiseDeft;
private final boolean mergeBlocks;

CompressorTask(Compressor comp, byte[] uncompressedData, boolean optimiseDeft) {
CompressorTask(Compressor comp, byte[] uncompressedData, boolean optimiseDeft, boolean mergeBlocks) {
compressor = comp;
this.uncompressedData = uncompressedData;
this.optimiseDeft = optimiseDeft;
this.mergeBlocks = mergeBlocks;
}

@Override
Expand All @@ -28,7 +30,7 @@ public byte[][] call() throws Exception {
final int length = compressed.length;

for (int i = 0; i < length; i++) {
compressed[i] = Deft.optimiseDeflateStream(compressed[i]);
compressed[i] = Deft.optimiseDeflateStream(compressed[i], mergeBlocks);
}
}

Expand Down
11 changes: 8 additions & 3 deletions ...container/src/main/java/com/github/NeRdTheNed/deft4j/container/DeflateFilesContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public interface DeflateFilesContainer extends Closeable {
boolean RECALC = true;

/** Optimise all given streams. Returns the total amount of bits saved. */
static long optimise(List<DeflateStream> streams) {
static long optimise(List<DeflateStream> streams, boolean mergeBlocks) {
long savedTotal = 0;
final int size = streams.size();

Expand All @@ -26,7 +26,7 @@ static long optimise(List<DeflateStream> streams) {
System.out.println("Stream " + defStream + " (" + stream.getName() + ")");
}

final long saved = stream.optimise();
final long saved = stream.optimise(mergeBlocks);

if (Deft.PRINT_OPT && (saved > 0)) {
System.out.println(saved + " bits saved in stream " + defStream + " (" + stream.getName() + ")");
Expand Down Expand Up @@ -64,7 +64,12 @@ default byte[] write() throws IOException {

/** Optimise all streams in this container. Returns the total amount of bits saved. */
default long optimise() {
return optimise(getDeflateStreams());
return optimise(true);
}

/** Optimise all streams in this container. Returns the total amount of bits saved. */
default long optimise(boolean mergeBlocks) {
return optimise(getDeflateStreams(), mergeBlocks);
}

/** Returns debug information for the given stream */
Expand Down
4 changes: 2 additions & 2 deletions runTestOpt.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@ java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/asyouli
java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/text.png ./test/text-opt.png | tee ./test/text-opt.png.txt
java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/apng/ball.png ./test/apng/ball-opt.png | tee ./test/apng/ball-opt.png.txt
java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/284-edge-case/284.png ./test/284-edge-case/284-opt.png | tee ./test/284-edge-case/284-opt.png.txt
java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/nerd/nerd.png ./test/nerd/nerd-opt.png | tee ./test/nerd/nerd-opt.png.txt
java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/nerd/nerd-extopt.png ./test/nerd/nerd-fullopt.png | tee ./test/nerd/nerd-fullopt.png.txt
java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise --no-merge-blocks ./test/nerd/nerd.png ./test/nerd/nerd-opt.png | tee ./test/nerd/nerd-opt.png.txt
java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise --no-merge-blocks ./test/nerd/nerd-extopt.png ./test/nerd/nerd-fullopt.png | tee ./test/nerd/nerd-fullopt.png.txt

0 comments on commit a25bcd5

Please sign in to comment.