Improved progress reporting for CustomExporter

Author: JuicyDragon

Java/SuperUtilities/.idea/uiDesigner.xml

@@ -7,7 +7,9 @@
 
 import com.nuix.superutilities.SuperUtilities;
 import com.nuix.superutilities.loadfiles.*;
+import com.nuix.superutilities.misc.BoundedProgressInfo;
 import com.nuix.superutilities.misc.FormatUtility;
+import com.nuix.superutilities.misc.PeriodicGatedConsumer;
 import com.nuix.superutilities.misc.PlaceholderResolver;
 import com.nuix.superutilities.reporting.SimpleWorksheet;
 import com.nuix.superutilities.reporting.SimpleXlsx;
@@ -57,38 +59,48 @@ public class CustomExporter {
     private SimpleTextFileWriter errorLog = null;
 
     private boolean exportText = false;
-    private Map<String, Object> textExportSettings = new HashMap<String, Object>();
+    private Map<String, Object> textExportSettings = new HashMap<>();
     private String textFileNameTemplate = "{export_directory}\\TEXT\\{guid}.txt";
 
     private boolean exportNatives = false;
-    private Map<String, Object> emailExportSettings = new HashMap<String, Object>();
+    private Map<String, Object> emailExportSettings = new HashMap<>();
     private String nativeFileNameTemplate = "{export_directory}\\NATIVES\\{guid}.{extension}";
 
     private boolean exportPdfs = false;
-    private Map<String, Object> pdfExportSettings = new HashMap<String, Object>();
+    private Map<String, Object> pdfExportSettings = new HashMap<>();
     private String pdfFileNameTemplate = "{export_directory}\\PDF\\{guid}.pdf";
 
     private boolean exportTiffs = false;
-    private Map<String, Object> tiffExportSettings = new HashMap<String, Object>();
+    private Map<String, Object> tiffExportSettings = new HashMap<>();
     private String tiffFileNameTemplate = "{export_directory}\\IMAGE\\{guid}.{extension}";
 
     private boolean exportJson = false;
     private String jsonFileNameTemplate = "{export_directory}\\JSON\\{guid}.json";
     private JsonExporter jsonExporter = null;
 
+    private Consumer<String> messageLoggedCallback = null;
+    private PeriodicGatedConsumer<BoundedProgressInfo> progressCallback = null;
+
     /**
      * -- SETTER --
      * Sets whether DAT contents should additionally be exported as an XLSX spreadsheet.
      */
     @Setter
     private boolean exportXlsx = false;
+
     /**
      * -- SETTER --
      * Sets whether final DAT will be kept by moving it to final export directory/
      */
     @Setter
     private boolean keepOriginalDat = false;
 
+    /**
+     * Whether to skip export of natives for slip-sheeted items.
+     */
+    @Setter
+    private boolean skipNativesSlipsheetedItems = false;
+
     /**
      * -- SETTER --
      * Allows you to provide a Map of headers to rename.  Intended to provide a way to rename headers that Nuix automatically adds
@@ -129,6 +141,31 @@ public class CustomExporter {
     public CustomExporter() {
     }
 
+    /***
+     * Provides a callback to be invoked when progress is made during initial batch export or during
+     * restructuring phase.  Will be wrapped in a {@link PeriodicGatedConsumer} with an interval of 5 seconds
+     * if not already an instance of {@link PeriodicGatedConsumer}.
+     * @param progressCallback The progress info consumer.
+     */
+    public void whenProgressEventOccurs(Consumer<BoundedProgressInfo> progressCallback) {
+        if (progressCallback instanceof PeriodicGatedConsumer) {
+            this.progressCallback = (PeriodicGatedConsumer<BoundedProgressInfo>) progressCallback;
+        } else {
+            this.progressCallback = new PeriodicGatedConsumer<>(progressCallback, 5000);
+        }
+    }
+
+    private void fireProgressEvent(String stage, long current, long total) {
+        BoundedProgressInfo info = new BoundedProgressInfo(stage, current, total);
+        if (progressCallback != null) {
+            progressCallback.accept(info);
+        }
+    }
+
+    public void whenMessageLogged(Consumer<String> messageLoggedCallback) {
+        this.messageLoggedCallback = messageLoggedCallback;
+    }
+
     /***
      * Assigns a dynamically calculated placeholder to this instance.
      * @param placeholderName Placeholder name with "{" or "}".  For example "my_value".  Placeholder in templates can then be referred to using "{my_value}". It is
@@ -245,7 +282,13 @@ public void setColumnRemovals(Collection<String> columnHeaders) {
 
     private void logInfo(String format, Object... params) {
         String message = String.format(format, params);
-        System.out.println(message);
+
+        if (messageLoggedCallback != null) {
+            messageLoggedCallback.accept(message);
+        } else {
+            System.out.println(message);
+        }
+
         if (generalLog != null) {
             try {
                 generalLog.writeLine(message);
@@ -259,7 +302,13 @@ private void logInfo(String format, Object... params) {
 
     private void logError(String format, Object... params) {
         String message = String.format(format, params);
-        System.out.println(message);
+
+        if (messageLoggedCallback != null) {
+            messageLoggedCallback.accept("ERROR: " + message);
+        } else {
+            System.err.println(message);
+        }
+
         if (errorLog != null) {
             try {
                 errorLog.writeLine(message);
@@ -298,12 +347,7 @@ public String evaluate(Item item) {
                 }
             }
             if (!hasGuid) {
-                exportProfile = exportProfile.addMetadata("GUID", new ItemExpression<String>() {
-                    @Override
-                    public String evaluate(Item item) {
-                        return item.getGuid();
-                    }
-                });
+                exportProfile = exportProfile.addMetadata("GUID", Item::getGuid);
             }
         }
         return exportProfile;
@@ -421,7 +465,6 @@ public void exportItems(Case nuixCase, File exportDirectory, List<Item> items) t
             exporter.addProduct("tiff", productSettings);
         }
 
-
         exportDirectory.mkdirs();
         File tempDatFile = new File(exportTempDirectory, "loadfile.dat");
         File finalDatFile = new File(exportDirectory, "loadfile.dat");
@@ -448,6 +491,8 @@ public void itemProcessed(ItemEventInfo info) {
                         logError("BatchExporter reports error while exporting item with GUID '%s':\n%s",
                                 info.getItem().getGuid(), FormatUtility.debugString(info.getFailure()));
                     }
+
+                    fireProgressEvent("BatchExport: " + info.getStage(), info.getStageCount(), items.size());
                 }
             });
 
@@ -469,6 +514,9 @@ public void itemProcessed(ItemEventInfo info) {
                 exporter.setStampingOptions(stampingSettings);
             }
 
+            // Forward setting regarding natives slipsheet generation
+            exporter.setSkipNativesSlipsheetedItems(skipNativesSlipsheetedItems);
+
             logInfo("Beginning temp export using BatchExporter...");
             exporter.exportItems(items);
             logInfo("Finished temp export using BatchExporter");
@@ -495,6 +543,8 @@ public void itemProcessed(ItemEventInfo info) {
 
                     @Override
                     public void accept(LinkedHashMap<String, String> record) {
+                        fireProgressEvent("Export Restructure", recordsProcessed, items.size());
+
                         // Periodically log progress
                         long diffMillis = System.currentTimeMillis() - restructureStartMillis;
                         if (diffMillis > 2 * 1000 || recordsProcessed % 100 == 0) {

Java/SuperUtilities/src/main/java/com/nuix/superutilities/export/CustomExporter.java

@@ -0,0 +1,34 @@
+package com.nuix.superutilities.misc;
+
+import lombok.Getter;
+
+/***
+ * Represents progress of an operation which has a bounded value, that is, we known what progress
+ * value is considered maximum/done.
+ */
+@Getter
+public class BoundedProgressInfo extends ProgressInfo {
+    /***
+     * The maximum progress value
+     */
+    protected final long maximum;
+
+    public BoundedProgressInfo(String stage, long current, long maximum) {
+        super(stage, current);
+        this.maximum = maximum;
+    }
+
+    /***
+     * Provides a percentage completed value by dividing double value of current by
+     * double value of maximum.
+     * @return A double value representing percentage complete
+     */
+    public double percentageComplete() {
+        return ((double) current) / ((double) maximum);
+    }
+
+    @Override
+    public String toString() {
+        return "BoundedProgressInfo [stage=" + stage + ", " + current + "/" + maximum + "]";
+    }
+}