BXC-4762 integrate jp24u jar into box-c (#1839)

* add jp24u submodule * build jp24u submodule * add jp24u to pom.xml and runJp24u method * use checkout v3 and remove separate checkout submodules step * add separate checkout submodules step * remove separate checkout submodules step * remove build jp24u with maven step * add new Jp2Processor and tests * skip jp24u tests during build * fix skip jp24u tests during build * skip jp24u during maven verify * forgot an ! * try removing quotes to fix build * try excluding jp24u logging dependencies * remove exclusions * update jp24u submodule git reference * add javadoc, remove "\"" and recipientList * fix tests * verify jp2Processor is called * update jp24u submodule and use runCommand method instead of main * add result class to extend ExecResult and set the body * suppress kakadu warnings
UNC-Libraries · Dec 12, 2024 · 5bc80d2 · 5bc80d2
1 parent 765809a
commit 5bc80d2
Show file tree

Hide file tree

Showing 13 changed files with 191 additions and 60 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -33,7 +33,9 @@ jobs:
       run: sudo chmod 2777 /tmp/boxc_test_storage
 
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
+      with:
+        submodules: 'recursive'
 
     - name: Get Container ID of fedora service
       id: get_container_id
@@ -58,9 +60,6 @@ jobs:
       run: |
         docker run -v /tmp/solr-config:/solr_config -d --rm -p 48983:8983 solr:9 solr-precreate access /solr_config/access
 
-    - name: Checkout submodules
-      run: git submodule update --init --recursive
-
     - name: Set up JDK 11
       uses: actions/setup-java@v1
       with:
@@ -98,7 +97,7 @@ jobs:
         CC_TEST_REPORTER_ID: ${{secrets.CC_TEST_REPORTER_ID}}
         JACOCO_SOURCE_PATH: "${{github.workspace}}/auth-api/src/main/java ${{github.workspace}}/auth-fcrepo/src/main/java ${{github.workspace}}/common-utils/src/main/java ${{github.workspace}}/deposit-app/src/main/java ${{github.workspace}}/deposit-utils/src/main/java ${{github.workspace}}/fcrepo-utils/src/main/java ${{github.workspace}}/indexing-solr/src/main/java ${{github.workspace}}/integration/src/main/java ${{github.workspace}}/model-api/src/main/java ${{github.workspace}}/model-fcrepo/src/main/java ${{github.workspace}}/operations/src/main/java ${{github.workspace}}/operations-jms/src/main/java ${{github.workspace}}/persistence/src/main/java ${{github.workspace}}/persistence-api/src/main/java ${{github.workspace}}/search-api/src/main/java ${{github.workspace}}/search-solr/src/main/java ${{github.workspace}}/services-camel-app/src/main/java ${{github.workspace}}/web-access-app/src/main/java ${{github.workspace}}/web-access-war/src/main/java ${{github.workspace}}/web-admin-app/src/main/java ${{github.workspace}}/web-common/src/main/java ${{github.workspace}}/web-services-app/src/main/java ${{github.workspace}}/web-sword/src/main/java  ${{github.workspace}}/clamav-java/src/main/java"
       with:
-        coverageCommand: mvn -pl !clamav-java verify
+        coverageCommand: mvn -pl !clamav-java,!jp24u verify
         coverageLocations: |
           ${{github.workspace}}/**/target/site/jacoco/jacoco.xml:jacoco
           ${{github.workspace}}/**/target/site/jacoco-it/jacoco.xml:jacoco

diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,6 @@
 [submodule "clamav-java"]
 	path = clamav-java
 	url = https://github.com/UNC-Libraries/clamav-java.git
+[submodule "jp24u"]
+	path = jp24u
+	url = https://github.com/UNC-Libraries/jp24u.git
diff --git a/jp24u b/jp24u
diff --git a/pom.xml b/pom.xml
@@ -66,6 +66,7 @@
         <fcrepo4.version>5.1.2</fcrepo4.version>
         <fcrepo-indexing-triplestore.version>5.0.0</fcrepo-indexing-triplestore.version>
         <clamav-java.version>1.0.2-SNAPSHOT</clamav-java.version>
+        <jp24u.version>1.0-SNAPSHOT</jp24u.version>
 
         <spring.version>5.3.32</spring.version>
         <spring.ws.version>3.1.8</spring.ws.version>
@@ -175,6 +176,7 @@
         <module>web-services-app</module>
         <module>web-sword</module>
         <module>integration</module>
+        <module>jp24u</module>
     </modules>
 
     <reporting>
@@ -583,6 +585,11 @@
                 <artifactId>web-sword</artifactId>
                 <version>${cdr.version}</version>
             </dependency>
+            <dependency>
+                <groupId>edu.unc.lib</groupId>
+                <artifactId>jp24u</artifactId>
+                <version>${jp24u.version}</version>
+            </dependency>
             <dependency>
                 <groupId>fi.solita.clamav</groupId>
                 <artifactId>clamav-client</artifactId>

diff --git a/services-camel-app/pom.xml b/services-camel-app/pom.xml
@@ -171,6 +171,10 @@
         <groupId>edu.unc.lib.cdr</groupId>
         <artifactId>operations</artifactId>
     </dependency>
+    <dependency>
+        <groupId>edu.unc.lib</groupId>
+        <artifactId>jp24u</artifactId>
+    </dependency>
 
     <!-- Jaxb 2.x dependencies, until camel-core no longer requires them -->
     <dependency>

diff --git a/...mel-app/src/main/java/edu/unc/lib/boxc/services/camel/images/ImageEnhancementsRouter.java b/...mel-app/src/main/java/edu/unc/lib/boxc/services/camel/images/ImageEnhancementsRouter.java
@@ -30,6 +30,9 @@ public class ImageEnhancementsRouter extends RouteBuilder {
     @BeanInject(value = "imageCacheInvalidationProcessor")
     private ImageCacheInvalidationProcessor imageCacheInvalidationProcessor;
 
+    @BeanInject(value = "jp2Processor")
+    private Jp2Processor jp2Processor;
+
     private UuidGenerator uuidGenerator;
 
     /**
@@ -59,8 +62,7 @@ public void configure() throws Exception {
                 .setBody(exchange -> uuidGenerator.generateUuid())
                 .setHeader(CdrFcrepoHeaders.CdrTempPath, simple("${properties:services.tempDirectory}/${body}-access"))
                 .doTry()
-                    .recipientList(simple("exec:/bin/sh?args=${properties:cdr.enhancement.bin}/convertJp2.sh "
-                            + "${headers[CdrImagePath]} ${headers[CdrMimeType]} ${headers[CdrTempPath]}"))
+                    .bean(jp2Processor)
                     .bean(addAccessCopyProcessor)
                     // Process cache invalidation asynchronously with a limited number of threads
                     .threads(CACHE_INVALIDATE_THREADS)

diff --git a/services-camel-app/src/main/java/edu/unc/lib/boxc/services/camel/images/Jp2Processor.java b/services-camel-app/src/main/java/edu/unc/lib/boxc/services/camel/images/Jp2Processor.java
@@ -0,0 +1,50 @@
+package edu.unc.lib.boxc.services.camel.images;
+
+import JP2ImageConverter.CLIMain;
+import edu.unc.lib.boxc.services.camel.util.CdrFcrepoHeaders;
+import org.apache.camel.Exchange;
+import org.apache.camel.LoggingLevel;
+import org.apache.camel.Message;
+import org.apache.camel.Processor;
+import org.apache.camel.component.exec.ExecCommand;
+import org.apache.camel.component.exec.ExecResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.util.Arrays;
+
+/**
+ * Processor that runs jp24u to generate jp2 derivatives
+ * @author krwong
+ */
+public class Jp2Processor implements Processor {
+    private static final Logger log = LoggerFactory.getLogger(Jp2Processor.class);
+
+    @Override
+    public void process(Exchange exchange) throws Exception {
+        Message in = exchange.getIn();
+        String imagePath = (String) in.getHeader(CdrFcrepoHeaders.CdrImagePath);
+        String tempPath = (String) in.getHeader(CdrFcrepoHeaders.CdrTempPath);
+        String mimetype = (String) in.getHeader(CdrFcrepoHeaders.CdrBinaryMimeType);
+
+        String[] command = new String[]{"jp24u", "kdu_compress", "-f", imagePath,
+                "-o", tempPath, "-sf", mimetype};
+        log.debug("Run jp24u command {} for type {}", command, mimetype);
+        int exitCode = CLIMain.runCommand(command);
+
+        Result result = new Result(new ByteArrayInputStream(tempPath.getBytes()), null, exitCode);
+        in.setBody(result);
+    }
+
+    public static class Result extends ExecResult {
+        private static final ExecCommand command = new ExecCommand("jp24u",
+                Arrays.asList("kdu_compress"), null, Long.valueOf(60), null,
+                null, null, false, LoggingLevel.OFF);
+
+        public Result(InputStream stdout, InputStream stderr, int exitCode) {
+            super(command, stdout, stderr, exitCode);
+        }
+    }
+}
diff --git a/services-camel-app/src/main/resources/logback.xml b/services-camel-app/src/main/resources/logback.xml
@@ -6,6 +6,7 @@
       <pattern>%d %-5level [%thread] %logger{20} - %msg%n</pattern>
     </encoder>
   </appender>
+  <logger name="JP2ImageConverter.services.KakaduService" level="OFF"/>
   <root level="INFO">
     <appender-ref ref="CONSOLE"/>
   </root>

diff --git a/services-camel-app/src/main/webapp/WEB-INF/service-context.xml b/services-camel-app/src/main/webapp/WEB-INF/service-context.xml
@@ -208,6 +208,9 @@
         <property name="titleRetrievalService" ref="titleRetrievalService" />
         <property name="memberOrderService" ref="memberOrderService" />
     </bean>
+
+    <bean id="jp2Processor" class="edu.unc.lib.boxc.services.camel.images.Jp2Processor">
+    </bean>
 
     <bean id="addAccessCopyProcessor" class="edu.unc.lib.boxc.services.camel.images.AddDerivativeProcessor">
         <constructor-arg value="#{T(edu.unc.lib.boxc.model.api.DatastreamType).JP2_ACCESS_COPY.getExtension()}" />

diff --git a/...app/src/test/java/edu/unc/lib/boxc/services/camel/images/ImageEnhancementsRouterTest.java b/...app/src/test/java/edu/unc/lib/boxc/services/camel/images/ImageEnhancementsRouterTest.java
@@ -1,10 +1,8 @@
 package edu.unc.lib.boxc.services.camel.images;
 
 import org.apache.camel.BeanInject;
-import org.apache.camel.CamelExecutionException;
 import org.apache.camel.EndpointInject;
 import org.apache.camel.Exchange;
-import org.apache.camel.Processor;
 import org.apache.camel.Produce;
 import org.apache.camel.ProducerTemplate;
 import org.apache.camel.PropertyInject;
@@ -28,16 +26,14 @@
 import static edu.unc.lib.boxc.model.api.rdf.Fcrepo4Repository.Binary;
 import static edu.unc.lib.boxc.model.fcrepo.ids.RepositoryPaths.idToPath;
 import static edu.unc.lib.boxc.services.camel.util.CdrFcrepoHeaders.CdrBinaryMimeType;
+import static edu.unc.lib.boxc.services.camel.util.CdrFcrepoHeaders.CdrImagePath;
+import static edu.unc.lib.boxc.services.camel.util.CdrFcrepoHeaders.CdrTempPath;
 import static org.fcrepo.camel.FcrepoHeaders.FCREPO_AGENT;
 import static org.fcrepo.camel.FcrepoHeaders.FCREPO_BASE_URL;
 import static org.fcrepo.camel.FcrepoHeaders.FCREPO_DATE_TIME;
 import static org.fcrepo.camel.FcrepoHeaders.FCREPO_EVENT_TYPE;
 import static org.fcrepo.camel.FcrepoHeaders.FCREPO_URI;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
 import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -53,6 +49,8 @@ public class ImageEnhancementsRouterTest extends CamelSpringTestSupport {
     private static final String fileID = "343b3da4-8876-42f5-8821-7aabb65e0f19";
     private final String eventTypes = EVENT_NS + "ResourceCreation";
     private final String accessCopyRoute = "AccessCopy";
+    private static final String fileName = "de/75/d8/11/de75d811-9e0f-4b1f-8631-2060ab3580cc";
+    private static final String derivTmpPath = "tmp/" + fileName;
 
     @PropertyInject(value = "fcrepo.baseUrl")
     private static String baseUri;
@@ -63,6 +61,9 @@ public class ImageEnhancementsRouterTest extends CamelSpringTestSupport {
     @Produce("direct:process.binary.original")
     protected ProducerTemplate template;
 
+    @BeanInject(value = "jp2Processor")
+    private Jp2Processor jp2Processor;
+
     @BeanInject(value = "addAccessCopyProcessor")
     private AddDerivativeProcessor addAccessCopyProcessor;
 
@@ -84,58 +85,28 @@ public void testAccessCopyRouteNoForceNoFileExists() throws Exception {
         when(addAccessCopyProcessor.needsRun(any())).thenReturn(true);
         createContext(accessCopyRoute);
 
-        var shEndpoint = getMockEndpoint("mock:exec:/bin/sh");
-        shEndpoint.expectedMessageCount(1);
-
         Map<String, Object> headers = createEvent(fileID, eventTypes, "false");
 
         template.sendBodyAndHeaders("", headers);
 
+        verify(jp2Processor).process(any(Exchange.class));
         verify(addAccessCopyProcessor).process(any(Exchange.class));
         verify(addAccessCopyProcessor).cleanupTempFile(any(Exchange.class));
         verify(imageCacheInvalidationProcessor).process(any());
-        shEndpoint.assertIsSatisfied();
-    }
-
-    @Test
-    public void testAccessCopyRouteScriptFails() throws Exception {
-        when(addAccessCopyProcessor.needsRun(any())).thenReturn(true);
-        createContext(accessCopyRoute);
-
-        MockEndpoint shEndpoint = getMockEndpoint("mock:exec:/bin/sh");
-        shEndpoint.expectedMessageCount(1);
-        shEndpoint.whenAnyExchangeReceived(exchange -> {
-            throw new IllegalStateException("Failing run of exec");
-        });
-
-        Map<String, Object> headers = createEvent(fileID, eventTypes, "false");
-        try {
-            template.sendBodyAndHeaders("", headers);
-            fail("Exception expected to be thrown");
-        } catch (CamelExecutionException e) {
-            assertTrue(e.getCause() instanceof IllegalStateException);
-        }
-
-        verify(addAccessCopyProcessor, never()).process(any(Exchange.class));
-        verify(addAccessCopyProcessor).cleanupTempFile(any(Exchange.class));
-        shEndpoint.assertIsSatisfied();
     }
 
     @Test
     public void testAccessCopyRouteForceNoFileExists() throws Exception {
         when(addAccessCopyProcessor.needsRun(any())).thenReturn(true);
         createContext(accessCopyRoute);
 
-        var shEndpoint = getMockEndpoint("mock:exec:/bin/sh");
-        shEndpoint.expectedMessageCount(1);
-
         Map<String, Object> headers = createEvent(fileID, eventTypes, "true");
 
         template.sendBodyAndHeaders("", headers);
 
+        verify(jp2Processor).process(any(Exchange.class));
         verify(addAccessCopyProcessor).process(any(Exchange.class));
         verify(imageCacheInvalidationProcessor).process(any());
-        shEndpoint.assertIsSatisfied();
     }
 
     @Test
@@ -146,17 +117,14 @@ public void testAccessCopyRouteNoForceFileExists() throws Exception {
 
         createContext(accessCopyRoute);
 
-        var shEndpoint = getMockEndpoint("mock:exec:/bin/sh");
-        shEndpoint.expectedMessageCount(0);
-
         Map<String, Object> headers = createEvent(fileID, eventTypes, "false");
 
         template.sendBodyAndHeaders("", headers);
 
+        verify(jp2Processor, never()).process(any(Exchange.class));
         verify(addAccessCopyProcessor, never()).process(any(Exchange.class));
         verify(addAccessCopyProcessor, never()).cleanupTempFile(any(Exchange.class));
         verify(imageCacheInvalidationProcessor, never()).process(any());
-        shEndpoint.assertIsSatisfied();
     }
 
     @Test
@@ -168,16 +136,13 @@ public void testAccessCopyRouteForceFileExists() throws Exception {
 
         createContext(accessCopyRoute);
 
-        var shEndpoint = getMockEndpoint("mock:exec:/bin/sh");
-        shEndpoint.expectedMessageCount(1);
-
         Map<String, Object> headers = createEvent(fileID, eventTypes, "true");
 
         template.sendBodyAndHeaders("", headers);
 
+        verify(jp2Processor).process(any(Exchange.class));
         verify(addAccessCopyProcessor).process(any(Exchange.class));
         verify(imageCacheInvalidationProcessor).process(any());
-        shEndpoint.assertIsSatisfied();
     }
 
     @Test
@@ -193,43 +158,39 @@ public void testAccessCopyRejection() throws Exception {
 
         template.sendBodyAndHeaders("", headers);
 
+        verify(jp2Processor, never()).process(any(Exchange.class));
         verify(addAccessCopyProcessor, never()).process(any(Exchange.class));
-        imageEndpoint.assertIsSatisfied();
     }
 
     @Test
     public void testAccessCopyDisallowedImageType() throws Exception {
         createContext(accessCopyRoute);
 
         when(addAccessCopyProcessor.needsRun(any())).thenReturn(true);
-        var shEndpoint = getMockEndpoint("mock:exec:/bin/sh");
-        shEndpoint.expectedMessageCount(0);
 
         Map<String, Object> headers = createEvent(fileID, eventTypes, "false");
         headers.put(CdrBinaryMimeType, "image/vnd.fpx");
 
         template.sendBodyAndHeaders("", headers);
 
+        verify(jp2Processor, never()).process(any(Exchange.class));
         verify(addAccessCopyProcessor, never()).process(any(Exchange.class));
-        shEndpoint.assertIsSatisfied();
     }
 
     @Test
     public void testAccessCopyIconFile() throws Exception {
         createContext(accessCopyRoute);
 
         when(addAccessCopyProcessor.needsRun(any())).thenReturn(true);
-        var shEndpoint = getMockEndpoint("mock:exec:/bin/sh");
-        shEndpoint.expectedMessageCount(0);
 
         Map<String, Object> headers = createEvent(fileID, eventTypes, "false");
         headers.put(CdrBinaryMimeType, "image/x-icon");
 
         template.sendBodyAndHeaders("", headers);
 
+        verify(jp2Processor, never()).process(any(Exchange.class));
         verify(addAccessCopyProcessor).needsRun(any(Exchange.class));
         verify(addAccessCopyProcessor, never()).process(any(Exchange.class));
-        shEndpoint.assertIsSatisfied();
     }
 
     private void createContext(String routeName) throws Exception {
@@ -251,6 +212,8 @@ private static Map<String, Object> createEvent(final String identifier, final St
         headers.put(EVENT_TYPE, "ResourceCreation");
         headers.put(IDENTIFIER, "original_file");
         headers.put(RESOURCE_TYPE, Binary.getURI());
+        headers.put(CdrTempPath, derivTmpPath);
+        headers.put(CdrImagePath, fileName);
         headers.put(CdrBinaryMimeType, "image/png");
         headers.put("force", force);