Merge pull request #399 from NCEAS/feature-jep

Replace Jython with Jep to enable use of modern python in checks

.github/workflows/build.yml

@@ -35,6 +35,23 @@ jobs:
       - name: Install linux deps
         run: sudo apt-get install -y libcurl4-openssl-dev
 
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install python dependencies
+        run: pip install jep==3.9  # Install the jep library
+
+      - name: Set JEP_LIBRARY_PATH environment variable
+        run: |
+          echo "JEP_LIBRARY_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')/jep" >> $GITHUB_ENV
+
+      - name: test
+        run: |
+          echo $JEP_LIBRARY_PATH
+          ls $JEP_LIBRARY_PATH
+
       - name: Install deps
         run: Rscript -e 'install.packages(c("httr", "jsonlite", "remotes"))'
 
@@ -58,7 +75,7 @@ jobs:
 
   docker-publish:
     name: Docker Build and Publish
-    if: github.ref_name == 'develop' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref_name, 'bugfix')
+    if: github.ref_name == 'develop' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref_name, 'feature')
     needs: maven-build
     runs-on: ubuntu-latest
     permissions:

.vscode/launch.json

@@ -20,6 +20,18 @@
                 ".",
                 "${fileDirname}"
             ]
+        },
+        {
+            "name": "Run MDQEngine",
+            "type": "java",
+            "request": "launch",
+            "mainClass": "edu.ucsb.nceas.mdqengine.MDQEngine",
+            "args": [
+                "/opt/local/metadig/suites/python-suite.xml",
+                "${workspaceFolder}/src/test/resources/test-docs/doi:10.18739_A2W08WG3R.xml",
+                "${workspaceFolder}/src/test/resources/test-docs/doi:10.18739_A2W08WG3R.sm",
+            ],
+            "cwd": "${workspaceFolder}"
         }
     ]
 }

Docker/metadig-worker/Dockerfile

@@ -11,7 +11,7 @@ LABEL maintainer="Jeanette Clark jclark@nceas.ucsb.edu"
 # Set the working directory 
 WORKDIR /var/lib/metadig
 
-ARG ENGINE_TAG=2.0.0
+ARG ENGINE_TAG=3.0.0-SNAPSHOT
 ENV ENGINE_TAG=${ENGINE_TAG}
 
 # The most recently built jar file is copied from the maven build directory to this dir by maven, so that
@@ -24,7 +24,11 @@ COPY target/metadig-engine-$ENGINE_TAG.jar metadig-engine-$ENGINE_TAG.jar
 COPY Docker/metadig-worker/solr solr/
 
 RUN apt-get update
-RUN apt-get install -y g++ r-base r-base-dev r-base-core r-doc-html libc6-dev libssl-dev libxml2 libxml2-dev libcurl4-openssl-dev
+RUN apt-get install -y g++ git r-base r-base-dev r-base-core r-doc-html libc6-dev libssl-dev libxml2 libxml2-dev libcurl4-openssl-dev python3 python3-pip python3-dev openjdk-17-jdk
+
+# Set Java environment variables
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
+ENV PATH="$PATH:$JAVA_HOME/bin"
 
 # Install required R packages
 RUN echo 'options(repos = c(CRAN = "http://cran.rstudio.com"))' >> /usr/lib/R/etc/Rprofile.site
@@ -34,6 +38,9 @@ RUN Rscript -e "remotes::install_github('NCEAS/metadig-r', ref = 'v.0.2.0')"
 # DataONE indexer prints copious error msgs if these files don't exist
 RUN mkdir -p /etc/dataone/index && touch /etc/dataone/index/d1client.properties && touch /etc/dataone/node.properties && touch /etc/dataone/index/cn-index-processor.properties
 
+COPY Docker/metadig-worker/requirements.txt /opt/local/metadig/
+RUN pip install -r /opt/local/metadig/requirements.txt
+
 # Run the Worker process
 # Note: docker --buile-arg only allows one argument (one token only, multiple tokens inside quotes doesn't work, so have
 #   to specify java options directly on command line.

Docker/metadig-worker/requirements.txt

@@ -0,0 +1,2 @@
+numpy
+metadig @ git+https://github.com/NCEAS/metadig-py.git@develop

bin/sendAssessmentTest.py

@@ -1,15 +1,12 @@
-#!/Users/clark/.virtualenvs/metadig/bin/python
-
-# this is a shebang to your python path. needs to be 2.7
-#/usr/bin/python
+#!/usr/bin/python
 
 # This Python program can be used for local testing of RabbitMQ messaging between
 # metadig services, assessment processing and Solr indexing.
 
 import sys
 import re
-print "Sending assessment request to metadig-controler (test mode):"
-print "Number of arguments: ", len(sys.argv)
+print("Sending assessment request to metadig-controller (test mode):")
+print("Number of arguments: ", len(sys.argv))
 import socket
 
 host="localhost"
@@ -24,7 +21,7 @@
 clientsocket.connect((host, portNum))
 
 testCount = 1
-suiteId = "test.suite"
+suiteId = "python.suite"
 nodeId = "urn:node:ARCTIC"
 
 idVal = "doi:10.18739/A2W08WG3R"
@@ -34,17 +31,17 @@
 smFile = "%s/%s.sm" % (testDir, fileIdVal)
 
 # First send the type of request, either 'graph' or 'quality'
-clientsocket.send("%s\n" % 'quality')
+clientsocket.send('quality\n'.encode('utf-8'))
 
 # Next send the number of tests that will be run
-clientsocket.send("%s\n" % testCount)
+clientsocket.send(f"{testCount}\n".encode('utf-8'))
 
 msg = '%s,%s,%s,%s,%s\n' % (idVal, mdFile, smFile, suiteId, nodeId)
 
 print(msg)
-clientsocket.send(msg)
+clientsocket.send(msg.encode('utf-8'))
 
 # Server will stop reading from the current port connection
-clientsocket.send("Done\n")
+clientsocket.send("Done\n".encode('utf-8'))
 # Server will stop reading from any port connection, but will continue to run
 clientsocket.close()

bin/startController.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-version=2.5.0
+version=3.0.0-SNAPSHOT
 
 # Use first argument if sending jobs via port, for testing, for example:
 java -cp /opt/local/metadig/config:./target/metadig-engine-${version}.jar edu.ucsb.nceas.mdqengine.Controller 33000

bin/startWorker.sh

@@ -1,10 +1,7 @@
 #!/bin/bash
 
 # Start a local copy of metadig-worker for RabbitMQ based development testing.
-version=2.5.0
+version=3.0.0-SNAPSHOT
 
-# Include the MetaDIG python library
-export JYTHONPATH=/opt/local/metadig/metadig-py
-
-java -cp /opt/local/metadig/config:./target/metadig-engine-${version}.jar:./target/classes/solr -Dpython.path=JYTHONPATH edu.ucsb.nceas.mdqengine.Worker
+java -cp /opt/local/metadig/config:./target/metadig-engine-${version}.jar:./target/classes/solr edu.ucsb.nceas.mdqengine.Worker
 

helm/metadig-controller/config.dev/metadig.properties

@@ -33,4 +33,6 @@ quartz.monitor = true
 quartz.monitor.schedule = 0 0 0 * * ?
 quartz.monitor.processing.time = 12
 quartz.monitor.run.limit = 10
+# jep library
+jep.path = /usr/local/lib/python3.10/dist-packages/jep
 

helm/metadig-controller/config/metadig.properties

@@ -59,3 +59,5 @@ quartz.monitor = true
 quartz.monitor.schedule = 0 0 0 * * ?
 quartz.monitor.processing.time = 12
 quartz.monitor.run.limit = 10
+# jep library
+jep.path = /usr/local/lib/python3.10/dist-packages/jep

pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>edu.ucsb.nceas</groupId>
   <artifactId>metadig-engine</artifactId>
-  <version>2.5.0</version>
+  <version>3.0.0-SNAPSHOT</version>
   <packaging>jar</packaging>
   <name>metadig-engine</name>
   <description>MetaDIG library for running metadata quality tests</description>
@@ -16,14 +16,14 @@
     <renjin.version>0.8.2567</renjin.version>
     <jython.version>2.7.3</jython.version>
     <gson.version>2.10.1</gson.version>
-    <json-path.version>2.4.0</json-path.version>
+    <json-path.version>2.8.0</json-path.version>
     <moxy.version>2.7.0</moxy.version>
     <log4j.version>2.20.0</log4j.version>
     <d1_cn_index_processor_version>2.3.15</d1_cn_index_processor_version>
     <d1_cn_index_common_version>2.3.2</d1_cn_index_common_version>
     <!-- Have to use a very old version of Spring in order to stay compatible with d1_cn_index_processor -->
     <!-- <spring.version>4.3.8.RELEASE</spring.version> -->
-    <spring.version>3.1.4.RELEASE</spring.version>
+    <spring.version>5.3.22</spring.version>
     <!-- <docker.registry>docker.io</docker.registry> -->
     <docker.repo>metadig</docker.repo>
     <docker.tag>2.5.0</docker.tag>
@@ -39,6 +39,7 @@
     <url>http://nceas.ucsb.edu</url>
   </organization>
   <dependencies>
+
     <!-- https://mvnrepository.com/artifact/org.quartz-scheduler/quartz -->
     <!--<dependency>-->
     <!--<groupId>org.quartz-scheduler</groupId>-->
@@ -63,9 +64,9 @@
     </dependency>
     <!-- https://mvnrepository.com/artifact/org.python/jython-standalone -->
     <dependency>
-      <groupId>org.python</groupId>
-      <artifactId>jython-standalone</artifactId>
-      <version>${jython.version}</version>
+      <groupId>black.ninia</groupId>
+      <artifactId>jep</artifactId>
+      <version>4.1.1</version>
     </dependency>
     <!-- https://mvnrepository.com/artifact/junit/junit -->
     <dependency>
@@ -217,7 +218,7 @@
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-log4j12</artifactId>
-      <version>1.6.1</version>
+      <version>1.7.33</version>
       <exclusions>
         <exclusion>
           <groupId>log4j</groupId>
@@ -320,6 +321,10 @@
           <groupId>org.slf4j</groupId>
           <artifactId>slf4j-api</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.springframework</groupId>
+          <artifactId>spring-context-support</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <!-- https://mvnrepository.com/artifact/org.json/json -->
@@ -421,10 +426,10 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>2.5.1</version>
+        <version>3.8.1</version>
         <configuration>
-          <source>1.8</source>
-          <target>1.8</target>
+          <source>17</source>
+          <target>17</target>
         </configuration>
       </plugin>
       <!-- Include all dependencies using a 'shade' jar so that the
@@ -436,7 +441,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.1.0</version>
+        <version>3.5.1</version>
         <executions>
           <execution>
             <phase>package</phase>
@@ -487,7 +492,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-antrun-plugin</artifactId>
-        <version>1.8</version>
+        <version>3.1.0</version>
         <executions>
           <execution>
             <id>copy-jar-file</id>

src/main/java/edu/ucsb/nceas/mdqengine/MDQEngine.java

@@ -1,5 +1,6 @@
 package edu.ucsb.nceas.mdqengine;
 
+import edu.ucsb.nceas.mdqengine.dispatch.Dispatcher;
 import edu.ucsb.nceas.mdqengine.dispatch.MDQCache;
 import edu.ucsb.nceas.mdqengine.exception.MetadigException;
 import edu.ucsb.nceas.mdqengine.model.*;
@@ -128,6 +129,8 @@ public Run runSuite(Suite suite, InputStream input, Map<String, Object> params,
 		}
 		run.setResult(results);
 
+		Dispatcher.getDispatcher("python").close();
+
 		log.trace("Run results: " + JsonMarshaller.toJson(run));
 
 		// clean up

src/main/java/edu/ucsb/nceas/mdqengine/Worker.java

@@ -2,6 +2,7 @@
 
 import com.rabbitmq.client.*;
 import edu.ucsb.nceas.mdqengine.collections.Runs;
+import edu.ucsb.nceas.mdqengine.dispatch.Dispatcher;
 import edu.ucsb.nceas.mdqengine.exception.MetadigException;
 import edu.ucsb.nceas.mdqengine.exception.MetadigIndexException;
 import edu.ucsb.nceas.mdqengine.exception.MetadigProcessException;
@@ -86,6 +87,7 @@ public static void main(String[] argv) throws Exception {
 
         Worker wkr = new Worker();
         MDQconfig cfg = new MDQconfig();
+        Dispatcher.setupJep();
 
         try {
             RabbitMQpassword = cfg.getString("RabbitMQ.password");