Merge pull request #409 from NCEAS/develop

Release 3.0.0

Author: jeanetteclark

.github/workflows/build.yml

@@ -35,6 +35,23 @@ jobs:
       - name: Install linux deps
         run: sudo apt-get install -y libcurl4-openssl-dev
 
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      
+      - name: Install python dependencies
+        run: pip install jep==3.9  # Install the jep library
+
+      - name: Set JEP_LIBRARY_PATH environment variable
+        run: |
+          echo "JEP_LIBRARY_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')/jep" >> $GITHUB_ENV
+
+      - name: test
+        run: |
+          echo $JEP_LIBRARY_PATH
+          ls $JEP_LIBRARY_PATH
+
       - name: Install deps
         run: Rscript -e 'install.packages(c("httr", "jsonlite", "remotes"))'
 
@@ -58,7 +75,7 @@ jobs:
 
   docker-publish:
     name: Docker Build and Publish
-    if: github.ref_name == 'develop' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref_name, 'bugfix')
+    if: github.ref_name == 'develop' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref_name, 'feature')
     needs: maven-build
     runs-on: ubuntu-latest
     permissions:

.vscode/launch.json

@@ -20,6 +20,18 @@
                 ".",
                 "${fileDirname}"
             ]
+        },
+        {
+            "name": "Run MDQEngine",
+            "type": "java",
+            "request": "launch",
+            "mainClass": "edu.ucsb.nceas.mdqengine.MDQEngine",
+            "args": [
+                "/opt/local/metadig/suites/python-suite.xml",
+                "${workspaceFolder}/src/test/resources/test-docs/doi:10.18739_A2W08WG3R.xml",
+                "${workspaceFolder}/src/test/resources/test-docs/doi:10.18739_A2W08WG3R.sm",
+            ],
+            "cwd": "${workspaceFolder}"
         }
     ]
 }

Docker/metadig-worker/Dockerfile

@@ -11,7 +11,7 @@ LABEL maintainer="Jeanette Clark jclark@nceas.ucsb.edu"
 # Set the working directory 
 WORKDIR /var/lib/metadig
 
-ARG ENGINE_TAG=2.0.0
+ARG ENGINE_TAG=3.0.0-SNAPSHOT
 ENV ENGINE_TAG=${ENGINE_TAG}
 
 # The most recently built jar file is copied from the maven build directory to this dir by maven, so that
@@ -24,7 +24,11 @@ COPY target/metadig-engine-$ENGINE_TAG.jar metadig-engine-$ENGINE_TAG.jar
 COPY Docker/metadig-worker/solr solr/
 
 RUN apt-get update
-RUN apt-get install -y g++ r-base r-base-dev r-base-core r-doc-html libc6-dev libssl-dev libxml2 libxml2-dev libcurl4-openssl-dev
+RUN apt-get install -y g++ git r-base r-base-dev r-base-core r-doc-html libc6-dev libssl-dev libxml2 libxml2-dev libcurl4-openssl-dev python3 python3-pip python3-dev openjdk-17-jdk
+
+# Set Java environment variables
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
+ENV PATH="$PATH:$JAVA_HOME/bin"
 
 # Install required R packages
 RUN echo 'options(repos = c(CRAN = "http://cran.rstudio.com"))' >> /usr/lib/R/etc/Rprofile.site
@@ -34,6 +38,9 @@ RUN Rscript -e "remotes::install_github('NCEAS/metadig-r', ref = 'v.0.2.0')"
 # DataONE indexer prints copious error msgs if these files don't exist
 RUN mkdir -p /etc/dataone/index && touch /etc/dataone/index/d1client.properties && touch /etc/dataone/node.properties && touch /etc/dataone/index/cn-index-processor.properties
 
+COPY Docker/metadig-worker/requirements.txt /opt/local/metadig/
+RUN pip install -r /opt/local/metadig/requirements.txt
+
 # Run the Worker process
 # Note: docker --buile-arg only allows one argument (one token only, multiple tokens inside quotes doesn't work, so have
 #   to specify java options directly on command line.

Docker/metadig-worker/requirements.txt

@@ -0,0 +1,2 @@
+numpy
+metadig @ git+https://github.com/NCEAS/metadig-py.git

bin/sendAssessmentTest.py

@@ -1,15 +1,12 @@
-#!/Users/clark/.virtualenvs/metadig/bin/python
-
-# this is a shebang to your python path. needs to be 2.7
-#/usr/bin/python
+#!/usr/bin/python
 
 # This Python program can be used for local testing of RabbitMQ messaging between
 # metadig services, assessment processing and Solr indexing.
 
 import sys
 import re
-print "Sending assessment request to metadig-controler (test mode):"
-print "Number of arguments: ", len(sys.argv)
+print("Sending assessment request to metadig-controller (test mode):")
+print("Number of arguments: ", len(sys.argv))
 import socket
 
 host="localhost"
@@ -24,7 +21,7 @@
 clientsocket.connect((host, portNum))
 
 testCount = 1
-suiteId = "test.suite"
+suiteId = "python.suite"
 nodeId = "urn:node:ARCTIC"
 
 idVal = "doi:10.18739/A2W08WG3R"
@@ -34,17 +31,17 @@
 smFile = "%s/%s.sm" % (testDir, fileIdVal)
 
 # First send the type of request, either 'graph' or 'quality'
-clientsocket.send("%s\n" % 'quality')
+clientsocket.send('quality\n'.encode('utf-8'))
 
 # Next send the number of tests that will be run
-clientsocket.send("%s\n" % testCount)
+clientsocket.send(f"{testCount}\n".encode('utf-8'))
 
 msg = '%s,%s,%s,%s,%s\n' % (idVal, mdFile, smFile, suiteId, nodeId)
 
 print(msg)
-clientsocket.send(msg)
+clientsocket.send(msg.encode('utf-8'))
 
 # Server will stop reading from the current port connection
-clientsocket.send("Done\n")
+clientsocket.send("Done\n".encode('utf-8'))
 # Server will stop reading from any port connection, but will continue to run
 clientsocket.close()

bin/startController.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-version=2.5.0
+version=3.0.0-SNAPSHOT
 
 # Use first argument if sending jobs via port, for testing, for example:
 java -cp /opt/local/metadig/config:./target/metadig-engine-${version}.jar edu.ucsb.nceas.mdqengine.Controller 33000

bin/startWorker.sh

@@ -1,10 +1,7 @@
 #!/bin/bash
 
 # Start a local copy of metadig-worker for RabbitMQ based development testing.
-version=2.5.0
+version=3.0.0-SNAPSHOT
 
-# Include the MetaDIG python library
-export JYTHONPATH=/opt/local/metadig/metadig-py
-
-java -cp /opt/local/metadig/config:./target/metadig-engine-${version}.jar:./target/classes/solr -Dpython.path=JYTHONPATH edu.ucsb.nceas.mdqengine.Worker
+java -cp /opt/local/metadig/config:./target/metadig-engine-${version}.jar:./target/classes/solr edu.ucsb.nceas.mdqengine.Worker
 

helm/metadig-controller/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.2
+version: 1.0.4
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "2.5.0"
+appVersion: "3.0.0"

helm/metadig-controller/config.dev/metadig.properties

@@ -33,4 +33,6 @@ quartz.monitor = true
 quartz.monitor.schedule = 0 0 0 * * ?
 quartz.monitor.processing.time = 12
 quartz.monitor.run.limit = 10
+# jep library
+jep.path = /usr/local/lib/python3.10/dist-packages/jep
 

helm/metadig-controller/config/metadig.properties

@@ -59,3 +59,5 @@ quartz.monitor = true
 quartz.monitor.schedule = 0 0 0 * * ?
 quartz.monitor.processing.time = 12
 quartz.monitor.run.limit = 10
+# jep library
+jep.path = /usr/local/lib/python3.10/dist-packages/jep

helm/metadig-controller/values.yaml

@@ -7,7 +7,7 @@ replicaCount: 1
 image:
   repository: ghcr.io/nceas/metadig-controller
   pullPolicy: Always
-  tag: "v.2.5.0"
+  tag: "v.3.0.0"
 
 
 imagePullSecrets: []

helm/metadig-scheduler/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.3
+version: 1.0.4
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "2.5.0"
+appVersion: "3.0.0"

helm/metadig-scheduler/config.dev/taskList.csv

@@ -19,8 +19,8 @@ task-type,task-name,task-group,cron-schedule,params
 #   - requestType: for score tasks, determine type of portal processing ("portal" or "node")
 #
 # Dataset quality scoring tasks
-quality,quality-test-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite.1;urn:node:mnTestARCTIC;2022-05-01T00:00:00.00Z;1;1000"
-quality,quality-test-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*;FAIR-suite-0.3.1;urn:node:cnStage;2022-05-01T00:00:00.00Z;1;1000"
+quality,quality-test-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite-1.2.0;urn:node:mnTestARCTIC;2022-05-01T00:00:00.00Z;1;1000"
+quality,quality-test-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*;FAIR-suite-0.4.0;urn:node:cnStage;2022-05-01T00:00:00.00Z;1;1000"
 #
 # Portal scoring tasks
 score,portal-test-arctic-FAIR,metadig,10 0/1 * * * ?,"*portals*;FAIR-suite-0.3.1;urn:node:mnTestARCTIC;2022-05-01T00:00:00.00Z;1;100;portal"

helm/metadig-scheduler/config/taskList.csv

@@ -20,14 +20,14 @@ task-type,task-name,task-group,cron-schedule,params
 #
 # Dataset quality scoring tasks
 quality,quality-knb,metadig,0 0/1 * * * ?,"^eml.*|^http.*eml.*;knb.suite.1;urn:node:KNB;2020-08-24T00:00:00.00Z;1;1000"
-quality,quality-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite.1;urn:node:ARCTIC;2022-04-01T00:00:00.00Z;1;1000"
-quality,quality-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:ARCTIC;2022-04-01T00:00:00.00Z;1;1000"
-quality,quality-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*;FAIR-suite-0.3.1;urn:node:CN;2010-01-01T00:00:00.00Z;1;1000"
-quality,quality-ess-dive,metadig,15 0/1 * * * ?,"^eml.*|^http.*eml.*;ess-dive.data.center.suite-1.1.0;urn:node:ESS_DIVE;2016-01-01T00:00:00.00Z;1;1000;"
-quality,quality-cerp_sfwmd,metadig,20 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:CERP_SFWMD;2010-10-07T00:00:00.00Z;1;1000;"
-quality,quality-CA_OPC,metadig,25 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:CA_OPC;2010-10-07T00:00:00.00Z;1;1000;"
-quality,quality-sctld,metadig,30 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:SCTLD;2010-10-07T00:00:00.00Z;1;1000;"
-quality,quality-drp,metadig,35 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.3.1;urn:node:DRP;2010-10-07T00:00:00.00Z;1;1000;"
+quality,quality-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;arctic.data.center.suite-1.2.0;urn:node:ARCTIC;2022-04-01T00:00:00.00Z;1;1000"
+quality,quality-arctic,metadig,5 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:ARCTIC;2022-04-01T00:00:00.00Z;1;1000"
+quality,quality-dataone-fair,metadig,10 0/1 * * * ?,"^eml.*|^http.*eml.*|.*www.isotc211.org.*;FAIR-suite-0.4.0;urn:node:CN;2010-01-01T00:00:00.00Z;1;1000"
+quality,quality-ess-dive,metadig,15 0/1 * * * ?,"^eml.*|^http.*eml.*;ess-dive.data.center.suite-1.2.0;urn:node:ESS_DIVE;2016-01-01T00:00:00.00Z;1;1000;"
+quality,quality-cerp_sfwmd,metadig,20 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:CERP_SFWMD;2010-10-07T00:00:00.00Z;1;1000;"
+quality,quality-CA_OPC,metadig,25 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:CA_OPC;2010-10-07T00:00:00.00Z;1;1000;"
+quality,quality-sctld,metadig,30 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:SCTLD;2010-10-07T00:00:00.00Z;1;1000;"
+quality,quality-drp,metadig,35 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:DRP;2010-10-07T00:00:00.00Z;1;1000;"
 #
 # Portal scoring tasks
 score,portal-KNB-FAIR,metadig,1 0/1 * * * ?,"*portals*;FAIR-suite-0.3.1;urn:node:KNB;2020-08-10T00:00:00.00Z;1;100;portal"

helm/metadig-scheduler/values.yaml

@@ -7,7 +7,7 @@ replicaCount: 1
 image:
   repository: ghcr.io/nceas/metadig-scheduler
   pullPolicy: Always
-  tag: "v.2.5.0"
+  tag: "v.3.0.0"
 
 imagePullSecrets: []
 nameOverride: ""

helm/metadig-scorer/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.2
+version: 1.0.4
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "2.5.0"
+appVersion: "3.0.0"

helm/metadig-scorer/values.yaml

@@ -7,7 +7,7 @@ replicaCount: 1
 image:
   repository: ghcr.io/nceas/metadig-scorer
   pullPolicy: Always
-  tag: "v.2.5.0"
+  tag: "v.3.0.0"
 
 
 imagePullSecrets: []

helm/metadig-worker/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.2
+version: 1.0.4
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "2.5.0"
+appVersion: "3.0.0"

helm/metadig-worker/values.yaml

@@ -7,7 +7,7 @@ replicaCount: 1
 image:
   repository: ghcr.io/nceas/metadig-worker
   pullPolicy: Always
-  tag: "v.2.5.0"
+  tag: "v.3.0.0"
 
 imagePullSecrets: []
 nameOverride: ""