Merge pull request #21 from samply/master_extra_collection_attributes

Master extra collection attributes

.github/workflows/ci.yml

@@ -17,6 +17,18 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+    - name: Clone directory-sync repository
+      uses: actions/checkout@v2
+      with:
+        repository: samply/directory-sync
+        path: directory-sync
+        ref: master
+
+    - name: Build directory-sync library
+      run: |
+        cd directory-sync
+        mvn clean install
+
     - name: Check out Git repository
       uses: actions/checkout@v3
 

README.md

@@ -24,25 +24,50 @@ Additionally, you will need to supply a URL for the local FHIR store.
 
 See below for ways to get these parameters to the application.
 
+## Collections
+
+It is generally advised that you assign collections to all Specimens. This is best achieved
+using an extension to the Specimen resource, which contains a reference to a collection,
+see the FHIR profile. The IDs of these collections
+will be passed on to the Directory during synchronization.
+
+However, for some biobanks
+it may not be feasible to assign individual specimens to a collection. A fallback
+solution is available in this case: the default collection ID. This is a single ID
+that will be assigned to all specimens without collection references. There are two possible
+ways to specify this ID: either as environment variable (see DS_DIRECTORY_DEFAULT_COLLECTION_ID
+in the table below) or as an orphan collection in FHIR that has no references from
+any specimens but whose identifier is a collection ID. 
+
 ## Usage with Docker
 
 We recommend using Docker for running Directory sync.
 
 First, you will need to set up the environment variables for this:
 
-|Variable              |Purpose                                           |Default if not specified          |
-|:---------------------|:-------------------------------------------------|:---------------------------------|
-|DS_DIRECTORY_URL      |Base URL of the Directory                         |https://bbmritestnn.gcc.rug.nl    |
-|DS_DIRECTORY_USER_NAME|User name for logging in to Directory             |                                  |
-|DS_DIRECTORY_USER_PASS|Password for logging in to Directory              |                                  |
-|DS_FHIR_STORE_URL     |URL for FHIR store                                |http://store:8080/fhir            |
-|DS_TIMER_CRON         |Execution interval for Directory sync, cron format|                                  |
-|DS_RETRY_MAX          |Maximum number of retries when sync fails         |10                                |
-|DS_RETRY_INTERVAL     |Interval between retries (seconds)                |20 seconds                        |
+|Variable                          |Purpose                                               |Default if not specified          |
+|:---------------------------------|:-----------------------------------------------------|:---------------------------------|
+|DS_DIRECTORY_URL                  |Base URL of the Directory                             |https://directory.bbmri-eric.eu   |
+|DS_DIRECTORY_USER_NAME            |User name for logging in to Directory                 |                                  |
+|DS_DIRECTORY_USER_PASS            |Password for logging in to Directory                  |                                  |
+|DS_DIRECTORY_DEFAULT_COLLECTION_ID|ID of collection to be used if not in samples         |                                  |
+|DS_DIRECTORY_MIN_DONORS           |Minimum number of donors per star model hypercube     |10                                |
+|DS_DIRECTORY_MAX_FACTS            |Max number of star model hypercubes to be generated   |                                  |
+|DS_DIRECTORY_ALLOW_STAR_MODEL     |Set to 'True' to send star model info to Directory    |False                             |
+|DS_FHIR_STORE_URL                 |URL for FHIR store                                    |http://bridgehead-bbmri-blaze:8080|
+|DS_TIMER_CRON                     |Execution interval for Directory sync, cron format    |                                  |
+|DS_RETRY_MAX                      |Maximum number of retries when sync fails             |10                                |
+|DS_RETRY_INTERVAL                 |Interval between retries (seconds)                    |20 seconds                        |
 
 DS_DIRECTORY_USER_NAME and DS_DIRECTORY_USER_PASS are mandatory. If you do not specify these,
 Directory sync will not run. Contact [Directory admin](directory@helpdesk.bbmri-eric.eu) to get login credentials.
 
+DS_DIRECTORY_DEFAULT_COLLECTION_ID is not mandatory, but you will need to specify it if your ETL does not assign Directory collection IDs to specimens.
+
+If you set DS_DIRECTORY_ALLOW_STAR_MODEL to 'True', then the star model summary information
+for your data will be generated and sent to the Directory. You are advised to talk to
+your local data protection group before doing this.
+
 If DS_TIMER_CRON is not specified, Directory sync will be executed once, and then the
 process will terminate.
 

pom.xml

@@ -51,7 +51,7 @@
       <dependency>
         <groupId>de.samply</groupId>
         <artifactId>directory-sync</artifactId>
-        <version>0.2.0</version>
+        <version>0.3.0</version>
       </dependency>
 
       <dependency>

src/main/java/de/samply/directory_sync_service/Configuration.java

@@ -28,4 +28,16 @@ public class Configuration {
 
     @Value("${ds.directory.user.pass}")
     private String directoryUserPass;
+
+    @Value("${ds.directory.default_collection_id}")
+    private String directoryDefaultCollectionId;
+
+    @Value("${ds.directory.allow_star_model}")
+    private String directoryAllowStarModel;
+
+    @Value("${ds.directory.min_donors}")
+    private String directoryMinDonors;
+
+    @Value("${ds.directory.max_facts}")
+    private String directoryMaxFacts;
 }

src/main/java/de/samply/directory_sync_service/DirectorySync.java

@@ -2,22 +2,25 @@
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.rest.client.api.IGenericClient;
+import ca.uhn.fhir.rest.client.api.IRestfulClientFactory;
+import ca.uhn.fhir.rest.client.impl.RestfulClientFactory;
 import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor;
 import de.samply.directory_sync.Sync;
 import de.samply.directory_sync.directory.DirectoryApi;
 import de.samply.directory_sync.directory.DirectoryService;
 import de.samply.directory_sync.fhir.FhirApi;
 import de.samply.directory_sync.fhir.FhirReporting;
 import io.vavr.control.Either;
-import org.apache.commons.validator.routines.UrlValidator;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.hl7.fhir.r4.model.OperationOutcome;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * This class sets up connections to the FHIR store and to the Directory and
@@ -61,61 +64,84 @@ private boolean syncWithDirectory(Configuration configuration) {
         String fhirStoreUrl = configuration.getFhirStoreUrl();
         String directoryUserName = configuration.getDirectoryUserName();
         String directoryUserPass = configuration.getDirectoryUserPass();
-
-        // Give advanced warning if there are problems with the properties
-        if (directoryUrl != null && !new UrlValidator().isValid(directoryUrl))
-            logger.warn("Direcory URL is invalid: " + directoryUrl);
-        if (fhirStoreUrl != null && !new UrlValidator().isValid(fhirStoreUrl))
-            logger.warn("FHIR store URL is invalid: " + fhirStoreUrl);
+        String directoryDefaultCollectionId = configuration.getDirectoryDefaultCollectionId();
+        boolean directoryAllowStarModel = Boolean.parseBoolean(configuration.getDirectoryAllowStarModel());
+        int directoryMinDonors = Integer.parseInt(configuration.getDirectoryMinDonors());
+        int directoryMaxFacts = Integer.parseInt(configuration.getDirectoryMaxFacts());
 
         DirectoryApi directoryApi = null;
         try {
             Either<OperationOutcome, DirectoryApi> directoryApiContainer = createDirectoryApi(directoryUserName, directoryUserPass, directoryUrl);
             if (directoryApiContainer.isLeft()) {
-                logger.warn("syncWithDirectory: problem setting up Directory API: " + directoryApiContainer.getLeft().getIssue());
+                logger.error("__________ syncWithDirectory: problem setting up Directory API: " + getErrorMessageFromOperationOutcome(directoryApiContainer.getLeft()));
                 return false;
             }
             directoryApi = directoryApiContainer.get();
         } catch (IOException e) {
-            logger.warn("syncWithDirectory: createDirectoryApi failed");
-            logger.warn(e.toString());
+            logger.error("__________ syncWithDirectory: createDirectoryApi failed: " + Utils.traceFromException(e));
             return false;
         } catch (NullPointerException e) {
-            logger.warn("syncWithDirectory: createDirectoryApi failed");
-            logger.warn(e.toString());
+            logger.error("__________ syncWithDirectory: createDirectoryApi failed: " + Utils.traceFromException(e));
             return false;
         }
         DirectoryService directoryService = new DirectoryService(directoryApi);
         FhirApi fhirApi = createFhirApi(fhirStoreUrl);
         FhirReporting fhirReporting = new FhirReporting(ctx, fhirApi);
         Sync sync = new Sync(fhirApi, fhirReporting, directoryApi, directoryService);
-        Either<String, Void> result = sync.initResources();
-        List<OperationOutcome> operationOutcomes = sync.syncCollectionSizesToDirectory();
+        if (sync.initResources().isLeft()) {
+            logger.error("__________ syncWithDirectory: problem initializing FHIR resources");
+            return false;
+        }
+        List<OperationOutcome> operationOutcomes;
+        operationOutcomes = sync.generateDiagnosisCorrections(directoryDefaultCollectionId);
         for (OperationOutcome operationOutcome : operationOutcomes) {
-            List<OperationOutcome.OperationOutcomeIssueComponent> issues = operationOutcome.getIssue();
-            for (OperationOutcome.OperationOutcomeIssueComponent issue: issues) {
-                OperationOutcome.IssueSeverity severity = issue.getSeverity();
-                if (severity == OperationOutcome.IssueSeverity.ERROR || severity == OperationOutcome.IssueSeverity.FATAL) {
-                    logger.warn("syncWithDirectory: there was a problem during sync to Directory: " + issue.getDiagnostics());
+            String errorMessage = getErrorMessageFromOperationOutcome(operationOutcome);
+            if (errorMessage.length() > 0) {
+                logger.error("__________ syncWithDirectory: there was a problem during diagnosis corrections: " + errorMessage);
+                return false;
+            }
+        }
+        if (directoryAllowStarModel) {
+            operationOutcomes = sync.sendStarModelUpdatesToDirectory(directoryDefaultCollectionId, directoryMinDonors, directoryMaxFacts);
+            for (OperationOutcome operationOutcome : operationOutcomes) {
+                String errorMessage = getErrorMessageFromOperationOutcome(operationOutcome);
+                if (errorMessage.length() > 0) {
+                    logger.error("__________ syncWithDirectory: there was a problem during star model update to Directory: " + errorMessage);
                     return false;
                 }
             }
         }
-        operationOutcomes = sync.updateAllBiobanksOnFhirServerIfNecessary();
+        operationOutcomes = sync.sendUpdatesToDirectory(directoryDefaultCollectionId);
         for (OperationOutcome operationOutcome : operationOutcomes) {
-            List<OperationOutcome.OperationOutcomeIssueComponent> issues = operationOutcome.getIssue();
-            for (OperationOutcome.OperationOutcomeIssueComponent issue: issues) {
-                OperationOutcome.IssueSeverity severity = issue.getSeverity();
-                if (severity == OperationOutcome.IssueSeverity.ERROR || severity == OperationOutcome.IssueSeverity.FATAL) {
-                    logger.warn("syncWithDirectory: there was a problem during sync from Directory: " + issue.getDiagnostics());
-                    return false;
-                }
+            String errorMessage = getErrorMessageFromOperationOutcome(operationOutcome);
+            if (errorMessage.length() > 0) {
+                logger.error("__________ syncWithDirectory: there was a problem during sync to Directory: " + errorMessage);
+                return false;
             }
         }
+       operationOutcomes = sync.updateAllBiobanksOnFhirServerIfNecessary();
+       for (OperationOutcome operationOutcome : operationOutcomes) {
+            String errorMessage = getErrorMessageFromOperationOutcome(operationOutcome);
+            if (errorMessage.length() > 0) {
+                logger.error("__________ syncWithDirectory: there was a problem during sync from Directory: " + errorMessage);
+                return false;
+            }
+       }
 
-        logger.info("syncWithDirectory: Directory synchronization completed successfully");
+       logger.info("__________ syncWithDirectory: completed");
+       return true;
+    }
+
+    private String getErrorMessageFromOperationOutcome(OperationOutcome operationOutcome) {
+        String errorMessage = "";
+        List<OperationOutcome.OperationOutcomeIssueComponent> issues = operationOutcome.getIssue();
+        for (OperationOutcome.OperationOutcomeIssueComponent issue: issues) {
+            OperationOutcome.IssueSeverity severity = issue.getSeverity();
+            if (severity == OperationOutcome.IssueSeverity.ERROR || severity == OperationOutcome.IssueSeverity.FATAL)
+                errorMessage += issue.getDiagnostics() + "\n";
+        }
 
-        return true;
+        return errorMessage;
     }
 
     /**
@@ -144,6 +170,7 @@ private Either<OperationOutcome, DirectoryApi> createDirectoryApi(String directo
     private FhirApi createFhirApi(String fhirStoreUrl) {
         IGenericClient client = ctx.newRestfulGenericClient(fhirStoreUrl);
         client.registerInterceptor(new LoggingInterceptor(true));
+
         return new FhirApi(client);
     }
 }

src/main/java/de/samply/directory_sync_service/DirectorySyncJob.java

@@ -6,13 +6,17 @@
 import org.quartz.JobDataMap;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
+import org.quartz.StatefulJob;
+import org.quartz.DisallowConcurrentExecution;
 
 /**
  * Job for starting a synchronization with the Directory.
  *
  * Can handle single-run or repeated run operations.
  */
-public class DirectorySyncJob implements Job {
+@DisallowConcurrentExecution
+//public class DirectorySyncJob implements Job {
+public class DirectorySyncJob implements StatefulJob  {
     private static Logger logger = LogManager.getLogger(DirectorySyncJob.class);
 
     /**

src/main/java/de/samply/directory_sync_service/Utils.java

@@ -0,0 +1,21 @@
+package de.samply.directory_sync_service;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+/**
+ * Parameters and functions used throughout the code.
+ */
+public class Utils {
+    /**
+     * Get a printable stack trace from an Exception object.
+     * @param e
+     * @return
+     */
+    public static String traceFromException(Exception e) {
+        StringWriter sw = new StringWriter();
+        PrintWriter pw = new PrintWriter(sw);
+        e.printStackTrace(pw);
+        return sw.toString();
+    }
+}

src/main/resources/application.yml

@@ -12,5 +12,9 @@ ds:
     user:
       name: ""
       pass: ""
+    default_collection_id:
+    allow_star_model: "False"
+    min_donors: "10"
+    max_facts: "-1"