Skip to content

Commit

Permalink
refactor: fix tests and change directory logic
Browse files Browse the repository at this point in the history
  • Loading branch information
p.delpy@dkfz-heidelberg.de committed May 15, 2024
1 parent de0ce5c commit 4481e52
Show file tree
Hide file tree
Showing 27 changed files with 7,726 additions and 471 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
.idea/*
/clinical_data/*
/clinical_data/**/*.xml
target
*.properties
docker-compose.override.yml
3 changes: 0 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,6 @@ Set the environment variables either directly in Java oder via docker compose (s
* ```FILE_PATH``` defines the directory of the clinical oBDS data in the docker container.
There souldn't be a reason to change this for docker.

* ```FILE_TYPE``` defines the data type of the clinical data (oBDS|ADT_GEKID).
This is an optional parameter to increase performance. if not set, each file will be checked individually.

* ```STORE_PATH``` defines the URL of the FHIR server API.
You can use the default value, when using the default BLAZE server (https://github.com/samply/blaze).

Expand Down
1 change: 1 addition & 0 deletions clinical_data/InputData/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
!.gitignore
2 changes: 0 additions & 2 deletions clinical_data/InputOBDS/.gitignore

This file was deleted.

1 change: 0 additions & 1 deletion clinical_data/Processed/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
!.gitignore
/clinical_data/Processed/*.xml
1 change: 1 addition & 0 deletions clinical_data/tmp/ADT_Patients/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
!.gitignore
1 change: 0 additions & 1 deletion clinical_data/tmp/FHIR_Patients/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
!.gitignore
/clinical_data/tmp/FHIR_Patients/FHIR*.xml
1 change: 0 additions & 1 deletion clinical_data/tmp/erroneous/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
!.gitignore
/clinical_data/tmp/erroneous/*.xml
1 change: 0 additions & 1 deletion clinical_data/tmp/oBDS_Patients/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
!.gitignore
/clinical_data/tmp/ADT_Patients/Patient*.xml
4 changes: 2 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.6.0</version>
<version>3.7.1</version>
<executions>
<execution>
<phase>package</phase>
Expand Down Expand Up @@ -119,7 +119,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.15.1</version>
<version>2.16.1</version>
<scope>test</scope>
</dependency>
</dependencies>
Expand Down
2 changes: 1 addition & 1 deletion src/docker/start.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env sh

echo "Checking required input and output directories..."
directories="InputOBDS Processed tmp tmp/oBDS_Patients tmp/FHIR_Patients tmp/erroneous"
directories="InputData Processed tmp tmp/oBDS_Patients tmp/ADT_Patients tmp/FHIR_Patients tmp/erroneous"
for dir in $directories; do
if [ -d "/obds2fhir/clinical_data/$dir" ]; then
echo "$dir exists"
Expand Down
121 changes: 47 additions & 74 deletions src/main/java/de/samply/obds2fhir/Obds2fhir.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,9 @@


public class Obds2fhir {
private static final String INPUT_oBDS ="/InputOBDS/";
private static final String INPUT_DATA ="/InputData/";
private static final String oBDS_PATIENTS ="/tmp/oBDS_Patients/";
private static final String ADT_PATIENTS ="/tmp/ADT_Patients/";
private static final String FHIR_PATIENTS="/tmp/FHIR_Patients/";
private static final String ERRONEOUS="/tmp/erroneous";
private static final String PROCESSED="/Processed/";
Expand All @@ -39,7 +40,6 @@ public class Obds2fhir {
private static final String ANSI_RED = "\u001B[31m";
private static final String ANSI_GREEN = "\u001B[32m";
private static final String DONE = ANSI_GREEN+"...done "+ANSI_RESET;
private static final String FILE_TYPE = System.getenv().getOrDefault("FILE_TYPE","");
private static TransformerFactoryImpl factory = null;
public static Transformer oBDS2SinglePatientTransformer = null;
public static Transformer ADT2SinglePatientTransformer = null;
Expand All @@ -60,13 +60,14 @@ public static void main(String[] args) {

long startTime = System.nanoTime();
System.out.println("Transforming to single Patients... \n");
processXmlFiles(INPUT_oBDS,1);
processXmlFiles(INPUT_DATA,1);
long stopTime = System.nanoTime();
System.out.println(DONE+(stopTime - startTime)/1000000000+ " seconds");

startTime = System.nanoTime();
System.out.println("Transforming to FHIR... \n");
processXmlFiles(oBDS_PATIENTS, 2);
processXmlFiles(ADT_PATIENTS, 2);
stopTime = System.nanoTime();
System.out.println(DONE+(stopTime - startTime)/1000000000+ " seconds");

Expand All @@ -80,30 +81,23 @@ public static void main(String[] args) {

startTime = System.nanoTime();
System.out.println("posting fhir resources to blaze store...\n");
processXmlFiles(FHIR_PATIENTS, null, httppost,3);
processXmlFiles(FHIR_PATIENTS, httppost,3);
stopTime = System.nanoTime();
System.out.println(DONE+(stopTime - startTime)/1000000000+ " seconds");
}
}


public static void processXmlFiles(String inputData, int step){
processXmlFiles (inputData, false, null,step);
processXmlFiles (inputData, null,step);
}

public static void processXmlFiles (String inputData, Boolean transformWrittenResults, int step){
processXmlFiles (inputData, transformWrittenResults, null,step);
}

public static void processXmlFiles (String inputData, HttpPost httppost, int step){
processXmlFiles (inputData,false, httppost,step);
}
public static void processXmlFiles(String inputData, Boolean transformWrittenResults, HttpPost httppost, int step){
public static void processXmlFiles(String inputDir, HttpPost httppost, int step){
//System.out.print("load "+ filetype + " files...");
File fileDir = new File(System.getenv().getOrDefault("FILE_PATH","") + inputData);
File[] listOfFiles = fileDir.listFiles();
File absoluteInputDir = new File(System.getenv().getOrDefault("FILE_PATH","") + inputDir);
File[] listOfFiles = absoluteInputDir.listFiles();
if (listOfFiles==null){
System.out.println("ABORTING: empty "+ fileDir +" dir");
System.out.println("ABORTING: empty "+ absoluteInputDir +" dir");
}
else {
int counter=0;
Expand All @@ -113,46 +107,37 @@ public static void processXmlFiles(String inputData, Boolean transformWrittenRes
if (inputFile.isFile() & inputFile.getName().toLowerCase().endsWith(".xml")) {
counter+=1;
System.out.println("\u001B[AFile " + counter + " of " + (listOfFiles.length) + " / Filename: " + inputFile);
if (step==3){
try {
try {
String inputFileString = Files.readString(Paths.get(String.valueOf(inputFile)));
if (step==1){
Transformer transformer = identifyTransformer(inputFileString);
transformer.setParameter("customPrefix", counter);
applyXslt(inputFileString, transformer);
inputFile.renameTo(new File(System.getenv().getOrDefault("FILE_PATH","/obds2fhir/clinical_data") + PROCESSED + inputFile.getName()));
} else if (step==2){
Transformer transformer= inputDir.equals(oBDS_PATIENTS) ? oBDS2MDSTransformer : ADT2MDSTransformer;
transformer.setParameter("customPrefix", counter);
String xmlResult = applyXslt(inputFileString, transformer);
MDS2FHIRTransformer.setParameter("customPrefix", inputFile.getName());
applyXslt(xmlResult, MDS2FHIRTransformer);
inputFile.delete();
} else if (step==3){
postToFhirStore(inputFile, httppost);
} catch (IOException e) {
counter-=1;
System.out.print("ERROR - FHIR import: problem with file " + inputFile);
e.printStackTrace();
}
}
else {
//System.out.print("processing file " + inputFile.getName() + "...");
String inputFileString = null;
try {
inputFileString = Files.readString(Paths.get(String.valueOf(inputFile)));
} catch (IOException e) {
counter-=1;
System.out.print("ERROR - reading: problem with file " + inputFile);
e.printStackTrace();
}
try {
if (step==1){
Transformer transformer= identifyTransformer(inputFileString, step);
transformer.setParameter("customPrefix", counter);
applyXslt(inputFileString, transformer);
} else if (step==2){
Transformer transformer= identifyTransformer(inputFileString, step);
transformer.setParameter("customPrefix", counter);
String xmlResult = applyXslt(inputFileString, transformer);
MDS2FHIRTransformer.setParameter("customPrefix", inputFile.getName());
applyXslt(xmlResult, MDS2FHIRTransformer);
inputFile.delete();
}
else {
inputFile.renameTo(new File(System.getenv().getOrDefault("FILE_PATH","") + PROCESSED + inputFile.getName()));
}
} catch (UnsupportedEncodingException | TransformerException | RuntimeException e) {
counter-=1;
System.out.print("ERROR - transformation: problem with file " + inputFile);
//e.printStackTrace();
else {
inputFile.renameTo(new File(System.getenv().getOrDefault("FILE_PATH","") + PROCESSED + inputFile.getName()));
}
} catch (IOException e) {
counter-=1;
System.out.print("ERROR - IOException with file " + inputFile);
e.printStackTrace();
} catch (TransformerException e) {
counter-=1;
System.out.print("ERROR - TransformerException with file " + inputFile+"\n\n"+inputDir+"\n"+oBDS_PATIENTS+"\n");
e.printStackTrace();
} catch (RuntimeException e) {
counter-=1;
System.out.print("ERROR - RuntimeException with file " + inputFile);
}
}
else if (inputFile.isFile() & inputFile.getName().toLowerCase().endsWith(".gitignore")) {
Expand All @@ -165,32 +150,20 @@ else if (inputFile.isFile() & inputFile.getName().toLowerCase().endsWith(".gitig
}
}

private static Transformer identifyTransformer(String file, int step) {
private static Transformer identifyTransformer(String file) {
int fileVersion = getFileVersion(file);
if (fileVersion == 2){
if (step == 1){
return ADT2SinglePatientTransformer;
} else {
return ADT2MDSTransformer;
}
} else if (fileVersion == 3){
if (step == 1){
return oBDS2SinglePatientTransformer;
} else {
return oBDS2MDSTransformer;
}
}
return null;
return fileVersion==3 ? oBDS2SinglePatientTransformer : ADT2SinglePatientTransformer;
}

private static int getFileVersion(String xmlContent) {
int transformation = 0;
private static int getFileVersion(String xmlContent) throws IllegalArgumentException {
if (xmlContent.contains("<ADT_GEKID")) {
transformation = 2;
return 2;
} else if (xmlContent.contains("<oBDS")) {
transformation = 3;
return 3;
} else {
int maxLength = xmlContent.length() < 200 ? xmlContent.length() : 200;
throw new IllegalArgumentException("Error: File does not contain oBDS or ADT/GEKID " + xmlContent.substring(0, maxLength));
}
return transformation;
}

private static void postToFhirStore(File inputFile, HttpPost httppost) throws IOException {
Expand Down Expand Up @@ -246,7 +219,7 @@ public static void initializeTransformers(boolean pseudonymizeFlag) {
ADT2MDSTransformer.setParameter("keep_internal_id", System.getenv().getOrDefault("KEEP_INTERNAL_ID","false"));
MDS2FHIRTransformer = factory.newTransformer(new StreamSource(Obds2fhir.class.getClassLoader().getResourceAsStream("MDS2FHIR.xsl")));
MDS2FHIRTransformer.setParameter("filepath", System.getenv().getOrDefault("FILE_PATH",""));
MDS2FHIRTransformer.setParameter("identifier_system", System.getenv().getOrDefault("IDENTIFIER_SYSTEM",""));
MDS2FHIRTransformer.setParameter("identifier_system", System.getenv().getOrDefault("IDENTIFIER_SYSTEM","http://dktk.dkfz.de/fhir/onco/core/CodeSystem/PseudonymArtCS"));
} catch (TransformerConfigurationException e) {
System.out.print("Transformer configuration error");
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/resources/ADT2MDS_FHIR.xsl
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
<xsl:otherwise><DKTK_Einwilligung_erfolgt>false</DKTK_Einwilligung_erfolgt></xsl:otherwise>
</xsl:choose>
<Vitalstatus_Gesamt>
<xsl:attribute name="Vitalstatus_ID"><xsl:value-of select="concat($Patient_Id,'vital')"/></xsl:attribute>
<xsl:attribute name="Vitalstatus_ID"><xsl:value-of select="hash:hash($Patient_Id,'vital','')"/></xsl:attribute>
<xsl:choose>
<xsl:when test="Patienten_Stammdaten/Vitalstatus_Datum"><Datum_des_letztbekannten_Vitalstatus><xsl:value-of select="Patienten_Stammdaten/Vitalstatus_Datum"/></Datum_des_letztbekannten_Vitalstatus></xsl:when>
<xsl:otherwise><xsl:copy-of select="xsi:Datum_des_letztbekannten_Vitalstatus(Menge_Meldung)"/></xsl:otherwise>
Expand Down
2 changes: 1 addition & 1 deletion src/main/resources/ADT2SinglePatient.xsl
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

<xsl:template match="/ADT_GEKID">
<xsl:for-each select="Menge_Patient/Patient">
<xsl:result-document method="xml" href="file:{$filepath}/tmp/oBDS_Patients/Patient_{hash:hash(Patienten_Stammdaten/@Patient_ID,'','')}_ADT_{$customPrefix}.xml">
<xsl:result-document method="xml" href="file:{$filepath}/tmp/ADT_Patients/Patient_{hash:hash(Patienten_Stammdaten/@Patient_ID,'','')}_ADT_{$customPrefix}.xml">
<!--<xsl:result-document method="xml" href="Patient_{Patienten_Stammdaten/@Patient_ID}.xml">-->
<ADT_GEKID Schema_Version="2.2.1">
<xsl:copy-of select="/ADT_GEKID/@*" />
Expand Down
Loading

0 comments on commit 4481e52

Please sign in to comment.