Skip to content

Commit

Permalink
split studies and write metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
@nr23730
nr23730 committed Oct 29, 2021
1 parent 3f84534 commit e7c9555
Show file tree
Hide file tree
Showing 45 changed files with 224 additions and 301 deletions.
27 changes: 2 additions & 25 deletions src/main/java/de/uzl/lied/mtbimporter/MtbImporter.java
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,18 @@
package de.uzl.lied.mtbimporter;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Timer;

import de.uzl.lied.mtbimporter.jobs.CheckDropzone;
import de.uzl.lied.mtbimporter.jobs.FhirResolver;
import de.uzl.lied.mtbimporter.jobs.StudyHandler;
import de.uzl.lied.mtbimporter.jobs.mdr.centraxx.CxxMdrLogin;
import de.uzl.lied.mtbimporter.model.CbioPortalStudy;
import de.uzl.lied.mtbimporter.settings.ConfigurationLoader;
import de.uzl.lied.mtbimporter.settings.Mdr;
import de.uzl.lied.mtbimporter.settings.Settings;
import de.uzl.lied.mtbimporter.tasks.AddClinicalData;
import de.uzl.lied.mtbimporter.tasks.AddGeneticData;
import de.uzl.lied.mtbimporter.tasks.AddMetaFile;
import de.uzl.lied.mtbimporter.tasks.AddResourceData;
import de.uzl.lied.mtbimporter.tasks.AddSignatureData;
import de.uzl.lied.mtbimporter.tasks.AddTimelineData;

public final class MtbImporter {
/**
Expand All @@ -43,24 +37,7 @@ public static void main(String[] args) throws InterruptedException, IOException
}
}

CbioPortalStudy study = new CbioPortalStudy();
AddGeneticData.processMafFile(study, new File(Settings.getStudyFolder() + Settings.getState() + "/data_mutation_extended.maf"));
AddGeneticData.processSegFile(study, new File(Settings.getStudyFolder() + Settings.getState() + "/data_cna.seg"));
AddGeneticData.processGenePanelFile(study, new File(Settings.getStudyFolder() + Settings.getState() + "/data_gene_panel_matrix.txt"));
AddClinicalData.processClinicalPatient(study, new File(Settings.getStudyFolder() + Settings.getState() + "/data_clinical_patient.txt"));
AddClinicalData.processClinicalSample(study, new File(Settings.getStudyFolder() + Settings.getState() + "/data_clinical_sample.txt"));
study.setSampleResources(AddResourceData.readResourceFile(new File(Settings.getStudyFolder() + Settings.getState() + "/data_resource_sample.txt")));
AddSignatureData.processContribution(study, new File(Settings.getStudyFolder() + Settings.getState() + "/data_mutational_signature_contribution.txt"));
AddSignatureData.processLimit(study, new File(Settings.getStudyFolder() + Settings.getState() + "/data_mutational_signature_limit.txt"));
AddGeneticData.processCnaFile(study, new File(Settings.getStudyFolder() + Settings.getState() + "/data_cna.txt"));
File[] timelines = new File(Settings.getStudyFolder() + Settings.getState()).listFiles((File f) -> f.getName().startsWith("data_timeline"));
File[] metaFiles = new File(Settings.getStudyFolder() + Settings.getState()).listFiles((File f) -> f.getName().startsWith("meta_"));
for(File t : timelines) {
AddTimelineData.processTimelineFile(study, t);
}
for(File m : metaFiles) {
AddMetaFile.processMetaFile(study, m);
}
CbioPortalStudy study = StudyHandler.load(Settings.getMainStudyId());

Timer t = new Timer();
CheckDropzone checkDropzone = new CheckDropzone(study);
Expand Down
95 changes: 63 additions & 32 deletions src/main/java/de/uzl/lied/mtbimporter/jobs/CheckDropzone.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,13 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.TimerTask;
import java.util.Map.Entry;
import java.util.concurrent.ExecutionException;

import org.apache.commons.io.FileUtils;
Expand All @@ -15,6 +20,7 @@
import de.samply.common.mdrclient.MdrConnectionException;
import de.samply.common.mdrclient.MdrInvalidResponseException;
import de.uzl.lied.mtbimporter.model.CbioPortalStudy;
import de.uzl.lied.mtbimporter.model.ClinicalPatient;
import de.uzl.lied.mtbimporter.settings.InputFolder;
import de.uzl.lied.mtbimporter.settings.Settings;
import de.uzl.lied.mtbimporter.tasks.AddGeneticData;
Expand All @@ -35,7 +41,6 @@ public CheckDropzone(CbioPortalStudy study) {
@Override
public void run() {

Long oldState = 0L;
Long newState = System.currentTimeMillis();
int count = 0;
CbioPortalStudy newStudy = new CbioPortalStudy();
Expand All @@ -44,7 +49,7 @@ public void run() {
for (InputFolder inputfolder : Settings.getInputFolders()) {
File[] files = new File(inputfolder.getSource()).listFiles();

if (files.length == 0) {
if (files.length == 0 || (files.length == 1 && files[0].getName().equals(".gitkeep"))) {
System.out.println("No new files at " + inputfolder.getSource() + ".");
continue;
} else {
Expand All @@ -62,37 +67,40 @@ public void run() {
}
}
for (File f : files) {
if(f.getName().equals(".gitkeep")) {
continue;
}
System.out.println("Found " + f.getAbsolutePath());
try {
switch (FilenameUtils.getExtension(f.getName())) {
case "maf":
AddGeneticData.processMafFile(newStudy, f);
break;
case "txt":
switch (determineDatatype(f)) {
case "cna":
AddGeneticData.processCnaFile(newStudy, f);
break;
case "mutsigLimit":
AddSignatureData.processLimit(newStudy, f);
break;
case "mutsigContribution":
AddSignatureData.processContribution(newStudy, f);
break;
}
break;
case "seg":
AddGeneticData.processSegFile(newStudy, f);
break;
case "pdf":
AddResourceData.processPdfFile(newStudy, f);
case "maf":
AddGeneticData.processMafFile(newStudy, f);
break;
case "txt":
switch (determineDatatype(f)) {
case "cna":
AddGeneticData.processCnaFile(newStudy, f);
break;
case "RData":
AddMetaData.processRData(newStudy, f);
case "mutsigLimit":
AddSignatureData.processLimit(newStudy, f);
break;
case "csv":
AddHisData.processCsv(newStudy, f);
case "mutsigContribution":
AddSignatureData.processContribution(newStudy, f);
break;
}
break;
case "seg":
AddGeneticData.processSegFile(newStudy, f);
break;
case "pdf":
AddResourceData.processPdfFile(newStudy, f);
break;
case "RData":
AddMetaData.processRData(newStudy, f);
break;
case "csv":
AddHisData.processCsv(newStudy, f);
break;
}

if (inputfolder.getTarget() == null || inputfolder.getTarget().length() == 0) {
Expand All @@ -112,14 +120,37 @@ public void run() {

if (count > 0) {
try {
oldState = Settings.getState();
FileUtils.copyDirectory(new File(Settings.getStudyFolder() + oldState),
new File(Settings.getStudyFolder() + newState));
FileUtils.copyDirectory(new File(Settings.getStudyFolder() + study.getStudyId() + "/" + study.getState()),
new File(Settings.getStudyFolder() + study.getStudyId() + "/" + newState));

StudyHandler.merge(study, newStudy);
StudyHandler.write(study, newState);
ImportStudy.importStudy(newState, Settings.getOverrideWarnings());
Settings.setState(newState);
ImportStudy.importStudy(study.getStudyId(), newState, Settings.getOverrideWarnings());
study.setState(newState);

Map<String, List<String>> patientsByDate = new HashMap<String, List<String>>();
Map<String, CbioPortalStudy> patientStudy = new HashMap<String, CbioPortalStudy>();
for (ClinicalPatient patient : newStudy.getPatients()) {
if (patient.getAdditionalAttributes().containsKey("PRESENTATION_DATE")) {
List<String> al = patientsByDate.getOrDefault(
(String) patient.getAdditionalAttributes().get("PRESENTATION_DATE"),
new ArrayList<String>());
al.add(patient.getPatientId());
patientsByDate.put((String) patient.getAdditionalAttributes().get("PRESENTATION_DATE"), al);
patientStudy.put(patient.getPatientId(), StudyHandler.getPatientStudy(study, patient.getPatientId()));
}
}
for(Entry<String, List<String>> e : patientsByDate.entrySet()) {
CbioPortalStudy s = StudyHandler.load(study.getStudyId() + "_" + e.getKey());
s.getMetaFile("meta_study.txt").setAdditionalAttributes("name", e.getKey() + " " + s.getMetaFile("meta_study.txt").getAdditionalAttributes().get("name"));
s.getMetaFile("meta_study.txt").setAdditionalAttributes("short_name", e.getKey() + " " + s.getMetaFile("meta_study.txt").getAdditionalAttributes().get("short_name"));
s.getMetaFile("meta_study.txt").setAdditionalAttributes("description", e.getKey() + " " + s.getMetaFile("meta_study.txt").getAdditionalAttributes().get("description"));
for(String patient : e.getValue()) {
StudyHandler.merge(s, patientStudy.get(patient));
}
StudyHandler.write(s, s.getState());
ImportStudy.importStudy(s.getStudyId(), s.getState(), Settings.getOverrideWarnings());
}
} catch (IOException | InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
Expand Down
Loading

0 comments on commit e7c9555

Please sign in to comment.