Skip to content

Commit

Permalink
Added support for PART25 and PCB datasets
Browse files Browse the repository at this point in the history
  • Loading branch information
dbeaudoinfortin committed May 27, 2024
1 parent ebf6c52 commit a046166
Showing 5 changed files with 13 additions and 9 deletions.
1 change: 1 addition & 0 deletions src/main/java/com/dbf/naps/data/loader/NAPSDataLoader.java
Original file line number Diff line number Diff line change
@@ -45,6 +45,7 @@ protected void run() {
}

log.info("Goodbye! 🙂");
System.exit(0);
}

private void initDB() throws IOException, SQLException {
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@
import java.io.File;
import java.util.List;
import com.dbf.naps.data.loader.NAPSDataLoader;
import com.dbf.naps.data.loader.integrated.runner.DICHFileLoadRunner;
import com.dbf.naps.data.loader.integrated.runner.CFFileLoadRunner;
import com.dbf.naps.data.loader.integrated.runner.SampleMetaDataFileLoadRunner;

public class NAPSIntegratedDataLoader extends NAPSDataLoader {
@@ -26,9 +26,10 @@ protected List<Class<?>> getDBMappers() {
protected Runnable processFile(File dataFile) {

String fileName = dataFile.getName().toUpperCase();
if(fileName.endsWith("_DICH.XLS")) {
return new DICHFileLoadRunner(getThreadID(), getOptions(), getSqlSessionFactory(), dataFile);
} else if(fileName.endsWith("_PAH.XLS") || fileName.endsWith("_HCB.XLS") || fileName.endsWith("_VOC.XLS") || fileName.endsWith("_PCDD.XLS")) {
if(fileName.endsWith("_DICH.XLS") || fileName.endsWith("_PART25.XLS")) {
return new CFFileLoadRunner(getThreadID(), getOptions(), getSqlSessionFactory(), dataFile);
} else if(fileName.endsWith("_PAH.XLS") || fileName.endsWith("_HCB.XLS") || fileName.endsWith("_VOC.XLS")
|| fileName.endsWith("_PCDD.XLS") || fileName.endsWith("_PCB.XLS")) {
return new SampleMetaDataFileLoadRunner(getThreadID(), getOptions(), getSqlSessionFactory(), dataFile);
}
throw new IllegalArgumentException("Unsupported data file: " + dataFile);
Original file line number Diff line number Diff line change
@@ -10,11 +10,11 @@
import com.dbf.naps.data.utilities.DataCleaner;

/**
* Extends the base IntegratedFileLoadRunner class to add support for DICHOT specific metadata.
* Extends the base IntegratedFileLoadRunner class to add support for DICHOT & PART25 specific metadata.
*/
public class DICHFileLoadRunner extends IntegratedFileLoadRunner {
public class CFFileLoadRunner extends IntegratedFileLoadRunner {

public DICHFileLoadRunner(int threadId, LoaderOptions config, SqlSessionFactory sqlSessionFactory, File rawFile) {
public CFFileLoadRunner(int threadId, LoaderOptions config, SqlSessionFactory sqlSessionFactory, File rawFile) {
super(threadId, config, sqlSessionFactory, rawFile);
}

Original file line number Diff line number Diff line change
@@ -46,6 +46,8 @@ public class IntegratedFileLoadRunner extends FileLoadRunner {
DEFAULT_IGNORED_HEADERS.add("CANISTER"); //Canister ID#
DEFAULT_IGNORED_HEADERS.add("START"); //Start Time
DEFAULT_IGNORED_HEADERS.add("DURATION"); //Duration
DEFAULT_IGNORED_HEADERS.add("SUM"); //Sum PCB TEQ*
DEFAULT_IGNORED_HEADERS.add("FIELD"); //Field ID
}

public IntegratedFileLoadRunner(int threadId, LoaderOptions config, SqlSessionFactory sqlSessionFactory, File rawFile) {
@@ -185,15 +187,15 @@ protected List<IntegratedDataRecord> processRow(int row, Date date) {
private boolean isColumnIgnored(String columnHeader) {
columnHeader = columnHeader.toUpperCase();
for(String ignoredHeader : getIgnoredColumnList()) {
if (columnHeader.startsWith(ignoredHeader)) return true;
if (columnHeader.startsWith(ignoredHeader) || columnHeader.endsWith(ignoredHeader)) return true;
}
return false;
}

/**
* Returns a List of Strings that contain all of the column headers for columns that should be ignored.
* All entries must be in upper case.
* All entries will be compared with a startsWith().
* All entries will be compared with a startsWith() and endsWith().
* Blank columns headers are automatically ignored and should not be included.
* For the default list of ignored headers see DEFAULT_IGNORED_HEADERS
*/
Binary file modified target/naps_data.jar
Binary file not shown.

0 comments on commit a046166

Please sign in to comment.