Skip to content

Commit

Permalink
Merge bulk upload fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@hplahar
hplahar committed Aug 12, 2015
2 parents f759c80 + 8e6c69e commit 95f4b76
Show file tree
Hide file tree
Showing 17 changed files with 293 additions and 138 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<groupId>org.jbei</groupId>
<artifactId>ice</artifactId>
<packaging>war</packaging>
<version>4.3.3</version>
<version>4.3.4</version>
<name>ice</name>
<description>Inventory of Composable Elements (ICE) for Synthetic Biology</description>
<repositories>
Expand Down
176 changes: 104 additions & 72 deletions src/main/java/org/jbei/ice/lib/bulkupload/BulkCSVUpload.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,24 @@
import org.apache.commons.io.LineIterator;
import org.apache.commons.lang.StringUtils;
import org.jbei.ice.lib.common.logging.Logger;
import org.jbei.ice.lib.dto.StorageLocation;
import org.jbei.ice.lib.dto.bulkupload.EntryField;
import org.jbei.ice.lib.dto.bulkupload.SampleField;
import org.jbei.ice.lib.dto.entry.AttachmentInfo;
import org.jbei.ice.lib.dto.entry.EntryType;
import org.jbei.ice.lib.dto.entry.PartData;
import org.jbei.ice.lib.dto.sample.PartSample;
import org.jbei.ice.lib.dto.sample.SampleType;
import org.jbei.ice.lib.entry.EntryUtil;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;

/**
* Helper class for dealing with bulk CSV uploads
Expand Down Expand Up @@ -49,7 +56,7 @@ public BulkCSVUpload(String userId, Path csvFilePath, EntryType addType) {
*/
public final long processUpload() throws IOException {
try (FileInputStream inputStream = new FileInputStream(csvFilePath.toFile())) {
List<PartData> updates = getBulkUploadDataFromFile(inputStream);
List<PartWithSample> updates = getBulkUploadDataFromFile(inputStream);

// create actual entries
BulkEntryCreator creator = new BulkEntryCreator();
Expand All @@ -68,41 +75,6 @@ public final long processUpload() throws IOException {
}
}

protected String validate(List<BulkUploadAutoUpdate> updates) {
for (BulkUploadAutoUpdate update : updates) {
ArrayList<EntryField> toValidate = new ArrayList<>(requiredFields);

for (Map.Entry<EntryField, String> entry : update.getKeyValue().entrySet()) {
EntryField entryField = entry.getKey();
String value = entry.getValue();

if (!requiredFields.contains(entryField))
continue;

toValidate.remove(entryField);

if (StringUtils.isBlank(value)) {
return "Error: \"" + entryField.toString() + "\" is a required field.";
}
}

if (!toValidate.isEmpty()) {
StringBuilder builder = new StringBuilder();
builder.append("Error: File is missing the following required fields [");
int i = 0;
for (EntryField field : toValidate) {
if (i > 0)
builder.append(",");
builder.append(field.toString());
i += 1;
}
builder.append("]");
return builder.toString();
}
}
return null;
}

EntryType detectSubType(String field) {
String[] fieldNames = field.split("\\s+");
return EntryType.nameToType(fieldNames[0]);
Expand All @@ -117,7 +89,7 @@ HeaderValue detectSubTypeHeaderValue(EntryType subType, String fieldStr) throws
throw new IOException("Unknown field [" + fieldStr + "] for upload [" + addType.getDisplay() + "]");
}

return new HeaderValue(true, field);
return new EntryHeaderValue(true, field);
}

/**
Expand All @@ -138,7 +110,15 @@ HashMap<Integer, HeaderValue> processColumnHeaders(String[] headerArray) throws
EntryField field = EntryField.fromString(fieldStr);
if (field != null) {
// field header maps as is to EntryField which indicates it is not a sub Type
HeaderValue headerValue = new HeaderValue(false, field);
HeaderValue headerValue = new EntryHeaderValue(false, field);
headers.put(i, headerValue);
continue;
}

// check if sample field
SampleField sampleField = SampleField.fromString(fieldStr);
if (sampleField != null) {
HeaderValue headerValue = new SampleHeaderValue(sampleField);
headers.put(i, headerValue);
continue;
}
Expand All @@ -160,12 +140,15 @@ HashMap<Integer, HeaderValue> processColumnHeaders(String[] headerArray) throws
headers.put(i, headerValue);
}

if (headers.size() != headerArray.length) {
throw new IOException("Header size and input header array length differ");
}
return headers;
}

// NOTE: this also validates the part data (with the exception of the actual files)
List<PartData> getBulkUploadDataFromFile(InputStream inputStream) throws IOException {
List<PartData> partDataList = new LinkedList<>();
List<PartWithSample> getBulkUploadDataFromFile(InputStream inputStream) throws IOException {
List<PartWithSample> partDataList = new LinkedList<>();

// initialize parser to null; when not-null in the loop below, then the header has been parsed
CSVParser parser = null;
Expand Down Expand Up @@ -198,54 +181,69 @@ List<PartData> getBulkUploadDataFromFile(InputStream inputStream) throws IOExcep
if (StringUtils.isBlank(line) || line.replaceAll(",", "").trim().isEmpty())
continue;

// at this point we must have headers since that should be the first item in the file
if (headers == null)
throw new IOException("Could not parse file headers");

// parser != null; process line contents with available headers
String[] valuesArray = parser.parseLine(line);
PartData partData = new PartData(addType);
PartSample partSample = null;

if (subType != null) {
partData.getLinkedParts().add(new PartData(subType));
}

// for each column
for (int i = 0; i < valuesArray.length; i += 1) {
HeaderValue headerForColumn = headers.get(i);
EntryField field = headerForColumn.getEntryField();
PartData data;
String value = valuesArray[i];
boolean isSubType = headerForColumn.isSubType();

if (isSubType)
data = partData.getLinkedParts().get(0);
else
data = partData;

switch (field) {
case ATT_FILENAME:
ArrayList<AttachmentInfo> attachments = data.getAttachments();
if (attachments == null) {
attachments = new ArrayList<>();
}
attachments.clear();
attachments.add(new AttachmentInfo(value));
break;

case SEQ_FILENAME:
data.setSequenceFileName(value);
break;

case SEQ_TRACE_FILES:
// todo
break;

default:
partData = EntryUtil.setPartDataFromField(partData, value, field, isSubType);
if (headerForColumn.isSampleField()) {
// todo : move to another method
if (partSample == null)
partSample = new PartSample();
setPartSampleData(((SampleHeaderValue) headerForColumn).getSampleField(),
partSample, valuesArray[i]);

} else {
EntryHeaderValue entryHeaderValue = (EntryHeaderValue) headerForColumn;
EntryField field = entryHeaderValue.getEntryField();
PartData data;
String value = valuesArray[i];
boolean isSubType = entryHeaderValue.isSubType();

if (isSubType)
data = partData.getLinkedParts().get(0);
else
data = partData;

// get the data for the field
switch (field) {
case ATT_FILENAME:
ArrayList<AttachmentInfo> attachments = data.getAttachments();
if (attachments == null) {
attachments = new ArrayList<>();
}
attachments.clear();
attachments.add(new AttachmentInfo(value));
break;

case SEQ_FILENAME:
data.setSequenceFileName(value);
break;

case SEQ_TRACE_FILES:
// todo
break;

default:
partData = EntryUtil.setPartDataFromField(partData, value, field, isSubType);
}
}
}

// validate
List<EntryField> fields = EntryUtil.validates(partData);
List<EntryField> fields = validate(partData);
if (!fields.isEmpty()) {
StringBuilder fieldsString = new StringBuilder();
for (EntryField field : fields) {
Expand All @@ -255,7 +253,8 @@ List<PartData> getBulkUploadDataFromFile(InputStream inputStream) throws IOExcep
}

partData.setIndex(index);
partDataList.add(partData);
PartWithSample partWithSample = new PartWithSample(partSample, partData);
partDataList.add(partWithSample);
index += 1;
}
} finally {
Expand All @@ -264,4 +263,37 @@ List<PartData> getBulkUploadDataFromFile(InputStream inputStream) throws IOExcep

return partDataList;
}

protected List<EntryField> validate(PartData partData) {
return EntryUtil.validates(partData);
}

protected void setPartSampleData(SampleField sampleField, PartSample partSample, String data) {
switch (sampleField) {
case LABEL:
partSample.setLabel(data);
break;

case SHELF:
StorageLocation storageLocation = new StorageLocation();
storageLocation.setType(SampleType.SHELF);
storageLocation.setDisplay(data);
partSample.setLocation(storageLocation);
break;

case BOX:
StorageLocation childLocation = new StorageLocation();
childLocation.setDisplay(data);
childLocation.setType(SampleType.BOX_INDEXED);
partSample.getLocation().setChild(childLocation);
break;

case WELL:
StorageLocation grandChildLocation = new StorageLocation();
grandChildLocation.setType(SampleType.WELL);
grandChildLocation.setDisplay(data);
partSample.getLocation().getChild().setChild(grandChildLocation);
break;
}
}
}
28 changes: 21 additions & 7 deletions src/main/java/org/jbei/ice/lib/bulkupload/BulkEntryCreator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,15 @@
import org.jbei.ice.lib.dto.entry.EntryType;
import org.jbei.ice.lib.dto.entry.PartData;
import org.jbei.ice.lib.dto.entry.Visibility;
import org.jbei.ice.lib.dto.sample.PartSample;
import org.jbei.ice.lib.entry.EntryController;
import org.jbei.ice.lib.entry.EntryCreator;
import org.jbei.ice.lib.entry.EntryEditor;
import org.jbei.ice.lib.entry.EntryFactory;
import org.jbei.ice.lib.entry.attachment.Attachment;
import org.jbei.ice.lib.entry.model.Entry;
import org.jbei.ice.lib.entry.model.Strain;
import org.jbei.ice.lib.entry.sample.SampleController;
import org.jbei.ice.lib.entry.sequence.SequenceController;
import org.jbei.ice.lib.models.Sequence;
import org.jbei.ice.lib.utils.Utils;
Expand Down Expand Up @@ -409,19 +411,24 @@ protected void addWritePermission(Account account, Entry entry) {
DAOFactory.getPermissionDAO().create(permission);
}

public boolean createEntries(String userId, long draftId, List<PartData> data, HashMap<String, InputStream> files) {
public boolean createEntries(String userId, long draftId, List<PartWithSample> data, HashMap<String, InputStream> files) {
BulkUpload draft = dao.get(draftId);
if (draft == null)
return false;

// check permissions
authorization.expectWrite(userId, draft);
SampleController sampleController = new SampleController();

for (PartData datum : data) {
if (datum == null)
for (PartWithSample partWithSample : data) {
if (partWithSample == null)
continue;

PartData partData = partWithSample.getPartData();
if (partData == null)
continue;

Entry entry = InfoToModelFactory.infoToEntry(datum);
Entry entry = InfoToModelFactory.infoToEntry(partData);
if (entry == null)
continue;

Expand All @@ -431,9 +438,9 @@ public boolean createEntries(String userId, long draftId, List<PartData> data, H
entry.setOwnerEmail(account.getEmail());

// check if there is any linked parts. create if so (expect a max of 1)
if (datum.getLinkedParts() != null && datum.getLinkedParts().size() > 0) {
if (partData.getLinkedParts() != null && partData.getLinkedParts().size() > 0) {
// create linked
PartData linked = datum.getLinkedParts().get(0);
PartData linked = partData.getLinkedParts().get(0);
Entry linkedEntry = InfoToModelFactory.infoToEntry(linked);
if (linkedEntry != null) {
linkedEntry.setVisibility(Visibility.DRAFT.getValue());
Expand Down Expand Up @@ -473,7 +480,14 @@ public boolean createEntries(String userId, long draftId, List<PartData> data, H
dao.update(draft);

// save files
saveFiles(datum, entry, files);
saveFiles(partData, entry, files);

// save sample, if available
PartSample partSample = partWithSample.getPartSample();
if (partSample == null)
continue;

sampleController.createSample(userId, entry.getId(), partSample, null);
}

return true;
Expand Down
18 changes: 9 additions & 9 deletions src/main/java/org/jbei/ice/lib/bulkupload/BulkZipUpload.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
package org.jbei.ice.lib.bulkupload;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.jbei.ice.lib.common.logging.Logger;
import org.jbei.ice.lib.dto.entry.EntryType;
import org.jbei.ice.lib.dto.entry.PartData;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
Expand All @@ -10,13 +16,6 @@
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import org.jbei.ice.lib.common.logging.Logger;
import org.jbei.ice.lib.dto.entry.EntryType;
import org.jbei.ice.lib.dto.entry.PartData;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;

/**
* Bulk Upload with zip files. It is expected that the zip contains a csv
* of the upload with the attachment and sequences files containing
Expand Down Expand Up @@ -86,12 +85,13 @@ public long processUpload() throws IOException {
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(csvFile.getBytes())) {

// retrieve the partData and validates
List<PartData> updates = csvUpload.getBulkUploadDataFromFile(inputStream);
List<PartWithSample> updates = csvUpload.getBulkUploadDataFromFile(inputStream);

// validate files to ensure that for each partData with a file, that the file is available
for (PartData data : updates) {
for (PartWithSample partWithSample : updates) {

// check sequences
PartData data = partWithSample.getPartData();
String sequenceFile = data.getSequenceFileName();
if (StringUtils.isNotBlank(sequenceFile) && files.get(sequenceFile) == null)
throw new Exception("Sequence file \"" + sequenceFile + "\" not found in the zip archive");
Expand Down
Loading

0 comments on commit 95f4b76

Please sign in to comment.