From 4d7097170d7e1399f751846e3dd6de07c853fc2f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 23 Mar 2020 13:37:31 -0400 Subject: [PATCH 001/354] initial semantic API endpoint --- .../harvard/iq/dataverse/api/Datasets.java | 233 +++++++++++++++++- .../iq/dataverse/util/bagit/OREMap.java | 1 - .../iq/dataverse/util/json/JSONLDUtil.java | 27 ++ .../iq/dataverse/util/json/JsonLDTerm.java | 3 - 4 files changed, 256 insertions(+), 8 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 38d3979577d..d321495c929 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -78,11 +78,9 @@ import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.S3PackageImporter; -import static edu.harvard.iq.dataverse.api.AbstractApiBean.error; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; -import edu.harvard.iq.dataverse.dataaccess.DataAccess; -import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.UnforcedCommandException; import edu.harvard.iq.dataverse.engine.command.impl.GetDatasetStorageSizeCommand; @@ -99,8 +97,11 @@ import edu.harvard.iq.dataverse.util.ArchiverUtil; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.EjbUtil; -import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.json.ControlledVocabularyException; +import edu.harvard.iq.dataverse.util.json.JSONLDUtil; +import edu.harvard.iq.dataverse.util.json.JsonLDNamespace; +import edu.harvard.iq.dataverse.util.json.JsonLDTerm; import edu.harvard.iq.dataverse.util.json.JsonParseException; import edu.harvard.iq.dataverse.search.IndexServiceBean; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; @@ -119,6 +120,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.EJB; @@ -129,6 +131,9 @@ import javax.json.JsonArrayBuilder; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; +import javax.json.JsonString; +import javax.json.JsonValue; +import javax.json.stream.JsonParsingException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; @@ -551,6 +556,7 @@ public Response updateDatasetPIDMetadataAll() { @PUT @Path("{id}/versions/{versionId}") + @Consumes(MediaType.APPLICATION_JSON) public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, @PathParam("versionId") String versionId ){ if ( ! ":draft".equals(versionId) ) { @@ -599,6 +605,225 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, } } + @PUT + @Path("{id}/versions/{versionId}/metadata") + @Consumes("application/json-ld") + public Response updateVersionMetadata( String jsonLDBody, @PathParam("id") String id, @PathParam("versionId") String versionId ){ + + if ( ! ":draft".equals(versionId) ) { + return error( Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); + } + + try ( StringReader rdr = new StringReader(jsonLDBody) ) { + DataverseRequest req = createDataverseRequest(findUserOrDie()); + Dataset ds = findDatasetOrDie(id); + boolean updateDraft = ds.getLatestVersion().isDraft(); + DatasetVersion dsv = ds.getEditVersion(); + JsonObject json = Json.createReader(rdr).readObject(); + Map context = JSONLDUtil.populateContext(json.get("@context")); + + //ToDo - check that the context and localContext align w.r.t. keys mapping to URIs (prefixes and terms) - can translate otherwise + + Map localContext = new TreeMap(); + //Add namespaces corresponding to core terms + localContext.put(JsonLDNamespace.dcterms.getPrefix(), JsonLDNamespace.dcterms.getUrl()); + localContext.put(JsonLDNamespace.dvcore.getPrefix(), JsonLDNamespace.dvcore.getUrl()); + localContext.put(JsonLDNamespace.schema.getPrefix(), JsonLDNamespace.schema.getUrl()); + + Map mdbMap = new TreeMap(); + Map dsftMap = new TreeMap(); + + List mdbList = metadataBlockService.listMetadataBlocks(); + for(MetadataBlock mdb: mdbList) { + if(mdb.getNamespaceUri() !=null) { + localContext.putIfAbsent(mdb.getName(), mdb.getNamespaceUri()); + mdbMap.put(mdb.getName(), mdb); + for(DatasetFieldType dsft: mdb.getDatasetFieldTypes()) { + dsftMap.put(mdb.getName() + ":" + dsft.getName(), dsft); + } + } else { + for(DatasetFieldType dftp: mdb.getDatasetFieldTypes()) { + if(dftp.getUri()!=null) { + localContext.putIfAbsent(dftp.getName(), dftp.getUri()); + + } + } + } + } + + json.remove("@context"); + List dsfl = new ArrayList(); + for(String key: json.keySet()) { + if(dsftMap.containsKey(key)) { + DatasetFieldType dsft = dsftMap.get(key); + DatasetField dsf = new DatasetField(); + dsf.setDatasetFieldType(dsft); + //Todo - normalize object vs. array + JsonValue val = json.get(key); + JsonArray valArray = null; + if (val instanceof JsonArray) { + if(!dsft.isAllowMultiples()) { + error(Response.Status.BAD_REQUEST, "Array for single value notsupported: " + dsft.getName()); + } else { + valArray = (JsonArray)val; + } + } else { + valArray = Json.createArrayBuilder().add(val).build(); + } + + if (dsft.isCompound()) { +/* List vals = parseCompoundValue(type, json, testType); + for (DatasetFieldCompoundValue dsfcv : vals) { + dsfcv.setParentDatasetField(ret); + } + dsf.setDatasetFieldCompoundValues(vals); +*/ + } else if (dsft.isControlledVocabulary()) { + + List vals = new LinkedList<>(); + for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + ControlledVocabularyValue cvv = datasetFieldSvc.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, strValue, true); + if (cvv == null) { + return error(Response.Status.BAD_REQUEST, "Unknown value for Controlled Vocab Field: " + dsft.getName() + " : " + strValue); + } + // Only add value to the list if it is not a duplicate + if (strValue.equals("Other")) { + System.out.println("vals = " + vals + ", contains: " + vals.contains(cvv)); + } + if (!vals.contains(cvv)) { + vals.add(cvv); + cvv.setDatasetFieldType(dsft); + } + } + dsf.setControlledVocabularyValues(vals); + + } else { + List vals = new LinkedList<>(); + + for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + + DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); + if(valArray.size()>1) { + datasetFieldValue.setDisplayOrder(vals.size() - 1); + } + datasetFieldValue.setValue(strValue.trim()); + vals.add(datasetFieldValue); + } + dsf.setDatasetFieldValues(vals); + + } + + //assemble new terms, add to existing + //multivalue? + //compound? + //merge with existing dv metadata + dsfl.add(dsf); + } else { + //Internal/non-metadatablock terms + // Add metadata related to the Dataset/DatasetVersion + + //("@id", id) - check is equal to existing globalID? + //Add to 'md on original' ? + //(JsonLDTerm.schemaOrg("version").getLabel(), version.getFriendlyVersionNumber()) + //Citation metadata? + //(JsonLDTerm.schemaOrg("datePublished").getLabel(), dataset.getPublicationDateFormattedYYYYMMDD()) + //(JsonLDTerm.schemaOrg("name").getLabel()) + //(JsonLDTerm.schemaOrg("dateModified").getLabel()) + + //Todo - handle non-CC0 licenses, without terms as an alternate field. + TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); + if(json.containsKey(JsonLDTerm.schemaOrg("license").getLabel())) { + if(json.get(JsonLDTerm.schemaOrg("license").getLabel()).toString().equals("https://creativecommons.org/publicdomain/zero/1.0/")) { + terms.setLicense(TermsOfUseAndAccess.defaultLicense); + } else { + terms.setLicense(TermsOfUseAndAccess.License.NONE); + } + } else { + if(json.containsKey(JsonLDTerm.termsOfUse.getLabel())) { + terms.setTermsOfUse(json.get(JsonLDTerm.termsOfUse.getLabel()).toString()); + } + } + if(json.containsKey(JsonLDTerm.confidentialityDeclaration.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.confidentialityDeclaration.getLabel())); + } + if(json.containsKey(JsonLDTerm.specialPermissions.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.specialPermissions.getLabel())); + } + if(json.containsKey(JsonLDTerm.restrictions.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.restrictions.getLabel())); + } + if(json.containsKey(JsonLDTerm.citationRequirements.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.citationRequirements.getLabel())); + } + if(json.containsKey(JsonLDTerm.depositorRequirements.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.depositorRequirements.getLabel())); + } + if(json.containsKey(JsonLDTerm.conditions.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.conditions.getLabel())); + } + if(json.containsKey(JsonLDTerm.disclaimer.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.disclaimer.getLabel())); + } + if(json.containsKey(JsonLDTerm.fileTermsOfAccess.getLabel())) { + JsonObject fAccessObject = json.getJsonObject(JsonLDTerm.fileTermsOfAccess.getLabel()); + if(fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getLabel())) { + terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getLabel())) { + terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.fileRequestAccess.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getLabel())) { + terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.originalArchive.getLabel())) { + terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.originalArchive.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getLabel())) { + terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.availabilityStatus.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.contactForAccess.getLabel())) { + terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getLabel())) { + terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.studyCompletion.getLabel())) { + terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.studyCompletion.getLabel())); + } + } + dsv.setTermsOfUseAndAccess(terms); + //move to new dataverse? + //aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), + // dataset.getDataverseContext().getDisplayName()); + + } + + } + dsv.setDatasetFields(dsfl); + + + DatasetVersion managedVersion; + if ( updateDraft ) { + Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); + managedVersion = managedDataset.getEditVersion(); + } else { + managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); + } + String info = updateDraft ? "Version Updated" : "Version Created"; + return ok(Json.createObjectBuilder().add(info, managedVersion.getVersionDate())); + + } catch (JsonParsingException ex) { + logger.log(Level.SEVERE, "Semantic error parsing dataset version Json: " + ex.getMessage(), ex); + return error( Response.Status.BAD_REQUEST, "Error parsing dataset version: " + ex.getMessage() ); + } catch (WrappedResponse ex) { + return ex.getResponse(); + + } + } + + @PUT @Path("{id}/deleteMetadata") public Response deleteVersionMetadata(String jsonBody, @PathParam("id") String id) throws WrappedResponse { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 5520de3954e..36b6858d63a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -20,7 +20,6 @@ import java.time.LocalDate; import java.util.List; import java.util.Map; -import java.util.ResourceBundle; import java.util.TreeMap; import java.util.Map.Entry; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java new file mode 100644 index 00000000000..373c26e91c9 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -0,0 +1,27 @@ +package edu.harvard.iq.dataverse.util.json; + +import java.util.Map; +import java.util.TreeMap; +import java.util.logging.Logger; + +import javax.json.JsonArray; +import javax.json.JsonObject; +import javax.json.JsonValue; + +public class JSONLDUtil { + + private static final Logger logger = Logger.getLogger(JSONLDUtil.class.getCanonicalName()); + + public static Map populateContext(JsonValue json) { + Map context = new TreeMap(); + if(json instanceof JsonArray) { + logger.warning("Array @context not yet supported"); + } else { + for(String key: ((JsonObject)json).keySet()) { + context.putIfAbsent(key, ((JsonObject)json).getString(key)); + } + } + return context; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java index 5acb0c437ae..20aeceda7de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java @@ -1,8 +1,5 @@ package edu.harvard.iq.dataverse.util.json; -import java.util.HashMap; -import java.util.Map; - public class JsonLDTerm { JsonLDNamespace namespace = null; From fb6421bbd1573f2eb012f1ab49f4e9daed704372 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 24 Mar 2020 16:29:29 -0400 Subject: [PATCH 002/354] merge new fields with existing ones --- .../harvard/iq/dataverse/api/Datasets.java | 318 +++++++++--------- 1 file changed, 166 insertions(+), 152 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index d321495c929..4cf49a213f2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -115,6 +115,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -650,176 +651,189 @@ public Response updateVersionMetadata( String jsonLDBody, @PathParam("id") Strin } } } - - json.remove("@context"); - List dsfl = new ArrayList(); + //get existing ones? + List dsfl = dsv.getDatasetFields(); + Map fieldByTypeMap = new HashMap(); + for(DatasetField dsf: dsfl) { + fieldByTypeMap.put(dsf.getDatasetFieldType(),dsf); + } for(String key: json.keySet()) { - if(dsftMap.containsKey(key)) { - DatasetFieldType dsft = dsftMap.get(key); - DatasetField dsf = new DatasetField(); - dsf.setDatasetFieldType(dsft); - //Todo - normalize object vs. array - JsonValue val = json.get(key); - JsonArray valArray = null; - if (val instanceof JsonArray) { - if(!dsft.isAllowMultiples()) { - error(Response.Status.BAD_REQUEST, "Array for single value notsupported: " + dsft.getName()); - } else { - valArray = (JsonArray)val; - } - } else { - valArray = Json.createArrayBuilder().add(val).build(); - } - - if (dsft.isCompound()) { -/* List vals = parseCompoundValue(type, json, testType); + if(!key.equals("@context")) { + if(dsftMap.containsKey(key)) { + DatasetFieldType dsft = dsftMap.get(key); + + DatasetField dsf = null; + if(fieldByTypeMap.containsKey(dsft)) { + dsf = fieldByTypeMap.get(dsft); + } else { + dsf = new DatasetField(); + dsfl.add(dsf); + } + dsf.setDatasetFieldType(dsft); + //Todo - normalize object vs. array + JsonValue val = json.get(key); + JsonArray valArray = null; + if (val instanceof JsonArray) { + if(!dsft.isAllowMultiples()) { + error(Response.Status.BAD_REQUEST, "Array for single value notsupported: " + dsft.getName()); + } else { + valArray = (JsonArray)val; + } + } else { + valArray = Json.createArrayBuilder().add(val).build(); + } + + if (dsft.isCompound()) { + /* List vals = parseCompoundValue(type, json, testType); for (DatasetFieldCompoundValue dsfcv : vals) { dsfcv.setParentDatasetField(ret); } dsf.setDatasetFieldCompoundValues(vals); -*/ - } else if (dsft.isControlledVocabulary()) { - - List vals = new LinkedList<>(); - for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { - String strValue = strVal.getString(); - ControlledVocabularyValue cvv = datasetFieldSvc.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, strValue, true); - if (cvv == null) { - return error(Response.Status.BAD_REQUEST, "Unknown value for Controlled Vocab Field: " + dsft.getName() + " : " + strValue); - } - // Only add value to the list if it is not a duplicate - if (strValue.equals("Other")) { - System.out.println("vals = " + vals + ", contains: " + vals.contains(cvv)); - } - if (!vals.contains(cvv)) { - vals.add(cvv); - cvv.setDatasetFieldType(dsft); - } - } - dsf.setControlledVocabularyValues(vals); + */ + } else if (dsft.isControlledVocabulary()) { + + List vals = new LinkedList<>(); + for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + ControlledVocabularyValue cvv = datasetFieldSvc.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, strValue, true); + if (cvv == null) { + return error(Response.Status.BAD_REQUEST, "Unknown value for Controlled Vocab Field: " + dsft.getName() + " : " + strValue); + } + // Only add value to the list if it is not a duplicate + if (strValue.equals("Other")) { + System.out.println("vals = " + vals + ", contains: " + vals.contains(cvv)); + } + if (!vals.contains(cvv)) { + vals.add(cvv); + cvv.setDatasetFieldType(dsft); + } + } + dsf.setControlledVocabularyValues(vals); + + } else { + List vals = new LinkedList<>(); + + for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + + DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); + if(valArray.size()>1) { + datasetFieldValue.setDisplayOrder(vals.size() - 1); + } + datasetFieldValue.setValue(strValue.trim()); + vals.add(datasetFieldValue); + } + dsf.setDatasetFieldValues(vals); - } else { - List vals = new LinkedList<>(); - - for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { - String strValue = strVal.getString(); - - DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); - if(valArray.size()>1) { - datasetFieldValue.setDisplayOrder(vals.size() - 1); - } - datasetFieldValue.setValue(strValue.trim()); - vals.add(datasetFieldValue); - } - dsf.setDatasetFieldValues(vals); + } + + //assemble new terms, add to existing + //multivalue? + //compound? + //merge with existing dv metadata + //dsfl.add(dsf); + } else { + //Internal/non-metadatablock terms + // Add metadata related to the Dataset/DatasetVersion + + //("@id", id) - check is equal to existing globalID? + //Add to 'md on original' ? + //(JsonLDTerm.schemaOrg("version").getLabel(), version.getFriendlyVersionNumber()) + //Citation metadata? + //(JsonLDTerm.schemaOrg("datePublished").getLabel(), dataset.getPublicationDateFormattedYYYYMMDD()) + //(JsonLDTerm.schemaOrg("name").getLabel()) + //(JsonLDTerm.schemaOrg("dateModified").getLabel()) + + //Todo - handle non-CC0 licenses, without terms as an alternate field. + TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); + if(json.containsKey(JsonLDTerm.schemaOrg("license").getLabel())) { + if(json.get(JsonLDTerm.schemaOrg("license").getLabel()).toString().equals("https://creativecommons.org/publicdomain/zero/1.0/")) { + terms.setLicense(TermsOfUseAndAccess.defaultLicense); + } else { + terms.setLicense(TermsOfUseAndAccess.License.NONE); + } + } else { + if(json.containsKey(JsonLDTerm.termsOfUse.getLabel())) { + terms.setTermsOfUse(json.get(JsonLDTerm.termsOfUse.getLabel()).toString()); + } + } + if(json.containsKey(JsonLDTerm.confidentialityDeclaration.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.confidentialityDeclaration.getLabel())); + } + if(json.containsKey(JsonLDTerm.specialPermissions.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.specialPermissions.getLabel())); + } + if(json.containsKey(JsonLDTerm.restrictions.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.restrictions.getLabel())); + } + if(json.containsKey(JsonLDTerm.citationRequirements.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.citationRequirements.getLabel())); + } + if(json.containsKey(JsonLDTerm.depositorRequirements.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.depositorRequirements.getLabel())); + } + if(json.containsKey(JsonLDTerm.conditions.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.conditions.getLabel())); + } + if(json.containsKey(JsonLDTerm.disclaimer.getLabel())) { + terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.disclaimer.getLabel())); + } + if(json.containsKey(JsonLDTerm.fileTermsOfAccess.getLabel())) { + JsonObject fAccessObject = json.getJsonObject(JsonLDTerm.fileTermsOfAccess.getLabel()); + if(fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getLabel())) { + terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getLabel())) { + terms.setFileAccessRequest(fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getLabel())) { + terms.setDataAccessPlace(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.originalArchive.getLabel())) { + terms.setOriginalArchive(fAccessObject.getString(JsonLDTerm.originalArchive.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getLabel())) { + terms.setAvailabilityStatus(fAccessObject.getString(JsonLDTerm.availabilityStatus.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.contactForAccess.getLabel())) { + terms.setContactForAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getLabel())) { + terms.setSizeOfCollection(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getLabel())); + } + if(fAccessObject.containsKey(JsonLDTerm.studyCompletion.getLabel())) { + terms.setStudyCompletion(fAccessObject.getString(JsonLDTerm.studyCompletion.getLabel())); + } + } + dsv.setTermsOfUseAndAccess(terms); + //move to new dataverse? + //aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), + // dataset.getDataverseContext().getDisplayName()); + + } - } - - //assemble new terms, add to existing - //multivalue? - //compound? - //merge with existing dv metadata - dsfl.add(dsf); - } else { - //Internal/non-metadatablock terms - // Add metadata related to the Dataset/DatasetVersion - - //("@id", id) - check is equal to existing globalID? - //Add to 'md on original' ? - //(JsonLDTerm.schemaOrg("version").getLabel(), version.getFriendlyVersionNumber()) - //Citation metadata? - //(JsonLDTerm.schemaOrg("datePublished").getLabel(), dataset.getPublicationDateFormattedYYYYMMDD()) - //(JsonLDTerm.schemaOrg("name").getLabel()) - //(JsonLDTerm.schemaOrg("dateModified").getLabel()) - - //Todo - handle non-CC0 licenses, without terms as an alternate field. - TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); - if(json.containsKey(JsonLDTerm.schemaOrg("license").getLabel())) { - if(json.get(JsonLDTerm.schemaOrg("license").getLabel()).toString().equals("https://creativecommons.org/publicdomain/zero/1.0/")) { - terms.setLicense(TermsOfUseAndAccess.defaultLicense); - } else { - terms.setLicense(TermsOfUseAndAccess.License.NONE); - } - } else { - if(json.containsKey(JsonLDTerm.termsOfUse.getLabel())) { - terms.setTermsOfUse(json.get(JsonLDTerm.termsOfUse.getLabel()).toString()); - } - } - if(json.containsKey(JsonLDTerm.confidentialityDeclaration.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.confidentialityDeclaration.getLabel())); - } - if(json.containsKey(JsonLDTerm.specialPermissions.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.specialPermissions.getLabel())); - } - if(json.containsKey(JsonLDTerm.restrictions.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.restrictions.getLabel())); - } - if(json.containsKey(JsonLDTerm.citationRequirements.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.citationRequirements.getLabel())); - } - if(json.containsKey(JsonLDTerm.depositorRequirements.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.depositorRequirements.getLabel())); - } - if(json.containsKey(JsonLDTerm.conditions.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.conditions.getLabel())); - } - if(json.containsKey(JsonLDTerm.disclaimer.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.disclaimer.getLabel())); - } - if(json.containsKey(JsonLDTerm.fileTermsOfAccess.getLabel())) { - JsonObject fAccessObject = json.getJsonObject(JsonLDTerm.fileTermsOfAccess.getLabel()); - if(fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getLabel())) { - terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getLabel())) { - terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.fileRequestAccess.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getLabel())) { - terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.originalArchive.getLabel())) { - terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.originalArchive.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getLabel())) { - terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.availabilityStatus.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.contactForAccess.getLabel())) { - terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getLabel())) { - terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.studyCompletion.getLabel())) { - terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.studyCompletion.getLabel())); - } - } - dsv.setTermsOfUseAndAccess(terms); - //move to new dataverse? - //aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), - // dataset.getDataverseContext().getDisplayName()); - } - } - dsv.setDatasetFields(dsfl); - + dsv.setDatasetFields(dsfl); + + DatasetVersion managedVersion; if ( updateDraft ) { - Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); - managedVersion = managedDataset.getEditVersion(); + Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); + managedVersion = managedDataset.getEditVersion(); } else { - managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); + managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); } String info = updateDraft ? "Version Updated" : "Version Created"; return ok(Json.createObjectBuilder().add(info, managedVersion.getVersionDate())); - + } catch (JsonParsingException ex) { - logger.log(Level.SEVERE, "Semantic error parsing dataset version Json: " + ex.getMessage(), ex); - return error( Response.Status.BAD_REQUEST, "Error parsing dataset version: " + ex.getMessage() ); + logger.log(Level.SEVERE, "Semantic error parsing dataset version Json: " + ex.getMessage(), ex); + return error( Response.Status.BAD_REQUEST, "Error parsing dataset version: " + ex.getMessage() ); } catch (WrappedResponse ex) { - return ex.getResponse(); - + return ex.getResponse(); + } } From f472b6c17f6419380447d8fde1095f0598318086 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 17 Apr 2020 14:31:00 -0400 Subject: [PATCH 003/354] differences from IQSS/develop that break compilation --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 70ec286092f..638336c3195 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -81,6 +81,7 @@ import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; +import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.UnforcedCommandException; import edu.harvard.iq.dataverse.engine.command.impl.GetDatasetStorageSizeCommand; @@ -97,8 +98,8 @@ import edu.harvard.iq.dataverse.util.ArchiverUtil; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.EjbUtil; +import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.util.json.ControlledVocabularyException; import edu.harvard.iq.dataverse.util.json.JSONLDUtil; import edu.harvard.iq.dataverse.util.json.JsonLDNamespace; import edu.harvard.iq.dataverse.util.json.JsonLDTerm; From d30586709792d3383d77d380521549022a3f729b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 8 Sep 2020 16:01:50 -0400 Subject: [PATCH 004/354] Add jsonld lib to compact to local context --- pom.xml | 5 ++++ .../harvard/iq/dataverse/api/Datasets.java | 25 +++++++++++++++++-- .../iq/dataverse/util/json/JSONLDUtil.java | 10 ++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 14bf0574d14..99588ae65b5 100644 --- a/pom.xml +++ b/pom.xml @@ -195,6 +195,11 @@ aws-java-sdk-s3 + + com.apicatalog + titanium-json-ld + 0.8.4 + org.apache.abdera diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 81a960c4890..ae5194143fe 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -105,6 +105,7 @@ import edu.harvard.iq.dataverse.util.json.JsonLDNamespace; import edu.harvard.iq.dataverse.util.json.JsonLDTerm; import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import edu.harvard.iq.dataverse.search.IndexServiceBean; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; @@ -163,6 +164,9 @@ import org.glassfish.jersey.media.multipart.FormDataParam; import com.amazonaws.services.s3.model.PartETag; +import com.apicatalog.jsonld.JsonLd; +import com.apicatalog.jsonld.api.JsonLdError; +import com.apicatalog.jsonld.document.JsonDocument; @Path("datasets") public class Datasets extends AbstractApiBean { @@ -631,10 +635,11 @@ public Response updateVersionMetadata( String jsonLDBody, @PathParam("id") Strin Dataset ds = findDatasetOrDie(id); boolean updateDraft = ds.getLatestVersion().isDraft(); DatasetVersion dsv = ds.getEditVersion(); - JsonObject json = Json.createReader(rdr).readObject(); - Map context = JSONLDUtil.populateContext(json.get("@context")); + //JsonObject json = Json.createReader(rdr).readObject(); + //Map context = JSONLDUtil.populateContext(json.get("@context")); //ToDo - check that the context and localContext align w.r.t. keys mapping to URIs (prefixes and terms) - can translate otherwise + Map localContext = new TreeMap(); //Add namespaces corresponding to core terms @@ -662,6 +667,22 @@ public Response updateVersionMetadata( String jsonLDBody, @PathParam("id") Strin } } } + +// Use JsonLd to expand/compact to localContext? +// Or just use expanded form against URIs? + + JsonObject jsonld = Json.createReader(rdr).readObject(); + JsonDocument doc = JsonDocument.of(jsonld); + JsonArray array = null; + try { + array = JsonLd.expand(doc).get(); + + jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(JSONLDUtil.getContext(localContext))).get(); + } catch (JsonLdError e) { + System.out.println(e.getMessage()); + } + + //get existing ones? List dsfl = dsv.getDatasetFields(); Map fieldByTypeMap = new HashMap(); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 373c26e91c9..ea5373564c8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -1,11 +1,14 @@ package edu.harvard.iq.dataverse.util.json; import java.util.Map; +import java.util.Map.Entry; import java.util.TreeMap; import java.util.logging.Logger; +import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; import javax.json.JsonValue; public class JSONLDUtil { @@ -23,5 +26,12 @@ public static Map populateContext(JsonValue json) { } return context; } + public static JsonObject getContext(Map contextMap) { + JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); + for(Entry e: contextMap.entrySet()) { + contextBuilder.add(e.getKey(), e.getValue()); + } + return contextBuilder.build(); + } } From 98d978e45faa9e436eff53559174e6a7f2dd52e3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Sep 2020 15:37:24 -0400 Subject: [PATCH 005/354] use expand/compact, refactor, add :startmigration endpoint --- .../harvard/iq/dataverse/api/Datasets.java | 514 +++++------------- .../harvard/iq/dataverse/api/Dataverses.java | 56 ++ .../iq/dataverse/util/json/JSONLDUtil.java | 302 +++++++++- 3 files changed, 501 insertions(+), 371 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index ae5194143fe..cad3b9b2c62 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -193,9 +193,6 @@ public class Datasets extends AbstractApiBean { @EJB DDIExportServiceBean ddiExportService; - @EJB - DatasetFieldServiceBean datasetfieldService; - @EJB MetadataBlockServiceBean metadataBlockService; @@ -621,254 +618,37 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, } } - @PUT - @Path("{id}/versions/{versionId}/metadata") - @Consumes("application/json-ld") - public Response updateVersionMetadata( String jsonLDBody, @PathParam("id") String id, @PathParam("versionId") String versionId ){ - - if ( ! ":draft".equals(versionId) ) { - return error( Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); - } - - try ( StringReader rdr = new StringReader(jsonLDBody) ) { - DataverseRequest req = createDataverseRequest(findUserOrDie()); - Dataset ds = findDatasetOrDie(id); - boolean updateDraft = ds.getLatestVersion().isDraft(); - DatasetVersion dsv = ds.getEditVersion(); - //JsonObject json = Json.createReader(rdr).readObject(); - //Map context = JSONLDUtil.populateContext(json.get("@context")); - - //ToDo - check that the context and localContext align w.r.t. keys mapping to URIs (prefixes and terms) - can translate otherwise - - - Map localContext = new TreeMap(); - //Add namespaces corresponding to core terms - localContext.put(JsonLDNamespace.dcterms.getPrefix(), JsonLDNamespace.dcterms.getUrl()); - localContext.put(JsonLDNamespace.dvcore.getPrefix(), JsonLDNamespace.dvcore.getUrl()); - localContext.put(JsonLDNamespace.schema.getPrefix(), JsonLDNamespace.schema.getUrl()); - - Map mdbMap = new TreeMap(); - Map dsftMap = new TreeMap(); - - List mdbList = metadataBlockService.listMetadataBlocks(); - for(MetadataBlock mdb: mdbList) { - if(mdb.getNamespaceUri() !=null) { - localContext.putIfAbsent(mdb.getName(), mdb.getNamespaceUri()); - mdbMap.put(mdb.getName(), mdb); - for(DatasetFieldType dsft: mdb.getDatasetFieldTypes()) { - dsftMap.put(mdb.getName() + ":" + dsft.getName(), dsft); - } - } else { - for(DatasetFieldType dftp: mdb.getDatasetFieldTypes()) { - if(dftp.getUri()!=null) { - localContext.putIfAbsent(dftp.getName(), dftp.getUri()); - - } - } - } - } - -// Use JsonLd to expand/compact to localContext? -// Or just use expanded form against URIs? - - JsonObject jsonld = Json.createReader(rdr).readObject(); - JsonDocument doc = JsonDocument.of(jsonld); - JsonArray array = null; - try { - array = JsonLd.expand(doc).get(); - - jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(JSONLDUtil.getContext(localContext))).get(); - } catch (JsonLdError e) { - System.out.println(e.getMessage()); - } - - - //get existing ones? - List dsfl = dsv.getDatasetFields(); - Map fieldByTypeMap = new HashMap(); - for(DatasetField dsf: dsfl) { - fieldByTypeMap.put(dsf.getDatasetFieldType(),dsf); - } - for(String key: json.keySet()) { - if(!key.equals("@context")) { - if(dsftMap.containsKey(key)) { - DatasetFieldType dsft = dsftMap.get(key); - - DatasetField dsf = null; - if(fieldByTypeMap.containsKey(dsft)) { - dsf = fieldByTypeMap.get(dsft); - } else { - dsf = new DatasetField(); - dsfl.add(dsf); - } - dsf.setDatasetFieldType(dsft); - //Todo - normalize object vs. array - JsonValue val = json.get(key); - JsonArray valArray = null; - if (val instanceof JsonArray) { - if(!dsft.isAllowMultiples()) { - error(Response.Status.BAD_REQUEST, "Array for single value notsupported: " + dsft.getName()); - } else { - valArray = (JsonArray)val; - } - } else { - valArray = Json.createArrayBuilder().add(val).build(); - } - - if (dsft.isCompound()) { - /* List vals = parseCompoundValue(type, json, testType); - for (DatasetFieldCompoundValue dsfcv : vals) { - dsfcv.setParentDatasetField(ret); - } - dsf.setDatasetFieldCompoundValues(vals); - */ - } else if (dsft.isControlledVocabulary()) { - - List vals = new LinkedList<>(); - for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { - String strValue = strVal.getString(); - ControlledVocabularyValue cvv = datasetFieldSvc.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, strValue, true); - if (cvv == null) { - return error(Response.Status.BAD_REQUEST, "Unknown value for Controlled Vocab Field: " + dsft.getName() + " : " + strValue); - } - // Only add value to the list if it is not a duplicate - if (strValue.equals("Other")) { - System.out.println("vals = " + vals + ", contains: " + vals.contains(cvv)); - } - if (!vals.contains(cvv)) { - vals.add(cvv); - cvv.setDatasetFieldType(dsft); - } - } - dsf.setControlledVocabularyValues(vals); - - } else { - List vals = new LinkedList<>(); - - for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { - String strValue = strVal.getString(); - - DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); - if(valArray.size()>1) { - datasetFieldValue.setDisplayOrder(vals.size() - 1); - } - datasetFieldValue.setValue(strValue.trim()); - vals.add(datasetFieldValue); - } - dsf.setDatasetFieldValues(vals); - - } - - //assemble new terms, add to existing - //multivalue? - //compound? - //merge with existing dv metadata - //dsfl.add(dsf); - } else { - //Internal/non-metadatablock terms - // Add metadata related to the Dataset/DatasetVersion - - //("@id", id) - check is equal to existing globalID? - //Add to 'md on original' ? - //(JsonLDTerm.schemaOrg("version").getLabel(), version.getFriendlyVersionNumber()) - //Citation metadata? - //(JsonLDTerm.schemaOrg("datePublished").getLabel(), dataset.getPublicationDateFormattedYYYYMMDD()) - //(JsonLDTerm.schemaOrg("name").getLabel()) - //(JsonLDTerm.schemaOrg("dateModified").getLabel()) - - //Todo - handle non-CC0 licenses, without terms as an alternate field. - TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); - if(json.containsKey(JsonLDTerm.schemaOrg("license").getLabel())) { - if(json.get(JsonLDTerm.schemaOrg("license").getLabel()).toString().equals("https://creativecommons.org/publicdomain/zero/1.0/")) { - terms.setLicense(TermsOfUseAndAccess.defaultLicense); - } else { - terms.setLicense(TermsOfUseAndAccess.License.NONE); - } - } else { - if(json.containsKey(JsonLDTerm.termsOfUse.getLabel())) { - terms.setTermsOfUse(json.get(JsonLDTerm.termsOfUse.getLabel()).toString()); - } - } - if(json.containsKey(JsonLDTerm.confidentialityDeclaration.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.confidentialityDeclaration.getLabel())); - } - if(json.containsKey(JsonLDTerm.specialPermissions.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.specialPermissions.getLabel())); - } - if(json.containsKey(JsonLDTerm.restrictions.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.restrictions.getLabel())); - } - if(json.containsKey(JsonLDTerm.citationRequirements.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.citationRequirements.getLabel())); - } - if(json.containsKey(JsonLDTerm.depositorRequirements.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.depositorRequirements.getLabel())); - } - if(json.containsKey(JsonLDTerm.conditions.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.conditions.getLabel())); - } - if(json.containsKey(JsonLDTerm.disclaimer.getLabel())) { - terms.setConfidentialityDeclaration(json.getString(JsonLDTerm.disclaimer.getLabel())); - } - if(json.containsKey(JsonLDTerm.fileTermsOfAccess.getLabel())) { - JsonObject fAccessObject = json.getJsonObject(JsonLDTerm.fileTermsOfAccess.getLabel()); - if(fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getLabel())) { - terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getLabel())) { - terms.setFileAccessRequest(fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getLabel())) { - terms.setDataAccessPlace(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.originalArchive.getLabel())) { - terms.setOriginalArchive(fAccessObject.getString(JsonLDTerm.originalArchive.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getLabel())) { - terms.setAvailabilityStatus(fAccessObject.getString(JsonLDTerm.availabilityStatus.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.contactForAccess.getLabel())) { - terms.setContactForAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getLabel())) { - terms.setSizeOfCollection(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getLabel())); - } - if(fAccessObject.containsKey(JsonLDTerm.studyCompletion.getLabel())) { - terms.setStudyCompletion(fAccessObject.getString(JsonLDTerm.studyCompletion.getLabel())); - } - } - dsv.setTermsOfUseAndAccess(terms); - //move to new dataverse? - //aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), - // dataset.getDataverseContext().getDisplayName()); - - } - - } - } - - dsv.setDatasetFields(dsfl); - - - DatasetVersion managedVersion; - if ( updateDraft ) { - Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); - managedVersion = managedDataset.getEditVersion(); - } else { - managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); - } - String info = updateDraft ? "Version Updated" : "Version Created"; - return ok(Json.createObjectBuilder().add(info, managedVersion.getVersionDate())); + @PUT + @Path("{id}/versions/{versionId}/metadata") + @Consumes("application/json-ld") + public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, + @PathParam("versionId") String versionId) { + + if (!":draft".equals(versionId)) { + return error(Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); + } + try { + Dataset ds = findDatasetOrDie(id); + DataverseRequest req = createDataverseRequest(findUserOrDie()); + DatasetVersion dsv = ds.getEditVersion(); + boolean updateDraft = ds.getLatestVersion().isDraft(); + dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc); + + DatasetVersion managedVersion; + if (updateDraft) { + Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); + managedVersion = managedDataset.getEditVersion(); + } else { + managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); + } + String info = updateDraft ? "Version Updated" : "Version Created"; + return ok(Json.createObjectBuilder().add(info, managedVersion.getVersionDate())); - } catch (JsonParsingException ex) { - logger.log(Level.SEVERE, "Semantic error parsing dataset version Json: " + ex.getMessage(), ex); - return error( Response.Status.BAD_REQUEST, "Error parsing dataset version: " + ex.getMessage() ); - } catch (WrappedResponse ex) { - return ex.getResponse(); + } catch (WrappedResponse ex) { + return ex.getResponse(); - } - } - + } + } @PUT @Path("{id}/deleteMetadata") @@ -1038,133 +818,137 @@ public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, } - private Response processDatasetUpdate(String jsonBody, String id, DataverseRequest req, Boolean replaceData){ - try (StringReader rdr = new StringReader(jsonBody)) { - - Dataset ds = findDatasetOrDie(id); - JsonObject json = Json.createReader(rdr).readObject(); - DatasetVersion dsv = ds.getEditVersion(); - - List fields = new LinkedList<>(); - DatasetField singleField = null; - - JsonArray fieldsJson = json.getJsonArray("fields"); - if( fieldsJson == null ){ - singleField = jsonParser().parseField(json, Boolean.FALSE); - fields.add(singleField); - } else{ - fields = jsonParser().parseMultipleFields(json); - } - + private Response processDatasetUpdate(String jsonBody, String id, DataverseRequest req, Boolean replaceData) { + try (StringReader rdr = new StringReader(jsonBody)) { - String valdationErrors = validateDatasetFieldValues(fields); + Dataset ds = findDatasetOrDie(id); + JsonObject json = Json.createReader(rdr).readObject(); + DatasetVersion dsv = ds.getEditVersion(); - if (!valdationErrors.isEmpty()) { - logger.log(Level.SEVERE, "Semantic error parsing dataset update Json: " + valdationErrors, valdationErrors); - return error(Response.Status.BAD_REQUEST, "Error parsing dataset update: " + valdationErrors); - } + List fields = new LinkedList<>(); + DatasetField singleField = null; - dsv.setVersionState(DatasetVersion.VersionState.DRAFT); + JsonArray fieldsJson = json.getJsonArray("fields"); + if (fieldsJson == null) { + singleField = jsonParser().parseField(json, Boolean.FALSE); + fields.add(singleField); + } else { + fields = jsonParser().parseMultipleFields(json); + } - //loop through the update fields - // and compare to the version fields - //if exist add/replace values - //if not add entire dsf - for (DatasetField updateField : fields) { - boolean found = false; - for (DatasetField dsf : dsv.getDatasetFields()) { - if (dsf.getDatasetFieldType().equals(updateField.getDatasetFieldType())) { - found = true; - if (dsf.isEmpty() || dsf.getDatasetFieldType().isAllowMultiples() || replaceData) { - List priorCVV = new ArrayList<>(); - String cvvDisplay = ""; + String valdationErrors = validateDatasetFieldValues(fields); - if (updateField.getDatasetFieldType().isControlledVocabulary()) { - cvvDisplay = dsf.getDisplayValue(); - for (ControlledVocabularyValue cvvOld : dsf.getControlledVocabularyValues()) { - priorCVV.add(cvvOld); - } - } + if (!valdationErrors.isEmpty()) { + logger.log(Level.SEVERE, "Semantic error parsing dataset update Json: " + valdationErrors, + valdationErrors); + return error(Response.Status.BAD_REQUEST, "Error parsing dataset update: " + valdationErrors); + } - if (replaceData) { - if (dsf.getDatasetFieldType().isAllowMultiples()) { - dsf.setDatasetFieldCompoundValues(new ArrayList<>()); - dsf.setDatasetFieldValues(new ArrayList<>()); - dsf.setControlledVocabularyValues(new ArrayList<>()); - priorCVV.clear(); - dsf.getControlledVocabularyValues().clear(); - } else { - dsf.setSingleValue(""); - dsf.setSingleControlledVocabularyValue(null); - } - } - if (updateField.getDatasetFieldType().isControlledVocabulary()) { - if (dsf.getDatasetFieldType().isAllowMultiples()) { - for (ControlledVocabularyValue cvv : updateField.getControlledVocabularyValues()) { - if (!cvvDisplay.contains(cvv.getStrValue())) { - priorCVV.add(cvv); - } - } - dsf.setControlledVocabularyValues(priorCVV); - } else { - dsf.setSingleControlledVocabularyValue(updateField.getSingleControlledVocabularyValue()); - } - } else { - if (!updateField.getDatasetFieldType().isCompound()) { - if (dsf.getDatasetFieldType().isAllowMultiples()) { - for (DatasetFieldValue dfv : updateField.getDatasetFieldValues()) { - if (!dsf.getDisplayValue().contains(dfv.getDisplayValue())) { - dfv.setDatasetField(dsf); - dsf.getDatasetFieldValues().add(dfv); - } - } - } else { - dsf.setSingleValue(updateField.getValue()); - } - } else { - for (DatasetFieldCompoundValue dfcv : updateField.getDatasetFieldCompoundValues()) { - if (!dsf.getCompoundDisplayValue().contains(updateField.getCompoundDisplayValue())) { - dfcv.setParentDatasetField(dsf); - dsf.setDatasetVersion(dsv); - dsf.getDatasetFieldCompoundValues().add(dfcv); - } - } - } - } - } else { - if (!dsf.isEmpty() && !dsf.getDatasetFieldType().isAllowMultiples() || !replaceData) { - return error(Response.Status.BAD_REQUEST, "You may not add data to a field that already has data and does not allow multiples. Use replace=true to replace existing data (" + dsf.getDatasetFieldType().getDisplayName() + ")"); - } - } - break; - } - } - if (!found) { - updateField.setDatasetVersion(dsv); - dsv.getDatasetFields().add(updateField); - } - } - boolean updateDraft = ds.getLatestVersion().isDraft(); - DatasetVersion managedVersion; + dsv.setVersionState(DatasetVersion.VersionState.DRAFT); + + // loop through the update fields + // and compare to the version fields + // if exist add/replace values + // if not add entire dsf + for (DatasetField updateField : fields) { + boolean found = false; + for (DatasetField dsf : dsv.getDatasetFields()) { + if (dsf.getDatasetFieldType().equals(updateField.getDatasetFieldType())) { + found = true; + if (dsf.isEmpty() || dsf.getDatasetFieldType().isAllowMultiples() || replaceData) { + List priorCVV = new ArrayList<>(); + String cvvDisplay = ""; + + if (updateField.getDatasetFieldType().isControlledVocabulary()) { + cvvDisplay = dsf.getDisplayValue(); + for (ControlledVocabularyValue cvvOld : dsf.getControlledVocabularyValues()) { + priorCVV.add(cvvOld); + } + } + + if (replaceData) { + if (dsf.getDatasetFieldType().isAllowMultiples()) { + dsf.setDatasetFieldCompoundValues(new ArrayList<>()); + dsf.setDatasetFieldValues(new ArrayList<>()); + dsf.setControlledVocabularyValues(new ArrayList<>()); + priorCVV.clear(); + dsf.getControlledVocabularyValues().clear(); + } else { + dsf.setSingleValue(""); + dsf.setSingleControlledVocabularyValue(null); + } + } + if (updateField.getDatasetFieldType().isControlledVocabulary()) { + if (dsf.getDatasetFieldType().isAllowMultiples()) { + for (ControlledVocabularyValue cvv : updateField.getControlledVocabularyValues()) { + if (!cvvDisplay.contains(cvv.getStrValue())) { + priorCVV.add(cvv); + } + } + dsf.setControlledVocabularyValues(priorCVV); + } else { + dsf.setSingleControlledVocabularyValue( + updateField.getSingleControlledVocabularyValue()); + } + } else { + if (!updateField.getDatasetFieldType().isCompound()) { + if (dsf.getDatasetFieldType().isAllowMultiples()) { + for (DatasetFieldValue dfv : updateField.getDatasetFieldValues()) { + if (!dsf.getDisplayValue().contains(dfv.getDisplayValue())) { + dfv.setDatasetField(dsf); + dsf.getDatasetFieldValues().add(dfv); + } + } + } else { + dsf.setSingleValue(updateField.getValue()); + } + } else { + for (DatasetFieldCompoundValue dfcv : updateField.getDatasetFieldCompoundValues()) { + if (!dsf.getCompoundDisplayValue() + .contains(updateField.getCompoundDisplayValue())) { + dfcv.setParentDatasetField(dsf); + dsf.setDatasetVersion(dsv); + dsf.getDatasetFieldCompoundValues().add(dfcv); + } + } + } + } + } else { + if (!dsf.isEmpty() && !dsf.getDatasetFieldType().isAllowMultiples() || !replaceData) { + return error(Response.Status.BAD_REQUEST, + "You may not add data to a field that already has data and does not allow multiples. Use replace=true to replace existing data (" + + dsf.getDatasetFieldType().getDisplayName() + ")"); + } + } + break; + } + } + if (!found) { + updateField.setDatasetVersion(dsv); + dsv.getDatasetFields().add(updateField); + } + } + boolean updateDraft = ds.getLatestVersion().isDraft(); + DatasetVersion managedVersion; - if (updateDraft) { - managedVersion = execCommand(new UpdateDatasetVersionCommand(ds, req)).getEditVersion(); - } else { - managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); - } + if (updateDraft) { + managedVersion = execCommand(new UpdateDatasetVersionCommand(ds, req)).getEditVersion(); + } else { + managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); + } - return ok(json(managedVersion)); + return ok(json(managedVersion)); - } catch (JsonParseException ex) { - logger.log(Level.SEVERE, "Semantic error parsing dataset update Json: " + ex.getMessage(), ex); - return error(Response.Status.BAD_REQUEST, "Error parsing dataset update: " + ex.getMessage()); + } catch (JsonParseException ex) { + logger.log(Level.SEVERE, "Semantic error parsing dataset update Json: " + ex.getMessage(), ex); + return error(Response.Status.BAD_REQUEST, "Error parsing dataset update: " + ex.getMessage()); - } catch (WrappedResponse ex) { - logger.log(Level.SEVERE, "Update metdata error: " + ex.getMessage(), ex); - return ex.getResponse(); + } catch (WrappedResponse ex) { + logger.log(Level.SEVERE, "Update metdata error: " + ex.getMessage(), ex); + return ex.getResponse(); - } - } + } + } private String validateDatasetFieldValues(List fields) { StringBuilder error = new StringBuilder(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 2c73c5ad36e..411f23e42ac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -61,6 +61,8 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.StringUtil; import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty; + +import edu.harvard.iq.dataverse.util.json.JSONLDUtil; import edu.harvard.iq.dataverse.util.json.JsonParseException; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.brief; import java.io.StringReader; @@ -84,6 +86,7 @@ import javax.json.stream.JsonParsingException; import javax.validation.ConstraintViolation; import javax.validation.ConstraintViolationException; +import javax.ws.rs.BadRequestException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -99,6 +102,7 @@ import static edu.harvard.iq.dataverse.util.json.JsonPrinter.toJsonArray; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; import java.text.MessageFormat; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.Optional; @@ -393,6 +397,58 @@ public Response importDatasetDdi(String xml, @PathParam("identifier") String par } } + @POST + @Path("{identifier}/datasets/:startmigration") + @Consumes("application/json-ld") + public Response recreateDataset(String jsonLDBody, @PathParam("identifier") String parentIdtf) { + try { + User u = findUserOrDie(); + if (!u.isSuperuser()) { + return error(Status.FORBIDDEN, "Not a superuser"); + } + Dataverse owner = findDataverseOrDie(parentIdtf); + + Dataset ds = new Dataset(); + + + ds = JSONLDUtil.updateDatasetFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc);; + //ToDo - verify PID is one Dataverse can manage (protocol/authority/shoulder match) + if(! + (ds.getAuthority().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Authority))&& + ds.getProtocol().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Protocol))&& + ds.getIdentifier().startsWith(settingsService.getValueForKey(SettingsServiceBean.Key.Shoulder)))) { + throw new BadRequestException("Cannot recreate a dataset that has a PID that doesn't match the server's settings"); + } + if(!datasetSvc.isIdentifierLocallyUnique(ds)) { + throw new BadRequestException("Cannot recreate a dataset whose PID is already in use"); + } + + ds.setOwner(owner); + + if (ds.getVersions().isEmpty()) { + return badRequest("Supplied json must contain a single dataset version."); + } + + DatasetVersion version = ds.getVersions().get(0); + if (!version.isPublished()) { + throw new BadRequestException("Cannot recreate a dataset that hasn't been published."); + } + version.setVersionState(DatasetVersion.VersionState.DRAFT); + + DataverseRequest request = createDataverseRequest(u); + + Dataset managedDs = execCommand(new ImportDatasetCommand(ds, request)); + JsonObjectBuilder responseBld = Json.createObjectBuilder() + .add("id", managedDs.getId()) + .add("persistentId", managedDs.getGlobalId().toString()); + + return created("/datasets/" + managedDs.getId(), responseBld); + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + private Dataset parseDataset(String datasetJson) throws WrappedResponse { try (StringReader rdr = new StringReader(datasetJson)) { return jsonParser().parseDataset(Json.createReader(rdr).readObject()); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index ea5373564c8..0cf6df77fec 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -1,6 +1,12 @@ package edu.harvard.iq.dataverse.util.json; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Map.Entry; import java.util.TreeMap; import java.util.logging.Logger; @@ -9,29 +15,313 @@ import javax.json.JsonArray; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; +import javax.json.JsonString; import javax.json.JsonValue; +import javax.ws.rs.BadRequestException; +import com.apicatalog.jsonld.JsonLd; +import com.apicatalog.jsonld.api.JsonLdError; +import com.apicatalog.jsonld.document.JsonDocument; + +import edu.harvard.iq.dataverse.ControlledVocabularyValue; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValue; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.MetadataBlock; +import edu.harvard.iq.dataverse.MetadataBlockServiceBean; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess; public class JSONLDUtil { private static final Logger logger = Logger.getLogger(JSONLDUtil.class.getCanonicalName()); - + public static Map populateContext(JsonValue json) { - Map context = new TreeMap(); - if(json instanceof JsonArray) { + Map context = new TreeMap(); + if (json instanceof JsonArray) { logger.warning("Array @context not yet supported"); } else { - for(String key: ((JsonObject)json).keySet()) { - context.putIfAbsent(key, ((JsonObject)json).getString(key)); + for (String key : ((JsonObject) json).keySet()) { + context.putIfAbsent(key, ((JsonObject) json).getString(key)); } } return context; } + public static JsonObject getContext(Map contextMap) { JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); - for(Entry e: contextMap.entrySet()) { + for (Entry e : contextMap.entrySet()) { contextBuilder.add(e.getKey(), e.getValue()); } return contextBuilder.build(); } + public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { + + DatasetVersion dsv = new DatasetVersion(); + JsonObject jsonld = recontextualizeJsonLD(jsonLDBody, metadataBlockSvc); + Optional maybePid = GlobalId.parse(jsonld.getString("@id")); + if (maybePid.isPresent()) { + ds.setGlobalId(maybePid.get()); + } else { + // unparsable PID passed. Terminate. + throw new BadRequestException ("Cannot parse the @id '" + jsonld.getString("@id") + "'. Make sure it is in valid form - see Dataverse Native API documentation."); + } + + dsv = updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc); + dsv.setDataset(ds); + + List versions = new ArrayList<>(1); + versions.add(dsv); + + ds.setVersions(versions); + return null; + } + + public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, String jsonLDBody, + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { + JsonObject jsonld = recontextualizeJsonLD(jsonLDBody, metadataBlockSvc); + return updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc); + } + + public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, JsonObject jsonld, + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { + + Map dsftMap = new TreeMap(); + + List mdbList = metadataBlockSvc.listMetadataBlocks(); + for (MetadataBlock mdb : mdbList) { + if (mdb.getNamespaceUri() != null) { + for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { + dsftMap.put(mdb.getName() + ":" + dsft.getName(), dsft); + } + } + } + // get existing ones? + List dsfl = dsv.getDatasetFields(); + Map fieldByTypeMap = new HashMap(); + for (DatasetField dsf : dsfl) { + fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); + } + for (String key : jsonld.keySet()) { + if (!key.equals("@context")) { + if (dsftMap.containsKey(key)) { + DatasetFieldType dsft = dsftMap.get(key); + + DatasetField dsf = null; + if (fieldByTypeMap.containsKey(dsft)) { + dsf = fieldByTypeMap.get(dsft); + } else { + dsf = new DatasetField(); + dsfl.add(dsf); + } + dsf.setDatasetFieldType(dsft); + // Todo - normalize object vs. array + JsonValue val = jsonld.get(key); + JsonArray valArray = null; + if (val instanceof JsonArray) { + if (!dsft.isAllowMultiples()) { + throw new BadRequestException("Array for single value notsupported: " + dsft.getName()); + } else { + valArray = (JsonArray) val; + } + } else { + valArray = Json.createArrayBuilder().add(val).build(); + } + + if (dsft.isCompound()) { + /* + * List vals = parseCompoundValue(type, json, + * testType); for (DatasetFieldCompoundValue dsfcv : vals) { + * dsfcv.setParentDatasetField(ret); } dsf.setDatasetFieldCompoundValues(vals); + */ + } else if (dsft.isControlledVocabulary()) { + + List vals = new LinkedList<>(); + for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + ControlledVocabularyValue cvv = datasetFieldSvc + .findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, strValue, true); + if (cvv == null) { + throw new BadRequestException("Unknown value for Controlled Vocab Field: " + + dsft.getName() + " : " + strValue); + } + // Only add value to the list if it is not a duplicate + if (strValue.equals("Other")) { + System.out.println("vals = " + vals + ", contains: " + vals.contains(cvv)); + } + if (!vals.contains(cvv)) { + vals.add(cvv); + cvv.setDatasetFieldType(dsft); + } + } + dsf.setControlledVocabularyValues(vals); + + } else { + List vals = new LinkedList<>(); + + for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + + DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); + if (valArray.size() > 1) { + datasetFieldValue.setDisplayOrder(vals.size() - 1); + } + datasetFieldValue.setValue(strValue.trim()); + vals.add(datasetFieldValue); + } + dsf.setDatasetFieldValues(vals); + + } + + // assemble new terms, add to existing + // multivalue? + // compound? + // merge with existing dv metadata + // dsfl.add(dsf); + } else { + // Internal/non-metadatablock terms + // Add metadata related to the Dataset/DatasetVersion + + // ("@id", id) - check is equal to existing globalID? + // Add to 'md on original' ? + // (JsonLDTerm.schemaOrg("version").getLabel(), + // version.getFriendlyVersionNumber()) + // Citation metadata? + // (JsonLDTerm.schemaOrg("datePublished").getLabel(), + // dataset.getPublicationDateFormattedYYYYMMDD()) + // (JsonLDTerm.schemaOrg("name").getLabel()) + // (JsonLDTerm.schemaOrg("dateModified").getLabel()) + + // Todo - handle non-CC0 licenses, without terms as an alternate field. + TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); + if (jsonld.containsKey(JsonLDTerm.schemaOrg("license").getLabel())) { + if (jsonld.get(JsonLDTerm.schemaOrg("license").getLabel()).toString() + .equals("https://creativecommons.org/publicdomain/zero/1.0/")) { + terms.setLicense(TermsOfUseAndAccess.defaultLicense); + } else { + terms.setLicense(TermsOfUseAndAccess.License.NONE); + } + } else { + if (jsonld.containsKey(JsonLDTerm.termsOfUse.getLabel())) { + terms.setTermsOfUse(jsonld.get(JsonLDTerm.termsOfUse.getLabel()).toString()); + } + } + if (jsonld.containsKey(JsonLDTerm.confidentialityDeclaration.getLabel())) { + terms.setConfidentialityDeclaration( + jsonld.getString(JsonLDTerm.confidentialityDeclaration.getLabel())); + } + if (jsonld.containsKey(JsonLDTerm.specialPermissions.getLabel())) { + terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.specialPermissions.getLabel())); + } + if (jsonld.containsKey(JsonLDTerm.restrictions.getLabel())) { + terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.restrictions.getLabel())); + } + if (jsonld.containsKey(JsonLDTerm.citationRequirements.getLabel())) { + terms.setConfidentialityDeclaration( + jsonld.getString(JsonLDTerm.citationRequirements.getLabel())); + } + if (jsonld.containsKey(JsonLDTerm.depositorRequirements.getLabel())) { + terms.setConfidentialityDeclaration( + jsonld.getString(JsonLDTerm.depositorRequirements.getLabel())); + } + if (jsonld.containsKey(JsonLDTerm.conditions.getLabel())) { + terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.conditions.getLabel())); + } + if (jsonld.containsKey(JsonLDTerm.disclaimer.getLabel())) { + terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.disclaimer.getLabel())); + } + if (jsonld.containsKey(JsonLDTerm.fileTermsOfAccess.getLabel())) { + JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getLabel()); + if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getLabel())) { + terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getLabel())); + } + if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getLabel())) { + terms.setFileAccessRequest( + fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getLabel())); + } + if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getLabel())) { + terms.setDataAccessPlace(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getLabel())); + } + if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getLabel())) { + terms.setOriginalArchive(fAccessObject.getString(JsonLDTerm.originalArchive.getLabel())); + } + if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getLabel())) { + terms.setAvailabilityStatus( + fAccessObject.getString(JsonLDTerm.availabilityStatus.getLabel())); + } + if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getLabel())) { + terms.setContactForAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getLabel())); + } + if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getLabel())) { + terms.setSizeOfCollection(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getLabel())); + } + if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getLabel())) { + terms.setStudyCompletion(fAccessObject.getString(JsonLDTerm.studyCompletion.getLabel())); + } + } + dsv.setTermsOfUseAndAccess(terms); + // move to new dataverse? + // aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), + // dataset.getDataverseContext().getDisplayName()); + + } + + } + } + + dsv.setDatasetFields(dsfl); + + return dsv; + } + + private static JsonObject recontextualizeJsonLD(String jsonLDString, MetadataBlockServiceBean metadataBlockSvc) { + try (StringReader rdr = new StringReader(jsonLDString)) { + + Map localContext = new TreeMap(); + // Add namespaces corresponding to core terms + localContext.put(JsonLDNamespace.dcterms.getPrefix(), JsonLDNamespace.dcterms.getUrl()); + localContext.put(JsonLDNamespace.dvcore.getPrefix(), JsonLDNamespace.dvcore.getUrl()); + localContext.put(JsonLDNamespace.schema.getPrefix(), JsonLDNamespace.schema.getUrl()); + + Map mdbMap = new TreeMap(); + Map dsftMap = new TreeMap(); + + List mdbList = metadataBlockSvc.listMetadataBlocks(); + for (MetadataBlock mdb : mdbList) { + if (mdb.getNamespaceUri() != null) { + localContext.putIfAbsent(mdb.getName(), mdb.getNamespaceUri()); + mdbMap.put(mdb.getName(), mdb); + for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { + dsftMap.put(mdb.getName() + ":" + dsft.getName(), dsft); + } + } else { + for (DatasetFieldType dftp : mdb.getDatasetFieldTypes()) { + if (dftp.getUri() != null) { + localContext.putIfAbsent(dftp.getName(), dftp.getUri()); + + } + } + } + } + + // Use JsonLd to expand/compact to localContext + JsonObject jsonld = Json.createReader(rdr).readObject(); + JsonDocument doc = JsonDocument.of(jsonld); + JsonArray array = null; + try { + array = JsonLd.expand(doc).get(); + + jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(JSONLDUtil.getContext(localContext))) + .get(); + return jsonld; + } catch (JsonLdError e) { + System.out.println(e.getMessage()); + return null; + } + } + } } From 244370246eba5e3e02681a478a59b2373ce21f17 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 16 Sep 2020 16:53:57 -0400 Subject: [PATCH 006/354] try fix for parse error --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 0cf6df77fec..a9f37542411 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -279,7 +279,7 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, } private static JsonObject recontextualizeJsonLD(String jsonLDString, MetadataBlockServiceBean metadataBlockSvc) { - try (StringReader rdr = new StringReader(jsonLDString)) { + try (StringReader rdr = new StringReader(jsonLDString.replaceAll("\r?\n", ""))) { Map localContext = new TreeMap(); // Add namespaces corresponding to core terms From d44264321edc00e2744a20b6b16bd199b3dda6ba Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 16 Sep 2020 16:55:01 -0400 Subject: [PATCH 007/354] log value --- src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index a9f37542411..83ef851912e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -279,6 +279,7 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, } private static JsonObject recontextualizeJsonLD(String jsonLDString, MetadataBlockServiceBean metadataBlockSvc) { + logger.fine(jsonLDString.replaceAll("\r?\n", "")); try (StringReader rdr = new StringReader(jsonLDString.replaceAll("\r?\n", ""))) { Map localContext = new TreeMap(); From fdeac97e24b9740ca7ea95759e20d6aece254f4b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 16 Sep 2020 17:30:39 -0400 Subject: [PATCH 008/354] return dataset --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 83ef851912e..7b030af6f9f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -78,7 +78,7 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, versions.add(dsv); ds.setVersions(versions); - return null; + return ds; } public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, String jsonLDBody, @@ -318,6 +318,7 @@ private static JsonObject recontextualizeJsonLD(String jsonLDString, MetadataBlo jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(JSONLDUtil.getContext(localContext))) .get(); + logger.fine("Compacted: " + jsonld); return jsonld; } catch (JsonLdError e) { System.out.println(e.getMessage()); From 554e62091b8b31d79b3c65882311b6dbe3db69a8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 17 Sep 2020 10:04:16 -0400 Subject: [PATCH 009/354] manage versionState, add debug output --- .../edu/harvard/iq/dataverse/api/Dataverses.java | 1 + .../harvard/iq/dataverse/util/json/JSONLDUtil.java | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 411f23e42ac..1e27cdc1448 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -433,6 +433,7 @@ public Response recreateDataset(String jsonLDBody, @PathParam("identifier") Stri if (!version.isPublished()) { throw new BadRequestException("Cannot recreate a dataset that hasn't been published."); } + //While the datasetversion whose metadata we're importing has been published, we consider it in draft until the API caller adds files and then completes the migration version.setVersionState(DatasetVersion.VersionState.DRAFT); DataverseRequest request = createDataverseRequest(u); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 7b030af6f9f..ae0f9f56107 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -33,6 +33,10 @@ import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.DatasetVersion.VersionState; +import edu.harvard.iq.dataverse.export.ExportService; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.bagit.OREMap; public class JSONLDUtil { @@ -62,6 +66,7 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { DatasetVersion dsv = new DatasetVersion(); + JsonObject jsonld = recontextualizeJsonLD(jsonLDBody, metadataBlockSvc); Optional maybePid = GlobalId.parse(jsonld.getString("@id")); if (maybePid.isPresent()) { @@ -274,7 +279,12 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, } dsv.setDatasetFields(dsfl); - + try { + logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } return dsv; } From bee7731fe34153ba1f89dc00b6e474d9c5b33a01 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 17 Sep 2020 10:25:44 -0400 Subject: [PATCH 010/354] move debug ore generation after configuring dataset --- .../edu/harvard/iq/dataverse/api/Dataverses.java | 4 ++-- .../harvard/iq/dataverse/util/json/JSONLDUtil.java | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 1e27cdc1448..2cf50e8a87c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -410,7 +410,7 @@ public Response recreateDataset(String jsonLDBody, @PathParam("identifier") Stri Dataset ds = new Dataset(); - + ds.setOwner(owner); ds = JSONLDUtil.updateDatasetFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc);; //ToDo - verify PID is one Dataverse can manage (protocol/authority/shoulder match) if(! @@ -423,7 +423,7 @@ public Response recreateDataset(String jsonLDBody, @PathParam("identifier") Stri throw new BadRequestException("Cannot recreate a dataset whose PID is already in use"); } - ds.setOwner(owner); + if (ds.getVersions().isEmpty()) { return badRequest("Supplied json must contain a single dataset version."); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index ae0f9f56107..810e01aaf47 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -83,6 +83,12 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, versions.add(dsv); ds.setVersions(versions); + try { + logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } return ds; } @@ -279,12 +285,7 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, } dsv.setDatasetFields(dsfl); - try { - logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + return dsv; } From f4cecd3f39992c895a73bbdf5bd2396518b0ce2c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 17 Sep 2020 12:16:56 -0400 Subject: [PATCH 011/354] set versionstate, simplify, move terms init outside loop --- .../iq/dataverse/util/json/JSONLDUtil.java | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 810e01aaf47..49c0242469e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -117,6 +117,7 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, for (DatasetField dsf : dsfl) { fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); } + TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); for (String key : jsonld.keySet()) { if (!key.equals("@context")) { if (dsftMap.containsKey(key)) { @@ -208,44 +209,37 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, // (JsonLDTerm.schemaOrg("dateModified").getLabel()) // Todo - handle non-CC0 licenses, without terms as an alternate field. - TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); - if (jsonld.containsKey(JsonLDTerm.schemaOrg("license").getLabel())) { + if(key.equals(JsonLDTerm.schemaOrg("datePublished").getLabel())) { + dsv.setVersionState(VersionState.RELEASED); + } + + if (key.equals(JsonLDTerm.schemaOrg("license").getLabel())) { if (jsonld.get(JsonLDTerm.schemaOrg("license").getLabel()).toString() .equals("https://creativecommons.org/publicdomain/zero/1.0/")) { terms.setLicense(TermsOfUseAndAccess.defaultLicense); } else { terms.setLicense(TermsOfUseAndAccess.License.NONE); } - } else { - if (jsonld.containsKey(JsonLDTerm.termsOfUse.getLabel())) { + } else if(key.equals(JsonLDTerm.termsOfUse.getLabel())) { terms.setTermsOfUse(jsonld.get(JsonLDTerm.termsOfUse.getLabel()).toString()); - } - } - if (jsonld.containsKey(JsonLDTerm.confidentialityDeclaration.getLabel())) { + } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getLabel())) { terms.setConfidentialityDeclaration( jsonld.getString(JsonLDTerm.confidentialityDeclaration.getLabel())); - } - if (jsonld.containsKey(JsonLDTerm.specialPermissions.getLabel())) { + } else if (key.equals(JsonLDTerm.specialPermissions.getLabel())) { terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.specialPermissions.getLabel())); - } - if (jsonld.containsKey(JsonLDTerm.restrictions.getLabel())) { + } else if (key.equals(JsonLDTerm.restrictions.getLabel())) { terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.restrictions.getLabel())); - } - if (jsonld.containsKey(JsonLDTerm.citationRequirements.getLabel())) { + } else if (key.equals(JsonLDTerm.citationRequirements.getLabel())) { terms.setConfidentialityDeclaration( jsonld.getString(JsonLDTerm.citationRequirements.getLabel())); - } - if (jsonld.containsKey(JsonLDTerm.depositorRequirements.getLabel())) { + } else if (key.equals(JsonLDTerm.depositorRequirements.getLabel())) { terms.setConfidentialityDeclaration( jsonld.getString(JsonLDTerm.depositorRequirements.getLabel())); - } - if (jsonld.containsKey(JsonLDTerm.conditions.getLabel())) { + } else if (key.equals(JsonLDTerm.conditions.getLabel())) { terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.conditions.getLabel())); - } - if (jsonld.containsKey(JsonLDTerm.disclaimer.getLabel())) { + } else if (key.equals(JsonLDTerm.disclaimer.getLabel())) { terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.disclaimer.getLabel())); - } - if (jsonld.containsKey(JsonLDTerm.fileTermsOfAccess.getLabel())) { + } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getLabel())) { JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getLabel()); if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getLabel())) { terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getLabel())); From a4189de0f3332ec75216c57374562a50cc4f5944 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 17 Sep 2020 13:30:32 -0400 Subject: [PATCH 012/354] parse version number --- .../harvard/iq/dataverse/util/json/JSONLDUtil.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 49c0242469e..c40d0ad90cc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -211,9 +211,14 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, // Todo - handle non-CC0 licenses, without terms as an alternate field. if(key.equals(JsonLDTerm.schemaOrg("datePublished").getLabel())) { dsv.setVersionState(VersionState.RELEASED); - } - - if (key.equals(JsonLDTerm.schemaOrg("license").getLabel())) { + } else if(key.equals(JsonLDTerm.schemaOrg("version").getLabel())) { + String friendlyVersion = jsonld.get(JsonLDTerm.schemaOrg("version").getLabel()).toString(); + int index = friendlyVersion.indexOf("."); + if(index>0) { + dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); + dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index+1))); + } + } else if (key.equals(JsonLDTerm.schemaOrg("license").getLabel())) { if (jsonld.get(JsonLDTerm.schemaOrg("license").getLabel()).toString() .equals("https://creativecommons.org/publicdomain/zero/1.0/")) { terms.setLicense(TermsOfUseAndAccess.defaultLicense); From e0de1dba2dc6f1959c03580acdbffa465332f40a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 17 Sep 2020 13:51:48 -0400 Subject: [PATCH 013/354] fix toStrings --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index c40d0ad90cc..0fcb8b952d8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -212,21 +212,21 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, if(key.equals(JsonLDTerm.schemaOrg("datePublished").getLabel())) { dsv.setVersionState(VersionState.RELEASED); } else if(key.equals(JsonLDTerm.schemaOrg("version").getLabel())) { - String friendlyVersion = jsonld.get(JsonLDTerm.schemaOrg("version").getLabel()).toString(); + String friendlyVersion = jsonld.getString(JsonLDTerm.schemaOrg("version").getLabel()); int index = friendlyVersion.indexOf("."); if(index>0) { dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index+1))); } } else if (key.equals(JsonLDTerm.schemaOrg("license").getLabel())) { - if (jsonld.get(JsonLDTerm.schemaOrg("license").getLabel()).toString() + if (jsonld.getString(JsonLDTerm.schemaOrg("license").getLabel()) .equals("https://creativecommons.org/publicdomain/zero/1.0/")) { terms.setLicense(TermsOfUseAndAccess.defaultLicense); } else { terms.setLicense(TermsOfUseAndAccess.License.NONE); } } else if(key.equals(JsonLDTerm.termsOfUse.getLabel())) { - terms.setTermsOfUse(jsonld.get(JsonLDTerm.termsOfUse.getLabel()).toString()); + terms.setTermsOfUse(jsonld.getString(JsonLDTerm.termsOfUse.getLabel())); } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getLabel())) { terms.setConfidentialityDeclaration( jsonld.getString(JsonLDTerm.confidentialityDeclaration.getLabel())); From 928a88e1df05300b45d7d03e0bb0f5dda73cd642 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 17 Sep 2020 16:13:12 -0400 Subject: [PATCH 014/354] debug null pointer in DataverseFieldTypeInputLevel --- .../java/edu/harvard/iq/dataverse/DatasetPage.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 458fcf56ab0..bad2628af5b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1661,8 +1661,16 @@ private void updateDatasetFieldInputLevels() { Use the DatasetFieldType id's which are the Map's keys --------------------------------------------------------- */ List idList = new ArrayList<>(mapDatasetFields.keySet()); - List dsFieldTypeInputLevels = dataverseFieldTypeInputLevelService.findByDataverseIdAndDatasetFieldTypeIdList(dvIdForInputLevel, idList); + List idStringList = new ArrayList(); + for(Long id: idList) { + idStringList.add(id.toString()); + } + logger.fine(String.join(",",idStringList)); + List dsFieldTypeInputLevels = dataverseFieldTypeInputLevelService.findByDataverseIdAndDatasetFieldTypeIdList(dvIdForInputLevel, idList); + if(dsFieldTypeInputLevels == null) { + logger.fine("dsFieldTypeInputLevels is null for ds: " + this.getId()); + } else { /* --------------------------------------------------------- Iterate through List of DataverseFieldTypeInputLevel objects Call "setInclude" on its related DatasetField object @@ -1678,8 +1686,11 @@ private void updateDatasetFieldInputLevels() { // remove from hash mapDatasetFields.remove(oneDSFieldTypeInputLevel.getDatasetFieldType().getId()); } + } else { + logger.fine("oneDSFieldTypeInputLevel is null"); } } // end: updateDatasetFieldInputLevels + } /* --------------------------------------------------------- Iterate through any DatasetField objects remaining in the hash From b78aed1886d4f9f52e418731e1119826f1b08c77 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 17 Sep 2020 18:06:12 -0400 Subject: [PATCH 015/354] add support for fields with their own formal URI --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 0fcb8b952d8..23923228f93 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -107,6 +107,10 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, for (MetadataBlock mdb : mdbList) { if (mdb.getNamespaceUri() != null) { for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { + if(dsft.getUri()!=null) { + dsftMap.put(dsft.getUri(), dsft); + } + //Can allow both, so don't put this in an else? dsftMap.put(mdb.getName() + ":" + dsft.getName(), dsft); } } From 3a47630f339d9eada23cbb12f65cd382b2c7f2d1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 17 Sep 2020 18:06:32 -0400 Subject: [PATCH 016/354] allow non-published to support debugging and future use --- src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 36b6858d63a..43cc0fd3e00 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -130,9 +130,10 @@ public JsonObject getOREMap() throws Exception { Json.createArrayBuilder().add(JsonLDTerm.ore("Aggregation").getLabel()) .add(JsonLDTerm.schemaOrg("Dataset").getLabel())) .add(JsonLDTerm.schemaOrg("version").getLabel(), version.getFriendlyVersionNumber()) - .add(JsonLDTerm.schemaOrg("datePublished").getLabel(), dataset.getPublicationDateFormattedYYYYMMDD()) .add(JsonLDTerm.schemaOrg("name").getLabel(), version.getTitle()) .add(JsonLDTerm.schemaOrg("dateModified").getLabel(), version.getLastUpdateTime().toString()); + //Allow oremap for non-published versions (not yet implemented) + addIfNotNull(aggBuilder, JsonLDTerm.schemaOrg("datePublished"), dataset.getPublicationDateFormattedYYYYMMDD()); TermsOfUseAndAccess terms = version.getTermsOfUseAndAccess(); if (terms.getLicense() == TermsOfUseAndAccess.License.CC0) { From 3f8534bd690c28b1203a1979130183c573dfff2f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 18 Sep 2020 11:18:21 -0400 Subject: [PATCH 017/354] refactor, use expanded version directly --- .../iq/dataverse/util/json/JSONLDUtil.java | 105 ++++++++++++------ 1 file changed, 69 insertions(+), 36 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 23923228f93..939c8ddcfff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -34,15 +34,13 @@ import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; -import edu.harvard.iq.dataverse.export.ExportService; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.bagit.OREMap; public class JSONLDUtil { private static final Logger logger = Logger.getLogger(JSONLDUtil.class.getCanonicalName()); - public static Map populateContext(JsonValue json) { +/* private static Map populateContext(JsonValue json) { Map context = new TreeMap(); if (json instanceof JsonArray) { logger.warning("Array @context not yet supported"); @@ -53,6 +51,7 @@ public static Map populateContext(JsonValue json) { } return context; } + */ public static JsonObject getContext(Map contextMap) { JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); @@ -67,7 +66,7 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, DatasetVersion dsv = new DatasetVersion(); - JsonObject jsonld = recontextualizeJsonLD(jsonLDBody, metadataBlockSvc); + JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); Optional maybePid = GlobalId.parse(jsonld.getString("@id")); if (maybePid.isPresent()) { ds.setGlobalId(maybePid.get()); @@ -94,27 +93,15 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, String jsonLDBody, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { - JsonObject jsonld = recontextualizeJsonLD(jsonLDBody, metadataBlockSvc); + JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); return updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc); } public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, JsonObject jsonld, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { - Map dsftMap = new TreeMap(); + populateFieldTypeMap(metadataBlockSvc); - List mdbList = metadataBlockSvc.listMetadataBlocks(); - for (MetadataBlock mdb : mdbList) { - if (mdb.getNamespaceUri() != null) { - for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { - if(dsft.getUri()!=null) { - dsftMap.put(dsft.getUri(), dsft); - } - //Can allow both, so don't put this in an else? - dsftMap.put(mdb.getName() + ":" + dsft.getName(), dsft); - } - } - } // get existing ones? List dsfl = dsv.getDatasetFields(); Map fieldByTypeMap = new HashMap(); @@ -292,36 +279,81 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, return dsv; } - private static JsonObject recontextualizeJsonLD(String jsonLDString, MetadataBlockServiceBean metadataBlockSvc) { - logger.fine(jsonLDString.replaceAll("\r?\n", "")); - try (StringReader rdr = new StringReader(jsonLDString.replaceAll("\r?\n", ""))) { + static Map localContext = new TreeMap(); + static Map dsftMap = new TreeMap(); + + private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockSvc) { + if (dsftMap.isEmpty()) { + + List mdbList = metadataBlockSvc.listMetadataBlocks(); - Map localContext = new TreeMap(); + for (MetadataBlock mdb : mdbList) { + boolean blockHasUri = mdb.getNamespaceUri() != null; + for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { + if (dsft.getUri() != null) { + dsftMap.put(dsft.getUri(), dsft); + } + if (blockHasUri) { + dsftMap.put(mdb.getNamespaceUri() + dsft.getName(), dsft); + } + } + } + logger.fine("DSFT Map: " + String.join(", ",dsftMap.keySet())); + } + } + + private static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { + if (localContext.isEmpty()) { + // Add namespaces corresponding to core terms localContext.put(JsonLDNamespace.dcterms.getPrefix(), JsonLDNamespace.dcterms.getUrl()); localContext.put(JsonLDNamespace.dvcore.getPrefix(), JsonLDNamespace.dvcore.getUrl()); localContext.put(JsonLDNamespace.schema.getPrefix(), JsonLDNamespace.schema.getUrl()); - - Map mdbMap = new TreeMap(); - Map dsftMap = new TreeMap(); - + List mdbList = metadataBlockSvc.listMetadataBlocks(); + for (MetadataBlock mdb : mdbList) { - if (mdb.getNamespaceUri() != null) { + boolean blockHasUri = mdb.getNamespaceUri() != null; + if (blockHasUri) { localContext.putIfAbsent(mdb.getName(), mdb.getNamespaceUri()); - mdbMap.put(mdb.getName(), mdb); - for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { - dsftMap.put(mdb.getName() + ":" + dsft.getName(), dsft); - } - } else { - for (DatasetFieldType dftp : mdb.getDatasetFieldTypes()) { - if (dftp.getUri() != null) { - localContext.putIfAbsent(dftp.getName(), dftp.getUri()); - } + } + for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { + if (dsft.getUri() != null) { + localContext.putIfAbsent(dsft.getName(), dsft.getUri()); } } } + logger.fine("LocalContext keys: " + String.join(", ", localContext.keySet())); + } + } + + private static JsonObject decontextualizeJsonLD(String jsonLDString) { + logger.fine(jsonLDString); + try (StringReader rdr = new StringReader(jsonLDString)) { + + // Use JsonLd to expand/compact to localContext + JsonObject jsonld = Json.createReader(rdr).readObject(); + JsonDocument doc = JsonDocument.of(jsonld); + JsonArray array = null; + try { + array = JsonLd.expand(doc).get(); + + jsonld = array.getJsonObject(0); + logger.fine("Expanded object: " + jsonld); + return jsonld; + } catch (JsonLdError e) { + System.out.println(e.getMessage()); + return null; + } + } + } + + private static JsonObject recontextualizeJsonLD(String jsonLDString, MetadataBlockServiceBean metadataBlockSvc) { + logger.fine(jsonLDString); + try (StringReader rdr = new StringReader(jsonLDString)) { + + populateContext(metadataBlockSvc); // Use JsonLd to expand/compact to localContext JsonObject jsonld = Json.createReader(rdr).readObject(); @@ -340,4 +372,5 @@ private static JsonObject recontextualizeJsonLD(String jsonLDString, MetadataBlo } } } + } From 64af0e8384ecb94dcf337bc8a2b66fe83c216c30 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 18 Sep 2020 11:28:17 -0400 Subject: [PATCH 018/354] add modification time --- .../edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 939c8ddcfff..94c45bf78be 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -1,7 +1,11 @@ package edu.harvard.iq.dataverse.util.json; import java.io.StringReader; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -82,6 +86,10 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, versions.add(dsv); ds.setVersions(versions); + if(jsonld.containsKey(JsonLDTerm.schemaOrg("dateModified").getUrl())) { + LocalDateTime dateTime = LocalDateTime.parse(jsonld.getString(JsonLDTerm.schemaOrg("dateModified").getUrl())); + ds.setModificationTime(Timestamp.valueOf(dateTime)); + } try { logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); } catch (Exception e) { From 04ee08ae2a47edf72a760bfebbea8312d61be5e2 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 18 Sep 2020 12:08:41 -0400 Subject: [PATCH 019/354] expanded has array with 1 val - handle it --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 94c45bf78be..b3130f2f893 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -133,8 +133,8 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, // Todo - normalize object vs. array JsonValue val = jsonld.get(key); JsonArray valArray = null; - if (val instanceof JsonArray) { - if (!dsft.isAllowMultiples()) { + if (val instanceof JsonArray ) { + if((((JsonArray) val).size()> 1) && !dsft.isAllowMultiples()) { throw new BadRequestException("Array for single value notsupported: " + dsft.getName()); } else { valArray = (JsonArray) val; From c7c257382fefac53bd7374f258f9dd5a4792f5a9 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 18 Sep 2020 12:13:10 -0400 Subject: [PATCH 020/354] log compound values to start --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index b3130f2f893..bd84b755ee0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -144,6 +144,9 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, } if (dsft.isCompound()) { + logger.fine("Compound: " + dsft.getName()); + logger.fine("val: " + jsonld.get(key).toString()); + /* * List vals = parseCompoundValue(type, json, * testType); for (DatasetFieldCompoundValue dsfcv : vals) { From fc77f924b113936a4c438ca4f889f1164af16540 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 18 Sep 2020 13:54:30 -0400 Subject: [PATCH 021/354] compact with no context for decontextualize --- .../iq/dataverse/util/json/JSONLDUtil.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index bd84b755ee0..da9b6be14f8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -29,6 +29,7 @@ import edu.harvard.iq.dataverse.ControlledVocabularyValue; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; import edu.harvard.iq.dataverse.DatasetFieldServiceBean; import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetFieldValue; @@ -147,11 +148,13 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, logger.fine("Compound: " + dsft.getName()); logger.fine("val: " + jsonld.get(key).toString()); - /* - * List vals = parseCompoundValue(type, json, - * testType); for (DatasetFieldCompoundValue dsfcv : vals) { - * dsfcv.setParentDatasetField(ret); } dsf.setDatasetFieldCompoundValues(vals); - */ + /*List vals = parseCompoundValue(type, jsonld.get(key),testType); + for (DatasetFieldCompoundValue dsfcv : vals) { + dsfcv.setParentDatasetField(ret); + } + dsf.setDatasetFieldCompoundValues(vals); + */ + } else if (dsft.isControlledVocabulary()) { List vals = new LinkedList<>(); @@ -349,9 +352,10 @@ private static JsonObject decontextualizeJsonLD(String jsonLDString) { JsonArray array = null; try { array = JsonLd.expand(doc).get(); - - jsonld = array.getJsonObject(0); - logger.fine("Expanded object: " + jsonld); + jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(Json.createObjectBuilder().build())) + .get(); + //jsonld = array.getJsonObject(0); + logger.fine("Decontextualized object: " + jsonld); return jsonld; } catch (JsonLdError e) { System.out.println(e.getMessage()); From e2876449654628b4ef51907909eb0ceda77d0093 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 18 Sep 2020 17:49:16 -0400 Subject: [PATCH 022/354] handle appending and compound fields --- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../harvard/iq/dataverse/api/Dataverses.java | 2 +- .../iq/dataverse/util/json/JSONLDUtil.java | 250 ++++++++++++------ 3 files changed, 172 insertions(+), 82 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index cad3b9b2c62..2f69c8a56ca 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -632,7 +632,7 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String DataverseRequest req = createDataverseRequest(findUserOrDie()); DatasetVersion dsv = ds.getEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); - dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc); + dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, true); DatasetVersion managedVersion; if (updateDraft) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 2cf50e8a87c..73f9c7790ec 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -411,7 +411,7 @@ public Response recreateDataset(String jsonLDBody, @PathParam("identifier") Stri Dataset ds = new Dataset(); ds.setOwner(owner); - ds = JSONLDUtil.updateDatasetFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc);; + ds = JSONLDUtil.updateDatasetFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc, false); //ToDo - verify PID is one Dataverse can manage (protocol/authority/shoulder match) if(! (ds.getAuthority().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Authority))&& diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index da9b6be14f8..b8f9fc4cadf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -67,7 +67,7 @@ public static JsonObject getContext(Map contextMap) { } public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, - MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append) { DatasetVersion dsv = new DatasetVersion(); @@ -80,7 +80,7 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, throw new BadRequestException ("Cannot parse the @id '" + jsonld.getString("@id") + "'. Make sure it is in valid form - see Dataverse Native API documentation."); } - dsv = updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc); + dsv = updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append); dsv.setDataset(ds); List versions = new ArrayList<>(1); @@ -101,13 +101,22 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, } public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, String jsonLDBody, - MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append) { JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); - return updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc); + return updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append); } + /** + * + * @param dsv + * @param jsonld + * @param metadataBlockSvc + * @param datasetFieldSvc + * @param append - if append, will add new top level field values for multi-valued fields, if true and field type isn't multiple, will fail. if false will replace all value(s) for fields found in the json-ld. + * @return + */ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, JsonObject jsonld, - MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append) { populateFieldTypeMap(metadataBlockSvc); @@ -115,84 +124,37 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, List dsfl = dsv.getDatasetFields(); Map fieldByTypeMap = new HashMap(); for (DatasetField dsf : dsfl) { + if (fieldByTypeMap.containsKey(dsf.getDatasetFieldType())) { + // May have multiple values per field, but not multiple fields of one type? + logger.warning("Multiple fields of type " + dsf.getDatasetFieldType().getName()); + } fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); } TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); + for (String key : jsonld.keySet()) { if (!key.equals("@context")) { if (dsftMap.containsKey(key)) { - DatasetFieldType dsft = dsftMap.get(key); + DatasetFieldType dsft = dsftMap.get(key); DatasetField dsf = null; if (fieldByTypeMap.containsKey(dsft)) { dsf = fieldByTypeMap.get(dsft); - } else { + // If there's an existing field, we use it with append and remove it for !append + if (!append) { + dsfl.remove(dsf); + } + } + if (dsf == null) { dsf = new DatasetField(); dsfl.add(dsf); - } - dsf.setDatasetFieldType(dsft); - // Todo - normalize object vs. array - JsonValue val = jsonld.get(key); - JsonArray valArray = null; - if (val instanceof JsonArray ) { - if((((JsonArray) val).size()> 1) && !dsft.isAllowMultiples()) { - throw new BadRequestException("Array for single value notsupported: " + dsft.getName()); - } else { - valArray = (JsonArray) val; - } - } else { - valArray = Json.createArrayBuilder().add(val).build(); + dsf.setDatasetFieldType(dsft); } - if (dsft.isCompound()) { - logger.fine("Compound: " + dsft.getName()); - logger.fine("val: " + jsonld.get(key).toString()); - - /*List vals = parseCompoundValue(type, jsonld.get(key),testType); - for (DatasetFieldCompoundValue dsfcv : vals) { - dsfcv.setParentDatasetField(ret); - } - dsf.setDatasetFieldCompoundValues(vals); - */ - - } else if (dsft.isControlledVocabulary()) { - - List vals = new LinkedList<>(); - for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { - String strValue = strVal.getString(); - ControlledVocabularyValue cvv = datasetFieldSvc - .findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, strValue, true); - if (cvv == null) { - throw new BadRequestException("Unknown value for Controlled Vocab Field: " - + dsft.getName() + " : " + strValue); - } - // Only add value to the list if it is not a duplicate - if (strValue.equals("Other")) { - System.out.println("vals = " + vals + ", contains: " + vals.contains(cvv)); - } - if (!vals.contains(cvv)) { - vals.add(cvv); - cvv.setDatasetFieldType(dsft); - } - } - dsf.setControlledVocabularyValues(vals); - - } else { - List vals = new LinkedList<>(); - - for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { - String strValue = strVal.getString(); - - DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); - if (valArray.size() > 1) { - datasetFieldValue.setDisplayOrder(vals.size() - 1); - } - datasetFieldValue.setValue(strValue.trim()); - vals.add(datasetFieldValue); - } - dsf.setDatasetFieldValues(vals); - - } + // Todo - normalize object vs. array + JsonArray valArray = getValues(jsonld.get(key), dsft.isAllowMultiples(), dsft.getName()); + + addField(dsf, valArray, dsft, datasetFieldSvc, append); // assemble new terms, add to existing // multivalue? @@ -214,24 +176,24 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, // (JsonLDTerm.schemaOrg("dateModified").getLabel()) // Todo - handle non-CC0 licenses, without terms as an alternate field. - if(key.equals(JsonLDTerm.schemaOrg("datePublished").getLabel())) { + if (key.equals(JsonLDTerm.schemaOrg("datePublished").getLabel())) { dsv.setVersionState(VersionState.RELEASED); - } else if(key.equals(JsonLDTerm.schemaOrg("version").getLabel())) { - String friendlyVersion = jsonld.getString(JsonLDTerm.schemaOrg("version").getLabel()); - int index = friendlyVersion.indexOf("."); - if(index>0) { - dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); - dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index+1))); - } - } else if (key.equals(JsonLDTerm.schemaOrg("license").getLabel())) { + } else if (key.equals(JsonLDTerm.schemaOrg("version").getLabel())) { + String friendlyVersion = jsonld.getString(JsonLDTerm.schemaOrg("version").getLabel()); + int index = friendlyVersion.indexOf("."); + if (index > 0) { + dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); + dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index + 1))); + } + } else if (key.equals(JsonLDTerm.schemaOrg("license").getLabel())) { if (jsonld.getString(JsonLDTerm.schemaOrg("license").getLabel()) .equals("https://creativecommons.org/publicdomain/zero/1.0/")) { terms.setLicense(TermsOfUseAndAccess.defaultLicense); } else { terms.setLicense(TermsOfUseAndAccess.License.NONE); } - } else if(key.equals(JsonLDTerm.termsOfUse.getLabel())) { - terms.setTermsOfUse(jsonld.getString(JsonLDTerm.termsOfUse.getLabel())); + } else if (key.equals(JsonLDTerm.termsOfUse.getLabel())) { + terms.setTermsOfUse(jsonld.getString(JsonLDTerm.termsOfUse.getLabel())); } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getLabel())) { terms.setConfidentialityDeclaration( jsonld.getString(JsonLDTerm.confidentialityDeclaration.getLabel())); @@ -293,6 +255,134 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, return dsv; } + private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldType dsft, + DatasetFieldServiceBean datasetFieldSvc, boolean append) { + + if (append && !dsft.isAllowMultiples()) { + if ((dsft.isCompound() && !dsf.getDatasetFieldCompoundValues().isEmpty()) + || (dsft.isAllowControlledVocabulary() + && !dsf.getControlledVocabularyValues().isEmpty()) + || !dsf.getDatasetFieldValues().isEmpty()) { + throw new BadRequestException( + "Can't append to a single-value field that already has a value: " + dsft.getName()); + } + } + + if (dsft.isCompound()) { + logger.fine("Compound: " + dsft.getName()); + logger.fine("val: " + valArray.toString()); + + /* + * List vals = parseCompoundValue(type, + * jsonld.get(key),testType); for (DatasetFieldCompoundValue dsfcv : vals) { + * dsfcv.setParentDatasetField(ret); } dsf.setDatasetFieldCompoundValues(vals); + */ + + List vals = new LinkedList<>(); + for (JsonValue val : valArray) { + if (!(val instanceof JsonObject)) { + throw new BadRequestException( + "Compound field values must be JSON objects, field: " + dsft.getName()); + } + DatasetFieldCompoundValue cv = null; + List cvList = dsf.getDatasetFieldCompoundValues(); + if (!cvList.isEmpty()) { + if (!append) { + cvList.clear(); + } else if (!dsft.isAllowMultiples() && cvList.size() == 1) { + // Trying to append but only a single value is allowed (and there already is + // one) + // (and we don't currently support appending new fields within a compound value) + throw new BadRequestException("Append with compound field with single value not yet supported: " + + dsft.getDisplayName()); + } else { + cv = cvList.get(0); + } + } + if (cv == null) { + cv = new DatasetFieldCompoundValue(); + cv.setDisplayOrder(cvList.size()); + cvList.add(cv); + cv.setParentDatasetField(dsf); + } + + JsonObject obj = (JsonObject) val; + for (String childKey : obj.keySet()) { + if (dsftMap.containsKey(childKey)) { + DatasetFieldType childft = dsftMap.get(childKey); + if (!dsft.getChildDatasetFieldTypes().contains(childft)) { + throw new BadRequestException( + "Compound field " + dsft.getName() + "can't include term " + childKey); + } + DatasetField childDsf = new DatasetField(); + cv.getChildDatasetFields().add(childDsf); + childDsf.setDatasetFieldType(childft); + childDsf.setParentDatasetFieldCompoundValue(cv); + + JsonArray childValArray = getValues(obj.get(childKey), childft.isAllowMultiples(), + childft.getName()); + addField(childDsf, childValArray, childft, datasetFieldSvc, append); + } + } + } + + } else if (dsft.isControlledVocabulary()) { + + List vals = dsf.getControlledVocabularyValues(); + for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + ControlledVocabularyValue cvv = datasetFieldSvc + .findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, strValue, true); + if (cvv == null) { + throw new BadRequestException( + "Unknown value for Controlled Vocab Field: " + dsft.getName() + " : " + strValue); + } + // Only add value to the list if it is not a duplicate + if (strValue.equals("Other")) { + System.out.println("vals = " + vals + ", contains: " + vals.contains(cvv)); + } + if (!vals.contains(cvv)) { + if (vals.size() > 0) { + cvv.setDisplayOrder(vals.size()); + } + vals.add(cvv); + cvv.setDatasetFieldType(dsft); + } + } + dsf.setControlledVocabularyValues(vals); + + } else { + List vals = dsf.getDatasetFieldValues(); + + for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + + DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); + if (vals.size() > 0) { + datasetFieldValue.setDisplayOrder(vals.size()); + } + datasetFieldValue.setValue(strValue.trim()); + vals.add(datasetFieldValue); + } + dsf.setDatasetFieldValues(vals); + + } + } + + private static JsonArray getValues(JsonValue val, boolean allowMultiples, String name) { + JsonArray valArray = null; + if (val instanceof JsonArray ) { + if((((JsonArray) val).size()> 1) && !allowMultiples) { + throw new BadRequestException("Array for single value notsupported: " + name); + } else { + valArray = (JsonArray) val; + } + } else { + valArray = Json.createArrayBuilder().add(val).build(); + } + return valArray; + } + static Map localContext = new TreeMap(); static Map dsftMap = new TreeMap(); From 8596ac80ef206f9141cfc9815c0e98ccf718b327 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sun, 20 Sep 2020 10:57:53 -0400 Subject: [PATCH 023/354] sort compound field children by display order --- .../edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java index 221922ea004..7d1e823976e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java @@ -9,6 +9,7 @@ import edu.harvard.iq.dataverse.util.MarkupChecker; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.LinkedHashMap; @@ -95,9 +96,10 @@ public void setParentDatasetField(DatasetField parentDatasetField) { public List getChildDatasetFields() { return childDatasetFields; } - + public void setChildDatasetFields(List childDatasetFields) { this.childDatasetFields = childDatasetFields; + this.childDatasetFields.sort(DatasetField.DisplayOrder); } @Override From ffbc05ad7422cbf5080c0f576986dd494946d935 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 22 Sep 2020 12:15:03 -0400 Subject: [PATCH 024/354] parse date/time correctly --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index b8f9fc4cadf..120cda54ed1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -4,6 +4,7 @@ import java.sql.Timestamp; import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -57,6 +58,8 @@ public class JSONLDUtil { return context; } */ + + private static DateTimeFormatter dateToMillis = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss. SSS"); public static JsonObject getContext(Map contextMap) { JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); @@ -88,7 +91,8 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, ds.setVersions(versions); if(jsonld.containsKey(JsonLDTerm.schemaOrg("dateModified").getUrl())) { - LocalDateTime dateTime = LocalDateTime.parse(jsonld.getString(JsonLDTerm.schemaOrg("dateModified").getUrl())); + + LocalDateTime dateTime = LocalDateTime.parse(jsonld.getString(JsonLDTerm.schemaOrg("dateModified").getUrl()), dateToMillis); ds.setModificationTime(Timestamp.valueOf(dateTime)); } try { From 0226b0d3c6cb15f3b7700eb1c259db90a9c75fa4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 22 Sep 2020 15:41:53 -0400 Subject: [PATCH 025/354] Revert "sort compound field children by display order" This reverts commit 8596ac80ef206f9141cfc9815c0e98ccf718b327. --- .../edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java index 7d1e823976e..221922ea004 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java @@ -9,7 +9,6 @@ import edu.harvard.iq.dataverse.util.MarkupChecker; import java.io.Serializable; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.LinkedHashMap; @@ -96,10 +95,9 @@ public void setParentDatasetField(DatasetField parentDatasetField) { public List getChildDatasetFields() { return childDatasetFields; } - + public void setChildDatasetFields(List childDatasetFields) { this.childDatasetFields = childDatasetFields; - this.childDatasetFields.sort(DatasetField.DisplayOrder); } @Override From fad62f44203210117b968ba84e4c6428aaa2fdd8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 22 Sep 2020 15:47:29 -0400 Subject: [PATCH 026/354] typo --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 120cda54ed1..26a91f93854 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -59,7 +59,7 @@ public class JSONLDUtil { } */ - private static DateTimeFormatter dateToMillis = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss. SSS"); + private static DateTimeFormatter dateToMillis = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); public static JsonObject getContext(Map contextMap) { JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); From c6b19a9da2d4c54367021db830309cac445ad335 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 22 Sep 2020 16:10:46 -0400 Subject: [PATCH 027/354] now use Uri instead of label when matching terms --- .../iq/dataverse/util/json/JSONLDUtil.java | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 26a91f93854..82c1bc8ea27 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -180,68 +180,68 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, // (JsonLDTerm.schemaOrg("dateModified").getLabel()) // Todo - handle non-CC0 licenses, without terms as an alternate field. - if (key.equals(JsonLDTerm.schemaOrg("datePublished").getLabel())) { + if (key.equals(JsonLDTerm.schemaOrg("datePublished").getUrl())) { dsv.setVersionState(VersionState.RELEASED); - } else if (key.equals(JsonLDTerm.schemaOrg("version").getLabel())) { - String friendlyVersion = jsonld.getString(JsonLDTerm.schemaOrg("version").getLabel()); + } else if (key.equals(JsonLDTerm.schemaOrg("version").getUrl())) { + String friendlyVersion = jsonld.getString(JsonLDTerm.schemaOrg("version").getUrl()); int index = friendlyVersion.indexOf("."); if (index > 0) { dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index + 1))); } - } else if (key.equals(JsonLDTerm.schemaOrg("license").getLabel())) { - if (jsonld.getString(JsonLDTerm.schemaOrg("license").getLabel()) + } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())) { + if (jsonld.getString(JsonLDTerm.schemaOrg("license").getUrl()) .equals("https://creativecommons.org/publicdomain/zero/1.0/")) { terms.setLicense(TermsOfUseAndAccess.defaultLicense); } else { terms.setLicense(TermsOfUseAndAccess.License.NONE); } - } else if (key.equals(JsonLDTerm.termsOfUse.getLabel())) { - terms.setTermsOfUse(jsonld.getString(JsonLDTerm.termsOfUse.getLabel())); - } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getLabel())) { + } else if (key.equals(JsonLDTerm.termsOfUse.getUrl())) { + terms.setTermsOfUse(jsonld.getString(JsonLDTerm.termsOfUse.getUrl())); + } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getUrl())) { terms.setConfidentialityDeclaration( - jsonld.getString(JsonLDTerm.confidentialityDeclaration.getLabel())); - } else if (key.equals(JsonLDTerm.specialPermissions.getLabel())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.specialPermissions.getLabel())); - } else if (key.equals(JsonLDTerm.restrictions.getLabel())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.restrictions.getLabel())); - } else if (key.equals(JsonLDTerm.citationRequirements.getLabel())) { + jsonld.getString(JsonLDTerm.confidentialityDeclaration.getUrl())); + } else if (key.equals(JsonLDTerm.specialPermissions.getUrl())) { + terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.specialPermissions.getUrl())); + } else if (key.equals(JsonLDTerm.restrictions.getUrl())) { + terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.restrictions.getUrl())); + } else if (key.equals(JsonLDTerm.citationRequirements.getUrl())) { terms.setConfidentialityDeclaration( - jsonld.getString(JsonLDTerm.citationRequirements.getLabel())); - } else if (key.equals(JsonLDTerm.depositorRequirements.getLabel())) { + jsonld.getString(JsonLDTerm.citationRequirements.getUrl())); + } else if (key.equals(JsonLDTerm.depositorRequirements.getUrl())) { terms.setConfidentialityDeclaration( - jsonld.getString(JsonLDTerm.depositorRequirements.getLabel())); - } else if (key.equals(JsonLDTerm.conditions.getLabel())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.conditions.getLabel())); - } else if (key.equals(JsonLDTerm.disclaimer.getLabel())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.disclaimer.getLabel())); - } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getLabel())) { - JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getLabel()); - if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getLabel())) { - terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getLabel())); + jsonld.getString(JsonLDTerm.depositorRequirements.getUrl())); + } else if (key.equals(JsonLDTerm.conditions.getUrl())) { + terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.conditions.getUrl())); + } else if (key.equals(JsonLDTerm.disclaimer.getUrl())) { + terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.disclaimer.getUrl())); + } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { + JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); + if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getUrl())) { + terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getLabel())) { + if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getUrl())) { terms.setFileAccessRequest( - fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getLabel())); + fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getLabel())) { - terms.setDataAccessPlace(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getLabel())); + if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getUrl())) { + terms.setDataAccessPlace(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getLabel())) { - terms.setOriginalArchive(fAccessObject.getString(JsonLDTerm.originalArchive.getLabel())); + if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getUrl())) { + terms.setOriginalArchive(fAccessObject.getString(JsonLDTerm.originalArchive.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getLabel())) { + if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getUrl())) { terms.setAvailabilityStatus( - fAccessObject.getString(JsonLDTerm.availabilityStatus.getLabel())); + fAccessObject.getString(JsonLDTerm.availabilityStatus.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getLabel())) { - terms.setContactForAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getLabel())); + if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getUrl())) { + terms.setContactForAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getLabel())) { - terms.setSizeOfCollection(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getLabel())); + if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getUrl())) { + terms.setSizeOfCollection(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getLabel())) { - terms.setStudyCompletion(fAccessObject.getString(JsonLDTerm.studyCompletion.getLabel())); + if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getUrl())) { + terms.setStudyCompletion(fAccessObject.getString(JsonLDTerm.studyCompletion.getUrl())); } } dsv.setTermsOfUseAndAccess(terms); From a014fc43aa41a58c856eca3c185d55959f097585 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 22 Sep 2020 17:22:22 -0400 Subject: [PATCH 028/354] set dsfield of dsfvalue --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 82c1bc8ea27..6de33e68989 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -367,6 +367,8 @@ private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldT } datasetFieldValue.setValue(strValue.trim()); vals.add(datasetFieldValue); + datasetFieldValue.setDatasetField(dsf); + } dsf.setDatasetFieldValues(vals); From 5bb5e68e3e4ceb90054d1bd085ac4c4909854813 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 23 Sep 2020 09:21:01 -0400 Subject: [PATCH 029/354] additional debug, always set display order --- .../iq/dataverse/util/json/JSONLDUtil.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 6de33e68989..9747a22af38 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -271,11 +271,13 @@ private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldT "Can't append to a single-value field that already has a value: " + dsft.getName()); } } - + logger.fine("Name: " + dsft.getName()); + logger.fine("val: " + valArray.toString()); + logger.fine("Compound: " + dsft.isCompound()); + logger.fine("CV: " + dsft.isAllowControlledVocabulary()); + + if (dsft.isCompound()) { - logger.fine("Compound: " + dsft.getName()); - logger.fine("val: " + valArray.toString()); - /* * List vals = parseCompoundValue(type, * jsonld.get(key),testType); for (DatasetFieldCompoundValue dsfcv : vals) { @@ -360,18 +362,15 @@ private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldT for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { String strValue = strVal.getString(); - DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); - if (vals.size() > 0) { - datasetFieldValue.setDisplayOrder(vals.size()); - } + + datasetFieldValue.setDisplayOrder(vals.size()); datasetFieldValue.setValue(strValue.trim()); vals.add(datasetFieldValue); datasetFieldValue.setDatasetField(dsf); - + } dsf.setDatasetFieldValues(vals); - } } From 6a47fad40f9afcc3f98ef9acfe2bceb119ff4ffe Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 23 Sep 2020 10:10:13 -0400 Subject: [PATCH 030/354] generate URIs for child types to match current ore maps --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 9747a22af38..5c6b3ac1e13 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -403,6 +403,10 @@ private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockS dsftMap.put(dsft.getUri(), dsft); } if (blockHasUri) { + if(dsft.getParentDatasetFieldType()!=null) { + //ToDo - why not getName for child type? Would have to fix in ORE generation code and handle legacy bags + dsftMap.put(mdb.getNamespaceUri() + dsft.getParentDatasetFieldType().getName() + "#" + dsft.getTitle(), dsft); + } dsftMap.put(mdb.getNamespaceUri() + dsft.getName(), dsft); } } From f34da09948bd726ea92c525d861ee6ec8b3d0fb4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 23 Sep 2020 10:12:51 -0400 Subject: [PATCH 031/354] allow oremap to work w/o modified date for debug --- .../java/edu/harvard/iq/dataverse/util/bagit/OREMap.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 43cc0fd3e00..570b4ef299a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -130,10 +130,11 @@ public JsonObject getOREMap() throws Exception { Json.createArrayBuilder().add(JsonLDTerm.ore("Aggregation").getLabel()) .add(JsonLDTerm.schemaOrg("Dataset").getLabel())) .add(JsonLDTerm.schemaOrg("version").getLabel(), version.getFriendlyVersionNumber()) - .add(JsonLDTerm.schemaOrg("name").getLabel(), version.getTitle()) - .add(JsonLDTerm.schemaOrg("dateModified").getLabel(), version.getLastUpdateTime().toString()); + .add(JsonLDTerm.schemaOrg("name").getLabel(), version.getTitle()); //Allow oremap for non-published versions (not yet implemented) addIfNotNull(aggBuilder, JsonLDTerm.schemaOrg("datePublished"), dataset.getPublicationDateFormattedYYYYMMDD()); + //Just for debugging - should always be set for real/persisted datasetversions + addIfNotNull(aggBuilder, JsonLDTerm.schemaOrg("dateModified"), version.getLastUpdateTime().toString()); TermsOfUseAndAccess terms = version.getTermsOfUseAndAccess(); if (terms.getLicense() == TermsOfUseAndAccess.License.CC0) { From 6b8bbc77a75bfb37e97acd6b4d1b057bc2f0ae88 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 23 Sep 2020 10:49:57 -0400 Subject: [PATCH 032/354] null check on date itself --- src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 570b4ef299a..53751335636 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -134,7 +134,9 @@ public JsonObject getOREMap() throws Exception { //Allow oremap for non-published versions (not yet implemented) addIfNotNull(aggBuilder, JsonLDTerm.schemaOrg("datePublished"), dataset.getPublicationDateFormattedYYYYMMDD()); //Just for debugging - should always be set for real/persisted datasetversions - addIfNotNull(aggBuilder, JsonLDTerm.schemaOrg("dateModified"), version.getLastUpdateTime().toString()); + if(version.getLastUpdateTime() != null) { + aggBuilder.add(JsonLDTerm.schemaOrg("dateModified").getLabel(), version.getLastUpdateTime().toString()); + } TermsOfUseAndAccess terms = version.getTermsOfUseAndAccess(); if (terms.getLicense() == TermsOfUseAndAccess.License.CC0) { From 8af09380bd21fe112e95af6a213f7f39eaf21970 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 23 Sep 2020 11:01:19 -0400 Subject: [PATCH 033/354] fix compound value iteration don't replace existing value - always add a new value, but, if not appending, clear the list of values to start --- .../iq/dataverse/util/json/JSONLDUtil.java | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 5c6b3ac1e13..723656467c1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -283,6 +283,18 @@ private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldT * jsonld.get(key),testType); for (DatasetFieldCompoundValue dsfcv : vals) { * dsfcv.setParentDatasetField(ret); } dsf.setDatasetFieldCompoundValues(vals); */ + List cvList = dsf.getDatasetFieldCompoundValues(); + if (!cvList.isEmpty()) { + if (!append) { + cvList.clear(); + } else if (!dsft.isAllowMultiples() && cvList.size() == 1) { + // Trying to append but only a single value is allowed (and there already is + // one) + // (and we don't currently support appending new fields within a compound value) + throw new BadRequestException("Append with compound field with single value not yet supported: " + + dsft.getDisplayName()); + } + } List vals = new LinkedList<>(); for (JsonValue val : valArray) { @@ -291,26 +303,11 @@ private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldT "Compound field values must be JSON objects, field: " + dsft.getName()); } DatasetFieldCompoundValue cv = null; - List cvList = dsf.getDatasetFieldCompoundValues(); - if (!cvList.isEmpty()) { - if (!append) { - cvList.clear(); - } else if (!dsft.isAllowMultiples() && cvList.size() == 1) { - // Trying to append but only a single value is allowed (and there already is - // one) - // (and we don't currently support appending new fields within a compound value) - throw new BadRequestException("Append with compound field with single value not yet supported: " - + dsft.getDisplayName()); - } else { - cv = cvList.get(0); - } - } - if (cv == null) { - cv = new DatasetFieldCompoundValue(); - cv.setDisplayOrder(cvList.size()); - cvList.add(cv); - cv.setParentDatasetField(dsf); - } + + cv = new DatasetFieldCompoundValue(); + cv.setDisplayOrder(cvList.size()); + cvList.add(cv); + cv.setParentDatasetField(dsf); JsonObject obj = (JsonObject) val; for (String childKey : obj.keySet()) { From 79048442a4b3421769c3d6c4c913d212ec1c6133 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 23 Sep 2020 11:12:31 -0400 Subject: [PATCH 034/354] fix ttype map for terms with no uri - use title not name as is done currently in generating the ORE map --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 723656467c1..74cb2e65976 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -403,8 +403,9 @@ private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockS if(dsft.getParentDatasetFieldType()!=null) { //ToDo - why not getName for child type? Would have to fix in ORE generation code and handle legacy bags dsftMap.put(mdb.getNamespaceUri() + dsft.getParentDatasetFieldType().getName() + "#" + dsft.getTitle(), dsft); + } else { + dsftMap.put(mdb.getNamespaceUri() + dsft.getTitle(), dsft); } - dsftMap.put(mdb.getNamespaceUri() + dsft.getName(), dsft); } } } From e4ceee3d7dc91d2f913cfc713502656faa459c1d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 23 Sep 2020 16:27:00 -0400 Subject: [PATCH 035/354] handle date format variations, including DV internal ones see note in Dataverses - using Date() versus Timestamp() causes a difference in precision and, perhaps surprisingly, a difference in the response from version.getLastUpdateTime().toString() in creating the OREmap. --- .../harvard/iq/dataverse/api/Dataverses.java | 1 + .../iq/dataverse/util/json/JSONLDUtil.java | 57 +++++++++++++++++-- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 73f9c7790ec..9a890771179 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -305,6 +305,7 @@ public Response importDataset(String jsonBody, @PathParam("identifier") String p latestVersion.setCreateTime(new Date()); } if (latestVersion.getLastUpdateTime() != null) { + //ToDo - using Date instead of Timestamp means datasets created this way don't have times with as much precision as when datasets are published, etc. latestVersion.setLastUpdateTime(new Date()); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 74cb2e65976..c6ce6b0df39 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -59,8 +59,6 @@ public class JSONLDUtil { } */ - private static DateTimeFormatter dateToMillis = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); - public static JsonObject getContext(Map contextMap) { JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); for (Entry e : contextMap.entrySet()) { @@ -91,8 +89,8 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, ds.setVersions(versions); if(jsonld.containsKey(JsonLDTerm.schemaOrg("dateModified").getUrl())) { - - LocalDateTime dateTime = LocalDateTime.parse(jsonld.getString(JsonLDTerm.schemaOrg("dateModified").getUrl()), dateToMillis); + String dateString = jsonld.getString(JsonLDTerm.schemaOrg("dateModified").getUrl()); + LocalDateTime dateTime = LocalDateTime.parse(dateString, determineDateFormat(dateString)); ds.setModificationTime(Timestamp.valueOf(dateTime)); } try { @@ -485,4 +483,55 @@ private static JsonObject recontextualizeJsonLD(String jsonLDString, MetadataBlo } } +//Modified from https://stackoverflow.com/questions/3389348/parse-any-date-in-java + + private static final Map DATE_FORMAT_REGEXPS = new HashMap() { + { + put("^\\d{8}$", "yyyyMMdd"); + put("^\\d{1,2}-\\d{1,2}-\\d{4}$", "dd-MM-yyyy"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}$", "yyyy-MM-dd"); + put("^\\d{1,2}/\\d{1,2}/\\d{4}$", "MM/dd/yyyy"); + put("^\\d{4}/\\d{1,2}/\\d{1,2}$", "yyyy/MM/dd"); + put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}$", "dd MMM yyyy"); + put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}$", "dd MMMM yyyy"); + put("^\\d{12}$", "yyyyMMddHHmm"); + put("^\\d{8}\\s\\d{4}$", "yyyyMMdd HHmm"); + put("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}$", "dd-MM-yyyy HH:mm"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}$", "yyyy-MM-dd HH:mm"); + put("^\\d{1,2}/\\d{1,2}/\\d{4}\\s\\d{1,2}:\\d{2}$", "MM/dd/yyyy HH:mm"); + put("^\\d{4}/\\d{1,2}/\\d{1,2}\\s\\d{1,2}:\\d{2}$", "yyyy/MM/dd HH:mm"); + put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}$", "dd MMM yyyy HH:mm"); + put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}$", "dd MMMM yyyy HH:mm"); + put("^\\d{14}$", "yyyyMMddHHmmss"); + put("^\\d{8}\\s\\d{6}$", "yyyyMMdd HHmmss"); + put("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd-MM-yyyy HH:mm:ss"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}$", "yyyy-MM-dd HH:mm:ss"); + put("^\\d{1,2}/\\d{1,2}/\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "MM/dd/yyyy HH:mm:ss"); + put("^\\d{4}/\\d{1,2}/\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}$", "yyyy/MM/dd HH:mm:ss"); + put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMM yyyy HH:mm:ss"); + put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMMM yyyy HH:mm:ss"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\.\\d{3}$", "yyyy-MM-dd HH:mm:ss.SSS"); + } + }; + + /** + * Determine DateTimeFormatter pattern matching with the given date string. + * Returns null if format is unknown. You can simply extend DateUtil with more + * formats if needed. + * + * @param dateString The date string to determine the SimpleDateFormat pattern + * for. + * @return The matching SimpleDateFormat pattern, or null if format is unknown. + * @see SimpleDateFormat + */ + public static DateTimeFormatter determineDateFormat(String dateString) { + for (String regexp : DATE_FORMAT_REGEXPS.keySet()) { + if (dateString.toLowerCase().matches(regexp)) { + return DateTimeFormatter.ofPattern(DATE_FORMAT_REGEXPS.get(regexp)); + } + } + logger.warning("Unknown date format: " + dateString); + return null; // Unknown format. + } + } From f176387c627a06dc89f67f4b266da56e4d7cb979 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 23 Sep 2020 17:09:28 -0400 Subject: [PATCH 036/354] and the format in current published bags --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index c6ce6b0df39..c3b646a947f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -511,6 +511,8 @@ private static JsonObject recontextualizeJsonLD(String jsonLDString, MetadataBlo put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMM yyyy HH:mm:ss"); put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMMM yyyy HH:mm:ss"); put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\.\\d{3}$", "yyyy-MM-dd HH:mm:ss.SSS"); + put("^[a-z,A-Z]{3}\\s[a-z,A-Z]{3}\\s\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\s[a-z,A-Z]{3}\\s\\d{4}$", "EEE MMM dd HH:mm:ss zzz yyyy"); //Wed Sep 23 19:33:46 UTC 2020 + } }; From 2724d9ebb8c543ab26eb1991d8eeeae33d13f025 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 24 Sep 2020 09:55:26 -0400 Subject: [PATCH 037/354] initial endpoint to release a migrated dataset --- .../harvard/iq/dataverse/api/Datasets.java | 86 +++++++++++++++++++ .../iq/dataverse/util/json/JSONLDUtil.java | 2 +- 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 2f69c8a56ca..aa3ba257f06 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -115,6 +115,7 @@ import java.io.StringReader; import java.sql.Timestamp; import java.text.MessageFormat; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -142,6 +143,7 @@ import javax.json.stream.JsonParsingException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.BadRequestException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -1076,6 +1078,90 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S } } + @POST + @Path("{id}/actions/:releasemigrated") + @Consumes("application/json-ld") + public Response publishMigratedDataset(String jsonldBody, @PathParam("id") String id) { + try { + AuthenticatedUser user = findAuthenticatedUserOrDie(); + if (!user.isSuperuser()) { + return error(Response.Status.FORBIDDEN, "Only superusers can release migrated datasets"); + } + + Dataset ds = findDatasetOrDie(id); + try { + JsonObject metadata = JSONLDUtil.decontextualizeJsonLD(jsonldBody); + String pubDate = metadata.getString(JsonLDTerm.schemaOrg("datePublished").getUrl()); + LocalDateTime dateTime = LocalDateTime.parse(pubDate, JSONLDUtil.determineDateFormat(pubDate)); + // dataset.getPublicationDateFormattedYYYYMMDD()) + ds.setPublicationDate(Timestamp.valueOf(dateTime)); + } catch (Exception e) { + throw new BadRequestException("Unable to set publication date (" + + JsonLDTerm.schemaOrg("datePublished").getUrl() + "): " + e.getMessage()); + } + /* + * Note: The code here mirrors that in the + * edu.harvard.iq.dataverse.DatasetPage:updateCurrentVersion method. Any changes + * to the core logic (i.e. beyond updating the messaging about results) should + * be applied to the code there as well. + */ + String errorMsg = null; + String successMsg = null; + try { + CuratePublishedDatasetVersionCommand cmd = new CuratePublishedDatasetVersionCommand(ds, + createDataverseRequest(user)); + ds = commandEngine.submit(cmd); + successMsg = BundleUtil.getStringFromBundle("datasetversion.update.success"); + + // If configured, update archive copy as well + String className = settingsService.get(SettingsServiceBean.Key.ArchiverClassName.toString()); + DatasetVersion updateVersion = ds.getLatestVersion(); + AbstractSubmitToArchiveCommand archiveCommand = ArchiverUtil.createSubmitToArchiveCommand(className, + createDataverseRequest(user), updateVersion); + if (archiveCommand != null) { + // Delete the record of any existing copy since it is now out of date/incorrect + updateVersion.setArchivalCopyLocation(null); + /* + * Then try to generate and submit an archival copy. Note that running this + * command within the CuratePublishedDatasetVersionCommand was causing an error: + * "The attribute [id] of class + * [edu.harvard.iq.dataverse.DatasetFieldCompoundValue] is mapped to a primary + * key column in the database. Updates are not allowed." To avoid that, and to + * simplify reporting back to the GUI whether this optional step succeeded, I've + * pulled this out as a separate submit(). + */ + try { + updateVersion = commandEngine.submit(archiveCommand); + if (updateVersion.getArchivalCopyLocation() != null) { + successMsg = BundleUtil.getStringFromBundle("datasetversion.update.archive.success"); + } else { + successMsg = BundleUtil.getStringFromBundle("datasetversion.update.archive.failure"); + } + } catch (CommandException ex) { + successMsg = BundleUtil.getStringFromBundle("datasetversion.update.archive.failure") + " - " + + ex.toString(); + logger.severe(ex.getMessage()); + } + } + } catch (CommandException ex) { + errorMsg = BundleUtil.getStringFromBundle("datasetversion.update.failure") + " - " + ex.toString(); + logger.severe(ex.getMessage()); + } + if (errorMsg != null) { + return error(Response.Status.INTERNAL_SERVER_ERROR, errorMsg); + } else { + JsonObjectBuilder responseBld = Json.createObjectBuilder() + .add("id", ds.getId()) + .add("persistentId", ds.getGlobalId().toString()); + return Response.ok(Json.createObjectBuilder().add("status", STATUS_OK).add("status_details", successMsg) + .add("data", responseBld).build()).type(MediaType.APPLICATION_JSON).build(); + } + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + @POST @Path("{id}/move/{targetDataverseAlias}") public Response moveDataset(@PathParam("id") String id, @PathParam("targetDataverseAlias") String targetDataverseAlias, @QueryParam("forceMove") Boolean force) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index c3b646a947f..f7f55d1df7d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -437,7 +437,7 @@ private static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { } } - private static JsonObject decontextualizeJsonLD(String jsonLDString) { + public static JsonObject decontextualizeJsonLD(String jsonLDString) { logger.fine(jsonLDString); try (StringReader rdr = new StringReader(jsonLDString)) { From 7d5006d93941a6a5f89d08ed235b10d28dfc8c80 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 24 Sep 2020 12:40:08 -0400 Subject: [PATCH 038/354] create metadataOnOrig field --- scripts/api/data/metadatablocks/citation.tsv | 1 + .../iq/dataverse/util/json/JSONLDUtil.java | 68 ++++++++++++++++--- .../iq/dataverse/util/json/JsonLDTerm.java | 2 + 3 files changed, 62 insertions(+), 9 deletions(-) diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index 3aa93d67aa3..04b373cd018 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -79,6 +79,7 @@ originOfSources Origin of Sources For historical materials, information about the origin of the sources and the rules followed in establishing the sources should be specified. textbox 75 FALSE FALSE FALSE FALSE FALSE FALSE citation characteristicOfSources Characteristic of Sources Noted Assessment of characteristics and source material. textbox 76 FALSE FALSE FALSE FALSE FALSE FALSE citation accessToSources Documentation and Access to Sources Level of documentation of the original sources. textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation + metadataOnOrig Metadata on the original source of migrated datasets. textbox 78 FALSE FALSE FALSE FALSE FALSE FALSE citation #controlledVocabulary DatasetField Value identifier displayOrder subject Agricultural Sciences D01 0 subject Arts and Humanities D0 1 diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index f7f55d1df7d..126f50ecfae 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.util.json; import java.io.StringReader; +import java.io.StringWriter; import java.sql.Timestamp; import java.time.LocalDateTime; import java.time.ZoneOffset; @@ -22,6 +23,9 @@ import javax.json.JsonObjectBuilder; import javax.json.JsonString; import javax.json.JsonValue; +import javax.json.JsonWriter; +import javax.json.JsonWriterFactory; +import javax.json.stream.JsonGenerator; import javax.ws.rs.BadRequestException; import com.apicatalog.jsonld.JsonLd; import com.apicatalog.jsonld.api.JsonLdError; @@ -41,6 +45,7 @@ import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.util.bagit.OREMap; +import edu.harvard.iq.dataverse.util.json.JsonLDTerm; public class JSONLDUtil { @@ -241,6 +246,44 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getUrl())) { terms.setStudyCompletion(fAccessObject.getString(JsonLDTerm.studyCompletion.getUrl())); } + } else { + if (dsftMap.containsKey(JsonLDTerm.metadataOnOrig.getUrl())) { + DatasetFieldType dsft = dsftMap.get(JsonLDTerm.metadataOnOrig.getUrl()); + + DatasetField dsf = null; + if (fieldByTypeMap.containsKey(dsft)) { + dsf = fieldByTypeMap.get(dsft); + // If there's an existing field, we use it with append and remove it for !append + if (!append) { + dsfl.remove(dsf); + } + } + if (dsf == null) { + dsf = new DatasetField(); + dsfl.add(dsf); + dsf.setDatasetFieldType(dsft); + } + + List vals = dsf.getDatasetFieldValues(); + + JsonObject currentValue = null; + DatasetFieldValue datasetFieldValue = null; + if (vals.isEmpty()) { + datasetFieldValue = new DatasetFieldValue(); + vals.add(datasetFieldValue); + datasetFieldValue.setDatasetField(dsf); + dsf.setDatasetFieldValues(vals); + + currentValue = Json.createObjectBuilder().build(); + } else { + datasetFieldValue = vals.get(0); + JsonObject currentVal = decontextualizeJsonLD(datasetFieldValue.getValueForEdit()); + + } + currentValue.put(key, jsonld.get(key)); + JsonObject newValue = recontextualizeJsonLD(currentValue, metadataBlockSvc); + datasetFieldValue.setValue(prettyPrint(newValue)); + } } dsv.setTermsOfUseAndAccess(terms); // move to new dataverse? @@ -459,29 +502,36 @@ public static JsonObject decontextualizeJsonLD(String jsonLDString) { } } - private static JsonObject recontextualizeJsonLD(String jsonLDString, MetadataBlockServiceBean metadataBlockSvc) { - logger.fine(jsonLDString); - try (StringReader rdr = new StringReader(jsonLDString)) { + private static JsonObject recontextualizeJsonLD(JsonObject jsonldObj, MetadataBlockServiceBean metadataBlockSvc) { populateContext(metadataBlockSvc); // Use JsonLd to expand/compact to localContext - JsonObject jsonld = Json.createReader(rdr).readObject(); - JsonDocument doc = JsonDocument.of(jsonld); + JsonDocument doc = JsonDocument.of(jsonldObj); JsonArray array = null; try { array = JsonLd.expand(doc).get(); - jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(JSONLDUtil.getContext(localContext))) + jsonldObj = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(JSONLDUtil.getContext(localContext))) .get(); - logger.fine("Compacted: " + jsonld); - return jsonld; + logger.fine("Compacted: " + jsonldObj.toString()); + return jsonldObj; } catch (JsonLdError e) { System.out.println(e.getMessage()); return null; } } - } + + public static String prettyPrint(JsonValue val) { + StringWriter sw = new StringWriter(); + Map properties = new HashMap<>(1); + properties.put(JsonGenerator.PRETTY_PRINTING, true); + JsonWriterFactory writerFactory = Json.createWriterFactory(properties); + JsonWriter jsonWriter = writerFactory.createWriter(sw); + jsonWriter.write(val); + jsonWriter.close(); + return sw.toString(); + } //Modified from https://stackoverflow.com/questions/3389348/parse-any-date-in-java diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java index 20aeceda7de..18e865b2f4d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java @@ -45,6 +45,8 @@ public class JsonLDTerm { public static JsonLDTerm totalSize = JsonLDTerm.DVCore("totalSize"); public static JsonLDTerm fileCount = JsonLDTerm.DVCore("fileCount"); public static JsonLDTerm maxFileSize = JsonLDTerm.DVCore("maxFileSize"); + + public static JsonLDTerm metadataOnOrig = JsonLDTerm.DVCore("metadataOnOrig"); public JsonLDTerm(JsonLDNamespace namespace, String term) { this.namespace = namespace; From 925070b4bbc856127dadc238d163d3fbe563ebba Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 24 Sep 2020 13:43:27 -0400 Subject: [PATCH 039/354] add metadataOnOrig to solr --- conf/solr/7.7.2/schema_dv_mdb_copies.xml | 1 + conf/solr/7.7.2/schema_dv_mdb_fields.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/conf/solr/7.7.2/schema_dv_mdb_copies.xml b/conf/solr/7.7.2/schema_dv_mdb_copies.xml index 0208fdf3910..d63da9d6f59 100644 --- a/conf/solr/7.7.2/schema_dv_mdb_copies.xml +++ b/conf/solr/7.7.2/schema_dv_mdb_copies.xml @@ -82,6 +82,7 @@ + diff --git a/conf/solr/7.7.2/schema_dv_mdb_fields.xml b/conf/solr/7.7.2/schema_dv_mdb_fields.xml index 6caa7c6de69..f93f3add513 100644 --- a/conf/solr/7.7.2/schema_dv_mdb_fields.xml +++ b/conf/solr/7.7.2/schema_dv_mdb_fields.xml @@ -82,6 +82,7 @@ + From 005db9753c633562987a0fe5f863d1c03a99bb6d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 24 Sep 2020 16:04:12 -0400 Subject: [PATCH 040/354] use Finalize Publication command Curate is for cases with an existing published version and migrated datasets only have 1 version Also - don't want to go through Publish command since it creates new version numbers, etc. --- .../harvard/iq/dataverse/api/Datasets.java | 6 ++++-- .../FinalizeDatasetPublicationCommand.java | 19 ++++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index aa3ba257f06..b9727758644 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -52,6 +52,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.DeleteDatasetLinkingDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.DeletePrivateUrlCommand; import edu.harvard.iq.dataverse.engine.command.impl.DestroyDatasetCommand; +import edu.harvard.iq.dataverse.engine.command.impl.FinalizeDatasetPublicationCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetDatasetCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetSpecificPublishedDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetDraftDatasetVersionCommand; @@ -1108,9 +1109,10 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin String errorMsg = null; String successMsg = null; try { - CuratePublishedDatasetVersionCommand cmd = new CuratePublishedDatasetVersionCommand(ds, - createDataverseRequest(user)); + FinalizeDatasetPublicationCommand cmd = new FinalizeDatasetPublicationCommand(ds, + createDataverseRequest(user), true); ds = commandEngine.submit(cmd); + //Todo - update messages successMsg = BundleUtil.getStringFromBundle("datasetversion.update.success"); // If configured, update archive copy as well diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 92bd22eb902..fd100693d9f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -169,6 +169,8 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // above takes proper care to "clean up after itself" in case of // a failure - it will remove any locks, and it will send a // proper notification to the user(s). + } else { + updateExternalIdentifier(theDataset, ctxt); } theDataset.getLatestVersion().setVersionState(RELEASED); } @@ -200,7 +202,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { return readyDataset; } - @Override + @Override public boolean onSuccess(CommandContext ctxt, Object r) { boolean retVal = true; Dataset dataset = null; @@ -345,14 +347,18 @@ private void publicizeExternalIdentifier(Dataset dataset, CommandContext ctxt) t if (!idServiceBean.publicizeIdentifier(df)) { throw new Exception(); } - df.setGlobalIdCreateTime(getTimestamp()); + if(df.getGlobalIdCreateTime() == null) { + df.setGlobalIdCreateTime(getTimestamp()); + } df.setIdentifierRegistered(true); } } if (!idServiceBean.publicizeIdentifier(dataset)) { throw new Exception(); } - dataset.setGlobalIdCreateTime(new Date()); // TODO these two methods should be in the responsibility of the idServiceBean. + if(dataset.getGlobalIdCreateTime() == null) { + dataset.setGlobalIdCreateTime(new Date()); // TODO these two methods should be in the responsibility of the idServiceBean. + } dataset.setIdentifierRegistered(true); } catch (Throwable e) { // Send failure notification to the user: @@ -364,6 +370,13 @@ private void publicizeExternalIdentifier(Dataset dataset, CommandContext ctxt) t } } + private void updateExternalIdentifier(Dataset theDataset, CommandContext ctxt) throws CommandException { + //A placeholder - for DataCite, and Fake, at least, the publicize method works - with a change to not update the globalId create dates if they already existed + //If that is not true for other providers, or changes , we can add modifications here + publicizeExternalIdentifier(theDataset, ctxt); + + } + private void updateFiles(Timestamp updateTime, CommandContext ctxt) throws CommandException { for (DataFile dataFile : getDataset().getFiles()) { if (dataFile.getPublicationDate() == null) { From f336cfda5bcd6ad591b22b987508abaa7e370a34 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 24 Sep 2020 17:05:50 -0400 Subject: [PATCH 041/354] add debug, allow more details in 400 responses --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 2 ++ .../errorhandlers/WebApplicationExceptionHandler.java | 1 + .../iq/dataverse/api/util/JsonResponseBuilder.java | 10 ++++++++++ 3 files changed, 13 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index b9727758644..bc9b8b1c057 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1093,10 +1093,12 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin try { JsonObject metadata = JSONLDUtil.decontextualizeJsonLD(jsonldBody); String pubDate = metadata.getString(JsonLDTerm.schemaOrg("datePublished").getUrl()); + logger.fine("Submitted date: " + pubDate); LocalDateTime dateTime = LocalDateTime.parse(pubDate, JSONLDUtil.determineDateFormat(pubDate)); // dataset.getPublicationDateFormattedYYYYMMDD()) ds.setPublicationDate(Timestamp.valueOf(dateTime)); } catch (Exception e) { + logger.fine(e.getMessage()); throw new BadRequestException("Unable to set publication date (" + JsonLDTerm.schemaOrg("datePublished").getUrl() + "): " + e.getMessage()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java index b6229e58192..b8de74db5e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java @@ -47,6 +47,7 @@ public Response toResponse(WebApplicationException ex) { jrb.message(BundleUtil.getStringFromBundle("access.api.exception.metadata.not.available.for.nontabular.file")); } else { jrb.message("Bad Request. The API request cannot be completed with the parameters supplied. Please check your code for typos, or consult our API guide at http://guides.dataverse.org."); + jrb.details(ex.getMessage()); jrb.request(request); } break; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java b/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java index d3c6fd2df50..881d6c79872 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java @@ -79,6 +79,16 @@ public JsonResponseBuilder message(String message) { return this; } + /** + * Add a detailed message to the response + * @param message A human readable message + * @return The enhanced builder + */ + public JsonResponseBuilder details(String message) { + this.entityBuilder.add("details", message); + return this; + } + /** * Set an identifier for this (usually included in logs, too). * @param id A String containing an (ideally unique) identifier From 0de70dd4f8a81fcaa74af63090436e6fbb46a85b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 24 Sep 2020 18:03:22 -0400 Subject: [PATCH 042/354] fix date-time issue --- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../iq/dataverse/util/json/JSONLDUtil.java | 197 +++++++++++------- 2 files changed, 117 insertions(+), 82 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index bc9b8b1c057..b7b37c8f6b4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1094,7 +1094,7 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin JsonObject metadata = JSONLDUtil.decontextualizeJsonLD(jsonldBody); String pubDate = metadata.getString(JsonLDTerm.schemaOrg("datePublished").getUrl()); logger.fine("Submitted date: " + pubDate); - LocalDateTime dateTime = LocalDateTime.parse(pubDate, JSONLDUtil.determineDateFormat(pubDate)); + LocalDateTime dateTime = JSONLDUtil.getDateTimeFrom(pubDate); // dataset.getPublicationDateFormattedYYYYMMDD()) ds.setPublicationDate(Timestamp.valueOf(dateTime)); } catch (Exception e) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 126f50ecfae..8c867e967ea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -3,6 +3,7 @@ import java.io.StringReader; import java.io.StringWriter; import java.sql.Timestamp; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -51,19 +52,15 @@ public class JSONLDUtil { private static final Logger logger = Logger.getLogger(JSONLDUtil.class.getCanonicalName()); -/* private static Map populateContext(JsonValue json) { - Map context = new TreeMap(); - if (json instanceof JsonArray) { - logger.warning("Array @context not yet supported"); - } else { - for (String key : ((JsonObject) json).keySet()) { - context.putIfAbsent(key, ((JsonObject) json).getString(key)); - } - } - return context; - } - */ - + /* + * private static Map populateContext(JsonValue json) { + * Map context = new TreeMap(); if (json + * instanceof JsonArray) { logger.warning("Array @context not yet supported"); } + * else { for (String key : ((JsonObject) json).keySet()) { + * context.putIfAbsent(key, ((JsonObject) json).getString(key)); } } return + * context; } + */ + public static JsonObject getContext(Map contextMap) { JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); for (Entry e : contextMap.entrySet()) { @@ -76,16 +73,17 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append) { DatasetVersion dsv = new DatasetVersion(); - + JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); Optional maybePid = GlobalId.parse(jsonld.getString("@id")); - if (maybePid.isPresent()) { - ds.setGlobalId(maybePid.get()); - } else { - // unparsable PID passed. Terminate. - throw new BadRequestException ("Cannot parse the @id '" + jsonld.getString("@id") + "'. Make sure it is in valid form - see Dataverse Native API documentation."); - } - + if (maybePid.isPresent()) { + ds.setGlobalId(maybePid.get()); + } else { + // unparsable PID passed. Terminate. + throw new BadRequestException("Cannot parse the @id '" + jsonld.getString("@id") + + "'. Make sure it is in valid form - see Dataverse Native API documentation."); + } + dsv = updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append); dsv.setDataset(ds); @@ -93,9 +91,9 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, versions.add(dsv); ds.setVersions(versions); - if(jsonld.containsKey(JsonLDTerm.schemaOrg("dateModified").getUrl())) { + if (jsonld.containsKey(JsonLDTerm.schemaOrg("dateModified").getUrl())) { String dateString = jsonld.getString(JsonLDTerm.schemaOrg("dateModified").getUrl()); - LocalDateTime dateTime = LocalDateTime.parse(dateString, determineDateFormat(dateString)); + LocalDateTime dateTime = getDateTimeFrom(dateString); ds.setModificationTime(Timestamp.valueOf(dateTime)); } try { @@ -119,7 +117,10 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, * @param jsonld * @param metadataBlockSvc * @param datasetFieldSvc - * @param append - if append, will add new top level field values for multi-valued fields, if true and field type isn't multiple, will fail. if false will replace all value(s) for fields found in the json-ld. + * @param append - if append, will add new top level field values for + * multi-valued fields, if true and field type isn't + * multiple, will fail. if false will replace all + * value(s) for fields found in the json-ld. * @return */ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, JsonObject jsonld, @@ -138,7 +139,7 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); } TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); - + for (String key : jsonld.keySet()) { if (!key.equals("@context")) { if (dsftMap.containsKey(key)) { @@ -160,7 +161,7 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, // Todo - normalize object vs. array JsonArray valArray = getValues(jsonld.get(key), dsft.isAllowMultiples(), dsft.getName()); - + addField(dsf, valArray, dsft, datasetFieldSvc, append); // assemble new terms, add to existing @@ -209,8 +210,7 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, } else if (key.equals(JsonLDTerm.restrictions.getUrl())) { terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.restrictions.getUrl())); } else if (key.equals(JsonLDTerm.citationRequirements.getUrl())) { - terms.setConfidentialityDeclaration( - jsonld.getString(JsonLDTerm.citationRequirements.getUrl())); + terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.citationRequirements.getUrl())); } else if (key.equals(JsonLDTerm.depositorRequirements.getUrl())) { terms.setConfidentialityDeclaration( jsonld.getString(JsonLDTerm.depositorRequirements.getUrl())); @@ -224,8 +224,7 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getUrl())); } if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getUrl())) { - terms.setFileAccessRequest( - fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getUrl())); + terms.setFileAccessRequest(fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getUrl())); } if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getUrl())) { terms.setDataAccessPlace(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getUrl())); @@ -305,8 +304,7 @@ private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldT if (append && !dsft.isAllowMultiples()) { if ((dsft.isCompound() && !dsf.getDatasetFieldCompoundValues().isEmpty()) - || (dsft.isAllowControlledVocabulary() - && !dsf.getControlledVocabularyValues().isEmpty()) + || (dsft.isAllowControlledVocabulary() && !dsf.getControlledVocabularyValues().isEmpty()) || !dsf.getDatasetFieldValues().isEmpty()) { throw new BadRequestException( "Can't append to a single-value field that already has a value: " + dsft.getName()); @@ -316,8 +314,7 @@ private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldT logger.fine("val: " + valArray.toString()); logger.fine("Compound: " + dsft.isCompound()); logger.fine("CV: " + dsft.isAllowControlledVocabulary()); - - + if (dsft.isCompound()) { /* * List vals = parseCompoundValue(type, @@ -332,8 +329,8 @@ private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldT // Trying to append but only a single value is allowed (and there already is // one) // (and we don't currently support appending new fields within a compound value) - throw new BadRequestException("Append with compound field with single value not yet supported: " - + dsft.getDisplayName()); + throw new BadRequestException( + "Append with compound field with single value not yet supported: " + dsft.getDisplayName()); } } @@ -401,12 +398,12 @@ private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldT for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { String strValue = strVal.getString(); DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); - + datasetFieldValue.setDisplayOrder(vals.size()); datasetFieldValue.setValue(strValue.trim()); vals.add(datasetFieldValue); datasetFieldValue.setDatasetField(dsf); - + } dsf.setDatasetFieldValues(vals); } @@ -414,8 +411,8 @@ private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldT private static JsonArray getValues(JsonValue val, boolean allowMultiples, String name) { JsonArray valArray = null; - if (val instanceof JsonArray ) { - if((((JsonArray) val).size()> 1) && !allowMultiples) { + if (val instanceof JsonArray) { + if ((((JsonArray) val).size() > 1) && !allowMultiples) { throw new BadRequestException("Array for single value notsupported: " + name); } else { valArray = (JsonArray) val; @@ -430,8 +427,8 @@ private static JsonArray getValues(JsonValue val, boolean allowMultiples, String static Map dsftMap = new TreeMap(); private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockSvc) { - if (dsftMap.isEmpty()) { - + if (dsftMap.isEmpty()) { + List mdbList = metadataBlockSvc.listMetadataBlocks(); for (MetadataBlock mdb : mdbList) { @@ -441,27 +438,29 @@ private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockS dsftMap.put(dsft.getUri(), dsft); } if (blockHasUri) { - if(dsft.getParentDatasetFieldType()!=null) { - //ToDo - why not getName for child type? Would have to fix in ORE generation code and handle legacy bags - dsftMap.put(mdb.getNamespaceUri() + dsft.getParentDatasetFieldType().getName() + "#" + dsft.getTitle(), dsft); + if (dsft.getParentDatasetFieldType() != null) { + // ToDo - why not getName for child type? Would have to fix in ORE generation + // code and handle legacy bags + dsftMap.put(mdb.getNamespaceUri() + dsft.getParentDatasetFieldType().getName() + "#" + + dsft.getTitle(), dsft); } else { - dsftMap.put(mdb.getNamespaceUri() + dsft.getTitle(), dsft); + dsftMap.put(mdb.getNamespaceUri() + dsft.getTitle(), dsft); } } } } - logger.fine("DSFT Map: " + String.join(", ",dsftMap.keySet())); + logger.fine("DSFT Map: " + String.join(", ", dsftMap.keySet())); } } - + private static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { - if (localContext.isEmpty()) { - + if (localContext.isEmpty()) { + // Add namespaces corresponding to core terms localContext.put(JsonLDNamespace.dcterms.getPrefix(), JsonLDNamespace.dcterms.getUrl()); localContext.put(JsonLDNamespace.dvcore.getPrefix(), JsonLDNamespace.dvcore.getUrl()); localContext.put(JsonLDNamespace.schema.getPrefix(), JsonLDNamespace.schema.getUrl()); - + List mdbList = metadataBlockSvc.listMetadataBlocks(); for (MetadataBlock mdb : mdbList) { @@ -479,7 +478,7 @@ private static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { logger.fine("LocalContext keys: " + String.join(", ", localContext.keySet())); } } - + public static JsonObject decontextualizeJsonLD(String jsonLDString) { logger.fine(jsonLDString); try (StringReader rdr = new StringReader(jsonLDString)) { @@ -492,7 +491,7 @@ public static JsonObject decontextualizeJsonLD(String jsonLDString) { array = JsonLd.expand(doc).get(); jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(Json.createObjectBuilder().build())) .get(); - //jsonld = array.getJsonObject(0); + // jsonld = array.getJsonObject(0); logger.fine("Decontextualized object: " + jsonld); return jsonld; } catch (JsonLdError e) { @@ -501,41 +500,53 @@ public static JsonObject decontextualizeJsonLD(String jsonLDString) { } } } - + private static JsonObject recontextualizeJsonLD(JsonObject jsonldObj, MetadataBlockServiceBean metadataBlockSvc) { - populateContext(metadataBlockSvc); + populateContext(metadataBlockSvc); - // Use JsonLd to expand/compact to localContext - JsonDocument doc = JsonDocument.of(jsonldObj); - JsonArray array = null; - try { - array = JsonLd.expand(doc).get(); - - jsonldObj = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(JSONLDUtil.getContext(localContext))) - .get(); - logger.fine("Compacted: " + jsonldObj.toString()); - return jsonldObj; - } catch (JsonLdError e) { - System.out.println(e.getMessage()); - return null; - } + // Use JsonLd to expand/compact to localContext + JsonDocument doc = JsonDocument.of(jsonldObj); + JsonArray array = null; + try { + array = JsonLd.expand(doc).get(); + + jsonldObj = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(JSONLDUtil.getContext(localContext))) + .get(); + logger.fine("Compacted: " + jsonldObj.toString()); + return jsonldObj; + } catch (JsonLdError e) { + System.out.println(e.getMessage()); + return null; } + } - public static String prettyPrint(JsonValue val) { - StringWriter sw = new StringWriter(); - Map properties = new HashMap<>(1); - properties.put(JsonGenerator.PRETTY_PRINTING, true); - JsonWriterFactory writerFactory = Json.createWriterFactory(properties); - JsonWriter jsonWriter = writerFactory.createWriter(sw); - jsonWriter.write(val); - jsonWriter.close(); - return sw.toString(); - } + public static String prettyPrint(JsonValue val) { + StringWriter sw = new StringWriter(); + Map properties = new HashMap<>(1); + properties.put(JsonGenerator.PRETTY_PRINTING, true); + JsonWriterFactory writerFactory = Json.createWriterFactory(properties); + JsonWriter jsonWriter = writerFactory.createWriter(sw); + jsonWriter.write(val); + jsonWriter.close(); + return sw.toString(); + } //Modified from https://stackoverflow.com/questions/3389348/parse-any-date-in-java private static final Map DATE_FORMAT_REGEXPS = new HashMap() { + { + put("^\\d{8}$", "yyyyMMdd"); + put("^\\d{1,2}-\\d{1,2}-\\d{4}$", "dd-MM-yyyy"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}$", "yyyy-MM-dd"); + put("^\\d{1,2}/\\d{1,2}/\\d{4}$", "MM/dd/yyyy"); + put("^\\d{4}/\\d{1,2}/\\d{1,2}$", "yyyy/MM/dd"); + put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}$", "dd MMM yyyy"); + put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}$", "dd MMMM yyyy"); + } + }; + + private static final Map DATETIME_FORMAT_REGEXPS = new HashMap() { { put("^\\d{8}$", "yyyyMMdd"); put("^\\d{1,2}-\\d{1,2}-\\d{4}$", "dd-MM-yyyy"); @@ -561,8 +572,9 @@ public static String prettyPrint(JsonValue val) { put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMM yyyy HH:mm:ss"); put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMMM yyyy HH:mm:ss"); put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\.\\d{3}$", "yyyy-MM-dd HH:mm:ss.SSS"); - put("^[a-z,A-Z]{3}\\s[a-z,A-Z]{3}\\s\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\s[a-z,A-Z]{3}\\s\\d{4}$", "EEE MMM dd HH:mm:ss zzz yyyy"); //Wed Sep 23 19:33:46 UTC 2020 - + put("^[a-z,A-Z]{3}\\s[a-z,A-Z]{3}\\s\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\s[a-z,A-Z]{3}\\s\\d{4}$", + "EEE MMM dd HH:mm:ss zzz yyyy"); // Wed Sep 23 19:33:46 UTC 2020 + } }; @@ -576,6 +588,16 @@ public static String prettyPrint(JsonValue val) { * @return The matching SimpleDateFormat pattern, or null if format is unknown. * @see SimpleDateFormat */ + public static DateTimeFormatter determineDateTimeFormat(String dateString) { + for (String regexp : DATETIME_FORMAT_REGEXPS.keySet()) { + if (dateString.toLowerCase().matches(regexp)) { + return DateTimeFormatter.ofPattern(DATE_FORMAT_REGEXPS.get(regexp)); + } + } + logger.warning("Unknown datetime format: " + dateString); + return null; // Unknown format. + } + public static DateTimeFormatter determineDateFormat(String dateString) { for (String regexp : DATE_FORMAT_REGEXPS.keySet()) { if (dateString.toLowerCase().matches(regexp)) { @@ -586,4 +608,17 @@ public static DateTimeFormatter determineDateFormat(String dateString) { return null; // Unknown format. } + public static LocalDateTime getDateTimeFrom(String dateString) { + DateTimeFormatter dtf = determineDateTimeFormat(dateString); + if (dtf != null) { + return LocalDateTime.parse(dateString, dtf); + } else { + dtf = determineDateFormat(dateString); + if (dtf != null) { + return LocalDate.parse(dateString, dtf).atStartOfDay(); + } + } + + return null; + } } From 395bb71ee512fc3e969ebd48a0e2b0f631617d80 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 25 Sep 2020 09:26:39 -0400 Subject: [PATCH 043/354] typos --- .../edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 8c867e967ea..39026b131bd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -548,13 +548,6 @@ public static String prettyPrint(JsonValue val) { private static final Map DATETIME_FORMAT_REGEXPS = new HashMap() { { - put("^\\d{8}$", "yyyyMMdd"); - put("^\\d{1,2}-\\d{1,2}-\\d{4}$", "dd-MM-yyyy"); - put("^\\d{4}-\\d{1,2}-\\d{1,2}$", "yyyy-MM-dd"); - put("^\\d{1,2}/\\d{1,2}/\\d{4}$", "MM/dd/yyyy"); - put("^\\d{4}/\\d{1,2}/\\d{1,2}$", "yyyy/MM/dd"); - put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}$", "dd MMM yyyy"); - put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}$", "dd MMMM yyyy"); put("^\\d{12}$", "yyyyMMddHHmm"); put("^\\d{8}\\s\\d{4}$", "yyyyMMdd HHmm"); put("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}$", "dd-MM-yyyy HH:mm"); @@ -591,7 +584,7 @@ public static String prettyPrint(JsonValue val) { public static DateTimeFormatter determineDateTimeFormat(String dateString) { for (String regexp : DATETIME_FORMAT_REGEXPS.keySet()) { if (dateString.toLowerCase().matches(regexp)) { - return DateTimeFormatter.ofPattern(DATE_FORMAT_REGEXPS.get(regexp)); + return DateTimeFormatter.ofPattern(DATETIME_FORMAT_REGEXPS.get(regexp)); } } logger.warning("Unknown datetime format: " + dateString); From 61c0349061e54548d877cd37403d0aeebf1d19f7 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 25 Sep 2020 13:00:54 -0400 Subject: [PATCH 044/354] create transfer bag type with orig files handle no checksums on orig files --- .../edu/harvard/iq/dataverse/api/Admin.java | 4 +- .../impl/AbstractSubmitToArchiveCommand.java | 8 ++- .../impl/DuraCloudSubmitToArchiveCommand.java | 6 +- .../impl/LocalSubmitToArchiveCommand.java | 6 +- .../iq/dataverse/export/OAI_OREExporter.java | 2 +- .../iq/dataverse/util/ArchiverUtil.java | 9 ++- .../iq/dataverse/util/bagit/BagGenerator.java | 35 +++++++----- .../iq/dataverse/util/bagit/OREMap.java | 56 +++++++++++++------ .../iq/dataverse/util/json/JSONLDUtil.java | 3 +- 9 files changed, 85 insertions(+), 44 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 226f39d0f52..0b8f3244007 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1671,7 +1671,7 @@ public Response validateDataFileHashValue(@PathParam("fileId") String fileId) { @GET @Path("/submitDataVersionToArchive/{id}/{version}") - public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, @PathParam("version") String versionNumber) { + public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, @PathParam("version") String versionNumber, @QueryParam(value = "type") String type) { try { AuthenticatedUser au = findAuthenticatedUserOrDie(); @@ -1685,7 +1685,7 @@ public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, @Pat DatasetVersion dv = datasetversionService.findByFriendlyVersionNumber(ds.getId(), versionNumber); if (dv.getArchivalCopyLocation() == null) { String className = settingsService.getValueForKey(SettingsServiceBean.Key.ArchiverClassName); - AbstractSubmitToArchiveCommand cmd = ArchiverUtil.createSubmitToArchiveCommand(className, dvRequestService.getDataverseRequest(), dv); + AbstractSubmitToArchiveCommand cmd = ArchiverUtil.createSubmitToArchiveCommand(className, type, dvRequestService.getDataverseRequest(), dv); if (cmd != null) { new Thread(new Runnable() { public void run() { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java index 77ea680598f..75bd6b50149 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java @@ -22,12 +22,14 @@ public abstract class AbstractSubmitToArchiveCommand extends AbstractCommand { private final DatasetVersion version; + private final String type; private final Map requestedSettings = new HashMap(); private static final Logger logger = Logger.getLogger(AbstractSubmitToArchiveCommand.class.getName()); - public AbstractSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { + public AbstractSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version, String type) { super(aRequest, version.getDataset()); this.version = version; + this.type = type; } @Override @@ -73,4 +75,8 @@ public String describe() { + version.getFriendlyVersionNumber()+")]"; } + public String getType() { + return type; + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java index 66e8770a641..9a031b64684 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java @@ -42,8 +42,8 @@ public class DuraCloudSubmitToArchiveCommand extends AbstractSubmitToArchiveComm private static final String DURACLOUD_HOST = ":DuraCloudHost"; private static final String DURACLOUD_CONTEXT = ":DuraCloudContext"; - public DuraCloudSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { - super(aRequest, version); + public DuraCloudSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version, String type) { + super(aRequest, version, type); } @Override @@ -122,7 +122,7 @@ public void run() { public void run() { try (PipedOutputStream out = new PipedOutputStream(in)){ // Generate bag - BagGenerator bagger = new BagGenerator(new OREMap(dv, false), dataciteXml); + BagGenerator bagger = new BagGenerator(new OREMap(dv, false, getType()), dataciteXml, getType()); bagger.setAuthenticationKey(token.getTokenString()); bagger.generateBag(out); } catch (Exception e) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java index 0a1de25bed0..091d5f364de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java @@ -42,8 +42,8 @@ public class LocalSubmitToArchiveCommand extends AbstractSubmitToArchiveCommand private static final Logger logger = Logger.getLogger(LocalSubmitToArchiveCommand.class.getName()); - public LocalSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { - super(aRequest, version); + public LocalSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version, String type) { + super(aRequest, version, type); } @Override @@ -69,7 +69,7 @@ public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken t FileUtils.writeStringToFile(new File(localPath+"/"+spaceName + "-datacite.v" + dv.getFriendlyVersionNumber()+".xml"), dataciteXml); - BagGenerator bagger = new BagGenerator(new OREMap(dv, false), dataciteXml); + BagGenerator bagger = new BagGenerator(new OREMap(dv, false, getType()), dataciteXml, getType()); bagger.setAuthenticationKey(token.getTokenString()); bagger.generateBag(new FileOutputStream(localPath+"/"+spaceName + "v" + dv.getFriendlyVersionNumber() + ".zip")); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java index 6cfcb590681..80eea321fa5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java @@ -25,7 +25,7 @@ public class OAI_OREExporter implements Exporter { public void exportDataset(DatasetVersion version, JsonObject json, OutputStream outputStream) throws ExportException { try { - new OREMap(version, ExportService.settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)).writeOREMap(outputStream); + new OREMap(version, ExportService.settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false), OREMap.ARCHIVE).writeOREMap(outputStream); } catch (Exception e) { logger.severe(e.getMessage()); e.printStackTrace(); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java index fc97f972f5c..43bd89e3473 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java @@ -20,13 +20,18 @@ public ArchiverUtil() { } public static AbstractSubmitToArchiveCommand createSubmitToArchiveCommand(String className, DataverseRequest dvr, DatasetVersion version) { + return createSubmitToArchiveCommand(className, null, dvr, version); + } + + public static AbstractSubmitToArchiveCommand createSubmitToArchiveCommand(String className, String type, + DataverseRequest dvr, DatasetVersion version) { if (className != null) { try { Class clazz = Class.forName(className); if (AbstractSubmitToArchiveCommand.class.isAssignableFrom(clazz)) { Constructor ctor; ctor = clazz.getConstructor(DataverseRequest.class, DatasetVersion.class); - return (AbstractSubmitToArchiveCommand) ctor.newInstance(new Object[] { dvr, version }); + return (AbstractSubmitToArchiveCommand) ctor.newInstance(new Object[] { dvr, version, type }); } } catch (Exception e) { logger.warning("Unable to instantiate an Archiver of class: " + className); @@ -34,5 +39,5 @@ public static AbstractSubmitToArchiveCommand createSubmitToArchiveCommand(String } } return null; - } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java index cc8977c24f7..a939a14535c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java @@ -136,11 +136,12 @@ public class BagGenerator { * and zipping are done in parallel, using a connection pool. The required space * on disk is ~ n+1/n of the final bag size, e.g. 125% of the bag size for a * 4-way parallel zip operation. + * @param type * @throws Exception * @throws JsonSyntaxException */ - public BagGenerator(OREMap oreMap, String dataciteXml) throws JsonSyntaxException, Exception { + public BagGenerator(OREMap oreMap, String dataciteXml, String type) throws JsonSyntaxException, Exception { this.oremap = oreMap; this.oremapObject = oreMap.getOREMap(); //(JsonObject) new JsonParser().parse(oreMap.getOREMap().toString()); @@ -178,7 +179,11 @@ public BagGenerator(OREMap oreMap, String dataciteXml) throws JsonSyntaxExceptio public void setIgnoreHashes(boolean val) { ignorehashes = val; } - + + public void setDefaultCheckSumType(ChecksumType type) { + hashtype=type; + } + public static void println(String s) { System.out.println(s); System.out.flush(); @@ -532,18 +537,22 @@ private void processContainer(JsonObject item, String currentPath) throws IOExce if (child.has(JsonLDTerm.checksum.getLabel())) { ChecksumType childHashType = ChecksumType.fromString( child.getAsJsonObject(JsonLDTerm.checksum.getLabel()).get("@type").getAsString()); - if (hashtype != null && !hashtype.equals(childHashType)) { - logger.warning("Multiple hash values in use - not supported"); - } - if (hashtype == null) + if (hashtype == null) { + //If one wasn't set as a default, pick up what the first child with one uses hashtype = childHashType; - childHash = child.getAsJsonObject(JsonLDTerm.checksum.getLabel()).get("@value").getAsString(); - if (checksumMap.containsValue(childHash)) { - // Something else has this hash - logger.warning("Duplicate/Collision: " + child.get("@id").getAsString() + " has SHA1 Hash: " - + childHash); - } - checksumMap.put(childPath, childHash); + } + if (hashtype != null && !hashtype.equals(childHashType)) { + logger.warning("Multiple hash values in use - will calculate " + hashtype.toString() + + " hashes for " + childTitle); + } else { + childHash = child.getAsJsonObject(JsonLDTerm.checksum.getLabel()).get("@value").getAsString(); + if (checksumMap.containsValue(childHash)) { + // Something else has this hash + logger.warning("Duplicate/Collision: " + child.get("@id").getAsString() + " has SHA1 Hash: " + + childHash); + } + checksumMap.put(childPath, childHash); + } } if ((hashtype == null) | ignorehashes) { // Pick sha512 when ignoring hashes or none exist diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 53751335636..f6c064183c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -9,6 +9,7 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand; import edu.harvard.iq.dataverse.export.OAI_OREExporter; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -33,13 +34,18 @@ public class OREMap { public static final String NAME = "OREMap"; + public static final String TRANSFER = "transfer"; + public static final String ARCHIVE = "archive"; + private Map localContext = new TreeMap(); private DatasetVersion version; + private String type; private boolean excludeEmail = false; - public OREMap(DatasetVersion version, boolean excludeEmail) { + public OREMap(DatasetVersion version, boolean excludeEmail, String type) { this.version = version; this.excludeEmail = excludeEmail; + this.type=type; } public void writeOREMap(OutputStream outputStream) throws Exception { @@ -207,31 +213,45 @@ public JsonObject getOREMap() throws Exception { if (df.getGlobalId().asString().length() != 0) { fileId = df.getGlobalId().asString(); fileSameAs = SystemConfig.getDataverseSiteUrlStatic() - + "/api/access/datafile/:persistentId?persistentId=" + fileId; + + "/api/access/datafile/:persistentId?persistentId=" + fileId + (type.equals(TRANSFER) ? "&format=original" : ""); } else { fileId = SystemConfig.getDataverseSiteUrlStatic() + "/file.xhtml?fileId=" + df.getId(); - fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId(); + fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId() + (type.equals(TRANSFER) ? "?format=original" : ""); } aggRes.add("@id", fileId); aggRes.add(JsonLDTerm.schemaOrg("sameAs").getLabel(), fileSameAs); fileArray.add(fileId); aggRes.add("@type", JsonLDTerm.ore("AggregatedResource").getLabel()); - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), df.getContentType()); - addIfNotNull(aggRes, JsonLDTerm.filesize, df.getFilesize()); - addIfNotNull(aggRes, JsonLDTerm.storageIdentifier, df.getStorageIdentifier()); - addIfNotNull(aggRes, JsonLDTerm.originalFileFormat, df.getOriginalFileFormat()); - addIfNotNull(aggRes, JsonLDTerm.originalFormatLabel, df.getOriginalFormatLabel()); - addIfNotNull(aggRes, JsonLDTerm.UNF, df.getUnf()); - addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId()); - addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId()); - JsonObject checksum = null; - // Add checksum. RDA recommends SHA-512 - if (df.getChecksumType() != null && df.getChecksumValue() != null) { - checksum = Json.createObjectBuilder().add("@type", df.getChecksumType().toString()) - .add("@value", df.getChecksumValue()).build(); - aggRes.add(JsonLDTerm.checksum.getLabel(), checksum); - } + switch (type) { + case TRANSFER: + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), df.getOriginalFileFormat()); + addIfNotNull(aggRes, JsonLDTerm.filesize, df.getOriginalFileSize()); + addIfNotNull(aggRes, JsonLDTerm.storageIdentifier, df.getStorageIdentifier()); + addIfNotNull(aggRes, JsonLDTerm.UNF, df.getUnf()); + addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId()); + addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId()); + //Checksum not available - will be generated in Bag + + break; + case ARCHIVE: + default: + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), df.getContentType()); + addIfNotNull(aggRes, JsonLDTerm.filesize, df.getFilesize()); + addIfNotNull(aggRes, JsonLDTerm.storageIdentifier, df.getStorageIdentifier()); + addIfNotNull(aggRes, JsonLDTerm.originalFileFormat, df.getOriginalFileFormat()); + addIfNotNull(aggRes, JsonLDTerm.originalFormatLabel, df.getOriginalFormatLabel()); + addIfNotNull(aggRes, JsonLDTerm.UNF, df.getUnf()); + addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId()); + addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId()); + JsonObject checksum = null; + // Add checksum. RDA recommends SHA-512 + if (df.getChecksumType() != null && df.getChecksumValue() != null) { + checksum = Json.createObjectBuilder().add("@type", df.getChecksumType().toString()) + .add("@value", df.getChecksumValue()).build(); + aggRes.add(JsonLDTerm.checksum.getLabel(), checksum); + } + } JsonArray tabTags = null; JsonArrayBuilder jab = JsonPrinter.getTabularFileTags(df); if (jab != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 39026b131bd..f83826e7607 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -44,6 +44,7 @@ import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.util.json.JsonLDTerm; @@ -97,7 +98,7 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, ds.setModificationTime(Timestamp.valueOf(dateTime)); } try { - logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); + logger.fine("Output dsv: " + new OREMap(dsv, false, OREMap.TRANSFER).getOREMap().toString()); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); From 1753257836e34ccb0017a0b8a3b58f39d4aaa18d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 30 Sep 2020 16:39:04 -0400 Subject: [PATCH 045/354] missing tab --- scripts/api/data/metadatablocks/citation.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index 04b373cd018..710f0ff4267 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -79,7 +79,7 @@ originOfSources Origin of Sources For historical materials, information about the origin of the sources and the rules followed in establishing the sources should be specified. textbox 75 FALSE FALSE FALSE FALSE FALSE FALSE citation characteristicOfSources Characteristic of Sources Noted Assessment of characteristics and source material. textbox 76 FALSE FALSE FALSE FALSE FALSE FALSE citation accessToSources Documentation and Access to Sources Level of documentation of the original sources. textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation - metadataOnOrig Metadata on the original source of migrated datasets. textbox 78 FALSE FALSE FALSE FALSE FALSE FALSE citation + metadataOnOrig Metadata on the original source of migrated datasets. textbox 78 FALSE FALSE FALSE FALSE FALSE FALSE citation #controlledVocabulary DatasetField Value identifier displayOrder subject Agricultural Sciences D01 0 subject Arts and Humanities D0 1 From e642c65c3490c1bfd22058c9a3321a6f115426fc Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 30 Sep 2020 17:20:20 -0400 Subject: [PATCH 046/354] add type param --- src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java index 43bd89e3473..72e5fd57593 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java @@ -30,7 +30,7 @@ public static AbstractSubmitToArchiveCommand createSubmitToArchiveCommand(String Class clazz = Class.forName(className); if (AbstractSubmitToArchiveCommand.class.isAssignableFrom(clazz)) { Constructor ctor; - ctor = clazz.getConstructor(DataverseRequest.class, DatasetVersion.class); + ctor = clazz.getConstructor(DataverseRequest.class, DatasetVersion.class, String.class); return (AbstractSubmitToArchiveCommand) ctor.newInstance(new Object[] { dvr, version, type }); } } catch (Exception e) { From 9ad779a2d0c7b74b5c4957d93bc1e2f3f7dab23f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 16 Nov 2020 14:55:12 -0500 Subject: [PATCH 047/354] add semantic metadata api call only --- .../edu/harvard/iq/dataverse/DatasetPage.java | 13 +-- .../edu/harvard/iq/dataverse/api/Admin.java | 4 +- .../harvard/iq/dataverse/api/Datasets.java | 104 +----------------- .../harvard/iq/dataverse/api/Dataverses.java | 58 ---------- .../WebApplicationExceptionHandler.java | 1 - .../api/util/JsonResponseBuilder.java | 10 -- .../impl/AbstractSubmitToArchiveCommand.java | 8 +- .../impl/DuraCloudSubmitToArchiveCommand.java | 6 +- .../FinalizeDatasetPublicationCommand.java | 19 +--- .../impl/LocalSubmitToArchiveCommand.java | 6 +- .../iq/dataverse/export/OAI_OREExporter.java | 2 +- .../iq/dataverse/util/ArchiverUtil.java | 11 +- .../iq/dataverse/util/bagit/BagGenerator.java | 3 +- .../iq/dataverse/util/bagit/OREMap.java | 67 ++++------- 14 files changed, 44 insertions(+), 268 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 7b2183a7b0d..5392371867e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1665,16 +1665,8 @@ private void updateDatasetFieldInputLevels() { Use the DatasetFieldType id's which are the Map's keys --------------------------------------------------------- */ List idList = new ArrayList<>(mapDatasetFields.keySet()); - List idStringList = new ArrayList(); - for(Long id: idList) { - idStringList.add(id.toString()); - } - logger.fine(String.join(",",idStringList)); - List dsFieldTypeInputLevels = dataverseFieldTypeInputLevelService.findByDataverseIdAndDatasetFieldTypeIdList(dvIdForInputLevel, idList); - if(dsFieldTypeInputLevels == null) { - logger.fine("dsFieldTypeInputLevels is null for ds: " + this.getId()); - } else { + /* --------------------------------------------------------- Iterate through List of DataverseFieldTypeInputLevel objects Call "setInclude" on its related DatasetField object @@ -1690,11 +1682,8 @@ private void updateDatasetFieldInputLevels() { // remove from hash mapDatasetFields.remove(oneDSFieldTypeInputLevel.getDatasetFieldType().getId()); } - } else { - logger.fine("oneDSFieldTypeInputLevel is null"); } } // end: updateDatasetFieldInputLevels - } /* --------------------------------------------------------- Iterate through any DatasetField objects remaining in the hash diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index db912413f56..e0165f45aa2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1671,7 +1671,7 @@ public Response validateDataFileHashValue(@PathParam("fileId") String fileId) { @GET @Path("/submitDataVersionToArchive/{id}/{version}") - public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, @PathParam("version") String versionNumber, @QueryParam(value = "type") String type) { + public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, @PathParam("version") String versionNumber) { try { AuthenticatedUser au = findAuthenticatedUserOrDie(); @@ -1685,7 +1685,7 @@ public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, @Pat DatasetVersion dv = datasetversionService.findByFriendlyVersionNumber(ds.getId(), versionNumber); if (dv.getArchivalCopyLocation() == null) { String className = settingsService.getValueForKey(SettingsServiceBean.Key.ArchiverClassName); - AbstractSubmitToArchiveCommand cmd = ArchiverUtil.createSubmitToArchiveCommand(className, type, dvRequestService.getDataverseRequest(), dv); + AbstractSubmitToArchiveCommand cmd = ArchiverUtil.createSubmitToArchiveCommand(className, dvRequestService.getDataverseRequest(), dv); if (cmd != null) { new Thread(new Runnable() { public void run() { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 578c0447dee..b22d707c744 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -51,7 +51,6 @@ import edu.harvard.iq.dataverse.engine.command.impl.DeleteDatasetLinkingDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.DeletePrivateUrlCommand; import edu.harvard.iq.dataverse.engine.command.impl.DestroyDatasetCommand; -import edu.harvard.iq.dataverse.engine.command.impl.FinalizeDatasetPublicationCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetDatasetCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetSpecificPublishedDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetDraftDatasetVersionCommand; @@ -102,7 +101,6 @@ import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JSONLDUtil; -import edu.harvard.iq.dataverse.util.json.JsonLDTerm; import edu.harvard.iq.dataverse.util.json.JsonParseException; import edu.harvard.iq.dataverse.search.IndexServiceBean; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; @@ -113,7 +111,6 @@ import java.io.StringReader; import java.sql.Timestamp; import java.text.MessageFormat; -import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -137,7 +134,6 @@ import javax.json.JsonReader; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.BadRequestException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -832,8 +828,7 @@ private Response processDatasetUpdate(String jsonBody, String id, DataverseReque String valdationErrors = validateDatasetFieldValues(fields); if (!valdationErrors.isEmpty()) { - logger.log(Level.SEVERE, "Semantic error parsing dataset update Json: " + valdationErrors, - valdationErrors); + logger.log(Level.SEVERE, "Semantic error parsing dataset update Json: " + valdationErrors, valdationErrors); return error(Response.Status.BAD_REQUEST, "Error parsing dataset update: " + valdationErrors); } @@ -880,8 +875,7 @@ private Response processDatasetUpdate(String jsonBody, String id, DataverseReque } dsf.setControlledVocabularyValues(priorCVV); } else { - dsf.setSingleControlledVocabularyValue( - updateField.getSingleControlledVocabularyValue()); + dsf.setSingleControlledVocabularyValue(updateField.getSingleControlledVocabularyValue()); } } else { if (!updateField.getDatasetFieldType().isCompound()) { @@ -897,8 +891,7 @@ private Response processDatasetUpdate(String jsonBody, String id, DataverseReque } } else { for (DatasetFieldCompoundValue dfcv : updateField.getDatasetFieldCompoundValues()) { - if (!dsf.getCompoundDisplayValue() - .contains(updateField.getCompoundDisplayValue())) { + if (!dsf.getCompoundDisplayValue().contains(updateField.getCompoundDisplayValue())) { dfcv.setParentDatasetField(dsf); dsf.setDatasetVersion(dsv); dsf.getDatasetFieldCompoundValues().add(dfcv); @@ -908,9 +901,7 @@ private Response processDatasetUpdate(String jsonBody, String id, DataverseReque } } else { if (!dsf.isEmpty() && !dsf.getDatasetFieldType().isAllowMultiples() || !replaceData) { - return error(Response.Status.BAD_REQUEST, - "You may not add data to a field that already has data and does not allow multiples. Use replace=true to replace existing data (" - + dsf.getDatasetFieldType().getDisplayName() + ")"); + return error(Response.Status.BAD_REQUEST, "You may not add data to a field that already has data and does not allow multiples. Use replace=true to replace existing data (" + dsf.getDatasetFieldType().getDisplayName() + ")"); } } break; @@ -1070,93 +1061,6 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S } @POST - @Path("{id}/actions/:releasemigrated") - @Consumes("application/json-ld") - public Response publishMigratedDataset(String jsonldBody, @PathParam("id") String id) { - try { - AuthenticatedUser user = findAuthenticatedUserOrDie(); - if (!user.isSuperuser()) { - return error(Response.Status.FORBIDDEN, "Only superusers can release migrated datasets"); - } - - Dataset ds = findDatasetOrDie(id); - try { - JsonObject metadata = JSONLDUtil.decontextualizeJsonLD(jsonldBody); - String pubDate = metadata.getString(JsonLDTerm.schemaOrg("datePublished").getUrl()); - logger.fine("Submitted date: " + pubDate); - LocalDateTime dateTime = JSONLDUtil.getDateTimeFrom(pubDate); - // dataset.getPublicationDateFormattedYYYYMMDD()) - ds.setPublicationDate(Timestamp.valueOf(dateTime)); - } catch (Exception e) { - logger.fine(e.getMessage()); - throw new BadRequestException("Unable to set publication date (" - + JsonLDTerm.schemaOrg("datePublished").getUrl() + "): " + e.getMessage()); - } - /* - * Note: The code here mirrors that in the - * edu.harvard.iq.dataverse.DatasetPage:updateCurrentVersion method. Any changes - * to the core logic (i.e. beyond updating the messaging about results) should - * be applied to the code there as well. - */ - String errorMsg = null; - String successMsg = null; - try { - FinalizeDatasetPublicationCommand cmd = new FinalizeDatasetPublicationCommand(ds, - createDataverseRequest(user), true); - ds = commandEngine.submit(cmd); - //Todo - update messages - successMsg = BundleUtil.getStringFromBundle("datasetversion.update.success"); - - // If configured, update archive copy as well - String className = settingsService.get(SettingsServiceBean.Key.ArchiverClassName.toString()); - DatasetVersion updateVersion = ds.getLatestVersion(); - AbstractSubmitToArchiveCommand archiveCommand = ArchiverUtil.createSubmitToArchiveCommand(className, - createDataverseRequest(user), updateVersion); - if (archiveCommand != null) { - // Delete the record of any existing copy since it is now out of date/incorrect - updateVersion.setArchivalCopyLocation(null); - /* - * Then try to generate and submit an archival copy. Note that running this - * command within the CuratePublishedDatasetVersionCommand was causing an error: - * "The attribute [id] of class - * [edu.harvard.iq.dataverse.DatasetFieldCompoundValue] is mapped to a primary - * key column in the database. Updates are not allowed." To avoid that, and to - * simplify reporting back to the GUI whether this optional step succeeded, I've - * pulled this out as a separate submit(). - */ - try { - updateVersion = commandEngine.submit(archiveCommand); - if (updateVersion.getArchivalCopyLocation() != null) { - successMsg = BundleUtil.getStringFromBundle("datasetversion.update.archive.success"); - } else { - successMsg = BundleUtil.getStringFromBundle("datasetversion.update.archive.failure"); - } - } catch (CommandException ex) { - successMsg = BundleUtil.getStringFromBundle("datasetversion.update.archive.failure") + " - " - + ex.toString(); - logger.severe(ex.getMessage()); - } - } - } catch (CommandException ex) { - errorMsg = BundleUtil.getStringFromBundle("datasetversion.update.failure") + " - " + ex.toString(); - logger.severe(ex.getMessage()); - } - if (errorMsg != null) { - return error(Response.Status.INTERNAL_SERVER_ERROR, errorMsg); - } else { - JsonObjectBuilder responseBld = Json.createObjectBuilder() - .add("id", ds.getId()) - .add("persistentId", ds.getGlobalId().toString()); - return Response.ok(Json.createObjectBuilder().add("status", STATUS_OK).add("status_details", successMsg) - .add("data", responseBld).build()).type(MediaType.APPLICATION_JSON).build(); - } - - } catch (WrappedResponse ex) { - return ex.getResponse(); - } - } - - @POST @Path("{id}/move/{targetDataverseAlias}") public Response moveDataset(@PathParam("id") String id, @PathParam("targetDataverseAlias") String targetDataverseAlias, @QueryParam("forceMove") Boolean force) { try { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 9a890771179..2c73c5ad36e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -61,8 +61,6 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.StringUtil; import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty; - -import edu.harvard.iq.dataverse.util.json.JSONLDUtil; import edu.harvard.iq.dataverse.util.json.JsonParseException; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.brief; import java.io.StringReader; @@ -86,7 +84,6 @@ import javax.json.stream.JsonParsingException; import javax.validation.ConstraintViolation; import javax.validation.ConstraintViolationException; -import javax.ws.rs.BadRequestException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -102,7 +99,6 @@ import static edu.harvard.iq.dataverse.util.json.JsonPrinter.toJsonArray; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; import java.text.MessageFormat; -import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.Optional; @@ -305,7 +301,6 @@ public Response importDataset(String jsonBody, @PathParam("identifier") String p latestVersion.setCreateTime(new Date()); } if (latestVersion.getLastUpdateTime() != null) { - //ToDo - using Date instead of Timestamp means datasets created this way don't have times with as much precision as when datasets are published, etc. latestVersion.setLastUpdateTime(new Date()); } } @@ -398,59 +393,6 @@ public Response importDatasetDdi(String xml, @PathParam("identifier") String par } } - @POST - @Path("{identifier}/datasets/:startmigration") - @Consumes("application/json-ld") - public Response recreateDataset(String jsonLDBody, @PathParam("identifier") String parentIdtf) { - try { - User u = findUserOrDie(); - if (!u.isSuperuser()) { - return error(Status.FORBIDDEN, "Not a superuser"); - } - Dataverse owner = findDataverseOrDie(parentIdtf); - - Dataset ds = new Dataset(); - - ds.setOwner(owner); - ds = JSONLDUtil.updateDatasetFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc, false); - //ToDo - verify PID is one Dataverse can manage (protocol/authority/shoulder match) - if(! - (ds.getAuthority().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Authority))&& - ds.getProtocol().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Protocol))&& - ds.getIdentifier().startsWith(settingsService.getValueForKey(SettingsServiceBean.Key.Shoulder)))) { - throw new BadRequestException("Cannot recreate a dataset that has a PID that doesn't match the server's settings"); - } - if(!datasetSvc.isIdentifierLocallyUnique(ds)) { - throw new BadRequestException("Cannot recreate a dataset whose PID is already in use"); - } - - - - if (ds.getVersions().isEmpty()) { - return badRequest("Supplied json must contain a single dataset version."); - } - - DatasetVersion version = ds.getVersions().get(0); - if (!version.isPublished()) { - throw new BadRequestException("Cannot recreate a dataset that hasn't been published."); - } - //While the datasetversion whose metadata we're importing has been published, we consider it in draft until the API caller adds files and then completes the migration - version.setVersionState(DatasetVersion.VersionState.DRAFT); - - DataverseRequest request = createDataverseRequest(u); - - Dataset managedDs = execCommand(new ImportDatasetCommand(ds, request)); - JsonObjectBuilder responseBld = Json.createObjectBuilder() - .add("id", managedDs.getId()) - .add("persistentId", managedDs.getGlobalId().toString()); - - return created("/datasets/" + managedDs.getId(), responseBld); - - } catch (WrappedResponse ex) { - return ex.getResponse(); - } - } - private Dataset parseDataset(String datasetJson) throws WrappedResponse { try (StringReader rdr = new StringReader(datasetJson)) { return jsonParser().parseDataset(Json.createReader(rdr).readObject()); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java index b8de74db5e4..b6229e58192 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java @@ -47,7 +47,6 @@ public Response toResponse(WebApplicationException ex) { jrb.message(BundleUtil.getStringFromBundle("access.api.exception.metadata.not.available.for.nontabular.file")); } else { jrb.message("Bad Request. The API request cannot be completed with the parameters supplied. Please check your code for typos, or consult our API guide at http://guides.dataverse.org."); - jrb.details(ex.getMessage()); jrb.request(request); } break; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java b/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java index 881d6c79872..d3c6fd2df50 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java @@ -79,16 +79,6 @@ public JsonResponseBuilder message(String message) { return this; } - /** - * Add a detailed message to the response - * @param message A human readable message - * @return The enhanced builder - */ - public JsonResponseBuilder details(String message) { - this.entityBuilder.add("details", message); - return this; - } - /** * Set an identifier for this (usually included in logs, too). * @param id A String containing an (ideally unique) identifier diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java index 75bd6b50149..77ea680598f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java @@ -22,14 +22,12 @@ public abstract class AbstractSubmitToArchiveCommand extends AbstractCommand { private final DatasetVersion version; - private final String type; private final Map requestedSettings = new HashMap(); private static final Logger logger = Logger.getLogger(AbstractSubmitToArchiveCommand.class.getName()); - public AbstractSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version, String type) { + public AbstractSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { super(aRequest, version.getDataset()); this.version = version; - this.type = type; } @Override @@ -75,8 +73,4 @@ public String describe() { + version.getFriendlyVersionNumber()+")]"; } - public String getType() { - return type; - } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java index b40ff4f7db4..468e99f24c1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java @@ -42,8 +42,8 @@ public class DuraCloudSubmitToArchiveCommand extends AbstractSubmitToArchiveComm private static final String DURACLOUD_HOST = ":DuraCloudHost"; private static final String DURACLOUD_CONTEXT = ":DuraCloudContext"; - public DuraCloudSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version, String type) { - super(aRequest, version, type); + public DuraCloudSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { + super(aRequest, version); } @Override @@ -127,7 +127,7 @@ public void run() { public void run() { try (PipedOutputStream out = new PipedOutputStream(in)){ // Generate bag - BagGenerator bagger = new BagGenerator(new OREMap(dv, false, getType()), dataciteXml, getType()); + BagGenerator bagger = new BagGenerator(new OREMap(dv, false), dataciteXml); bagger.setAuthenticationKey(token.getTokenString()); bagger.generateBag(out); } catch (Exception e) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 667a9dd497d..7ab83a27746 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -175,8 +175,6 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // above takes proper care to "clean up after itself" in case of // a failure - it will remove any locks, and it will send a // proper notification to the user(s). - } else { - updateExternalIdentifier(theDataset, ctxt); } theDataset.getLatestVersion().setVersionState(RELEASED); } @@ -211,7 +209,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { return readyDataset; } - @Override + @Override public boolean onSuccess(CommandContext ctxt, Object r) { boolean retVal = true; Dataset dataset = null; @@ -371,18 +369,14 @@ private void publicizeExternalIdentifier(Dataset dataset, CommandContext ctxt) t if (!idServiceBean.publicizeIdentifier(df)) { throw new Exception(); } - if(df.getGlobalIdCreateTime() == null) { - df.setGlobalIdCreateTime(getTimestamp()); - } + df.setGlobalIdCreateTime(getTimestamp()); df.setIdentifierRegistered(true); } } if (!idServiceBean.publicizeIdentifier(dataset)) { throw new Exception(); } - if(dataset.getGlobalIdCreateTime() == null) { - dataset.setGlobalIdCreateTime(new Date()); // TODO these two methods should be in the responsibility of the idServiceBean. - } + dataset.setGlobalIdCreateTime(new Date()); // TODO these two methods should be in the responsibility of the idServiceBean. dataset.setIdentifierRegistered(true); } catch (Throwable e) { logger.warning("Failed to register the identifier "+dataset.getGlobalId().asString()+", or to register a file in the dataset; notifying the user(s), unlocking the dataset"); @@ -396,13 +390,6 @@ private void publicizeExternalIdentifier(Dataset dataset, CommandContext ctxt) t } } - private void updateExternalIdentifier(Dataset theDataset, CommandContext ctxt) throws CommandException { - //A placeholder - for DataCite, and Fake, at least, the publicize method works - with a change to not update the globalId create dates if they already existed - //If that is not true for other providers, or changes , we can add modifications here - publicizeExternalIdentifier(theDataset, ctxt); - - } - private void updateFiles(Timestamp updateTime, CommandContext ctxt) throws CommandException { for (DataFile dataFile : getDataset().getFiles()) { if (dataFile.getPublicationDate() == null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java index 091d5f364de..0a1de25bed0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java @@ -42,8 +42,8 @@ public class LocalSubmitToArchiveCommand extends AbstractSubmitToArchiveCommand private static final Logger logger = Logger.getLogger(LocalSubmitToArchiveCommand.class.getName()); - public LocalSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version, String type) { - super(aRequest, version, type); + public LocalSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { + super(aRequest, version); } @Override @@ -69,7 +69,7 @@ public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken t FileUtils.writeStringToFile(new File(localPath+"/"+spaceName + "-datacite.v" + dv.getFriendlyVersionNumber()+".xml"), dataciteXml); - BagGenerator bagger = new BagGenerator(new OREMap(dv, false, getType()), dataciteXml, getType()); + BagGenerator bagger = new BagGenerator(new OREMap(dv, false), dataciteXml); bagger.setAuthenticationKey(token.getTokenString()); bagger.generateBag(new FileOutputStream(localPath+"/"+spaceName + "v" + dv.getFriendlyVersionNumber() + ".zip")); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java index 80eea321fa5..6cfcb590681 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java @@ -25,7 +25,7 @@ public class OAI_OREExporter implements Exporter { public void exportDataset(DatasetVersion version, JsonObject json, OutputStream outputStream) throws ExportException { try { - new OREMap(version, ExportService.settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false), OREMap.ARCHIVE).writeOREMap(outputStream); + new OREMap(version, ExportService.settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)).writeOREMap(outputStream); } catch (Exception e) { logger.severe(e.getMessage()); e.printStackTrace(); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java index 72e5fd57593..fc97f972f5c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java @@ -20,18 +20,13 @@ public ArchiverUtil() { } public static AbstractSubmitToArchiveCommand createSubmitToArchiveCommand(String className, DataverseRequest dvr, DatasetVersion version) { - return createSubmitToArchiveCommand(className, null, dvr, version); - } - - public static AbstractSubmitToArchiveCommand createSubmitToArchiveCommand(String className, String type, - DataverseRequest dvr, DatasetVersion version) { if (className != null) { try { Class clazz = Class.forName(className); if (AbstractSubmitToArchiveCommand.class.isAssignableFrom(clazz)) { Constructor ctor; - ctor = clazz.getConstructor(DataverseRequest.class, DatasetVersion.class, String.class); - return (AbstractSubmitToArchiveCommand) ctor.newInstance(new Object[] { dvr, version, type }); + ctor = clazz.getConstructor(DataverseRequest.class, DatasetVersion.class); + return (AbstractSubmitToArchiveCommand) ctor.newInstance(new Object[] { dvr, version }); } } catch (Exception e) { logger.warning("Unable to instantiate an Archiver of class: " + className); @@ -39,5 +34,5 @@ public static AbstractSubmitToArchiveCommand createSubmitToArchiveCommand(String } } return null; - } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java index a939a14535c..1f8d2f4f617 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java @@ -136,12 +136,11 @@ public class BagGenerator { * and zipping are done in parallel, using a connection pool. The required space * on disk is ~ n+1/n of the final bag size, e.g. 125% of the bag size for a * 4-way parallel zip operation. - * @param type * @throws Exception * @throws JsonSyntaxException */ - public BagGenerator(OREMap oreMap, String dataciteXml, String type) throws JsonSyntaxException, Exception { + public BagGenerator(OREMap oreMap, String dataciteXml) throws JsonSyntaxException, Exception { this.oremap = oreMap; this.oremapObject = oreMap.getOREMap(); //(JsonObject) new JsonParser().parse(oreMap.getOREMap().toString()); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index f6c064183c9..5520de3954e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -9,7 +9,6 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; -import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand; import edu.harvard.iq.dataverse.export.OAI_OREExporter; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -21,6 +20,7 @@ import java.time.LocalDate; import java.util.List; import java.util.Map; +import java.util.ResourceBundle; import java.util.TreeMap; import java.util.Map.Entry; @@ -34,18 +34,13 @@ public class OREMap { public static final String NAME = "OREMap"; - public static final String TRANSFER = "transfer"; - public static final String ARCHIVE = "archive"; - private Map localContext = new TreeMap(); private DatasetVersion version; - private String type; private boolean excludeEmail = false; - public OREMap(DatasetVersion version, boolean excludeEmail, String type) { + public OREMap(DatasetVersion version, boolean excludeEmail) { this.version = version; this.excludeEmail = excludeEmail; - this.type=type; } public void writeOREMap(OutputStream outputStream) throws Exception { @@ -136,13 +131,9 @@ public JsonObject getOREMap() throws Exception { Json.createArrayBuilder().add(JsonLDTerm.ore("Aggregation").getLabel()) .add(JsonLDTerm.schemaOrg("Dataset").getLabel())) .add(JsonLDTerm.schemaOrg("version").getLabel(), version.getFriendlyVersionNumber()) - .add(JsonLDTerm.schemaOrg("name").getLabel(), version.getTitle()); - //Allow oremap for non-published versions (not yet implemented) - addIfNotNull(aggBuilder, JsonLDTerm.schemaOrg("datePublished"), dataset.getPublicationDateFormattedYYYYMMDD()); - //Just for debugging - should always be set for real/persisted datasetversions - if(version.getLastUpdateTime() != null) { - aggBuilder.add(JsonLDTerm.schemaOrg("dateModified").getLabel(), version.getLastUpdateTime().toString()); - } + .add(JsonLDTerm.schemaOrg("datePublished").getLabel(), dataset.getPublicationDateFormattedYYYYMMDD()) + .add(JsonLDTerm.schemaOrg("name").getLabel(), version.getTitle()) + .add(JsonLDTerm.schemaOrg("dateModified").getLabel(), version.getLastUpdateTime().toString()); TermsOfUseAndAccess terms = version.getTermsOfUseAndAccess(); if (terms.getLicense() == TermsOfUseAndAccess.License.CC0) { @@ -213,45 +204,31 @@ public JsonObject getOREMap() throws Exception { if (df.getGlobalId().asString().length() != 0) { fileId = df.getGlobalId().asString(); fileSameAs = SystemConfig.getDataverseSiteUrlStatic() - + "/api/access/datafile/:persistentId?persistentId=" + fileId + (type.equals(TRANSFER) ? "&format=original" : ""); + + "/api/access/datafile/:persistentId?persistentId=" + fileId; } else { fileId = SystemConfig.getDataverseSiteUrlStatic() + "/file.xhtml?fileId=" + df.getId(); - fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId() + (type.equals(TRANSFER) ? "?format=original" : ""); + fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId(); } aggRes.add("@id", fileId); aggRes.add(JsonLDTerm.schemaOrg("sameAs").getLabel(), fileSameAs); fileArray.add(fileId); aggRes.add("@type", JsonLDTerm.ore("AggregatedResource").getLabel()); - switch (type) { - case TRANSFER: - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), df.getOriginalFileFormat()); - addIfNotNull(aggRes, JsonLDTerm.filesize, df.getOriginalFileSize()); - addIfNotNull(aggRes, JsonLDTerm.storageIdentifier, df.getStorageIdentifier()); - addIfNotNull(aggRes, JsonLDTerm.UNF, df.getUnf()); - addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId()); - addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId()); - //Checksum not available - will be generated in Bag - - break; - case ARCHIVE: - default: - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), df.getContentType()); - addIfNotNull(aggRes, JsonLDTerm.filesize, df.getFilesize()); - addIfNotNull(aggRes, JsonLDTerm.storageIdentifier, df.getStorageIdentifier()); - addIfNotNull(aggRes, JsonLDTerm.originalFileFormat, df.getOriginalFileFormat()); - addIfNotNull(aggRes, JsonLDTerm.originalFormatLabel, df.getOriginalFormatLabel()); - addIfNotNull(aggRes, JsonLDTerm.UNF, df.getUnf()); - addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId()); - addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId()); - JsonObject checksum = null; - // Add checksum. RDA recommends SHA-512 - if (df.getChecksumType() != null && df.getChecksumValue() != null) { - checksum = Json.createObjectBuilder().add("@type", df.getChecksumType().toString()) - .add("@value", df.getChecksumValue()).build(); - aggRes.add(JsonLDTerm.checksum.getLabel(), checksum); - } - } + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), df.getContentType()); + addIfNotNull(aggRes, JsonLDTerm.filesize, df.getFilesize()); + addIfNotNull(aggRes, JsonLDTerm.storageIdentifier, df.getStorageIdentifier()); + addIfNotNull(aggRes, JsonLDTerm.originalFileFormat, df.getOriginalFileFormat()); + addIfNotNull(aggRes, JsonLDTerm.originalFormatLabel, df.getOriginalFormatLabel()); + addIfNotNull(aggRes, JsonLDTerm.UNF, df.getUnf()); + addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId()); + addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId()); + JsonObject checksum = null; + // Add checksum. RDA recommends SHA-512 + if (df.getChecksumType() != null && df.getChecksumValue() != null) { + checksum = Json.createObjectBuilder().add("@type", df.getChecksumType().toString()) + .add("@value", df.getChecksumValue()).build(); + aggRes.add(JsonLDTerm.checksum.getLabel(), checksum); + } JsonArray tabTags = null; JsonArrayBuilder jab = JsonPrinter.getTabularFileTags(df); if (jab != null) { From 8abd55e9e6a0031e60cf701790aaced3e2a4cbd0 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 1 Dec 2020 12:43:56 -0500 Subject: [PATCH 048/354] remove OREMap parameter --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index f83826e7607..6f7b5ef8ccc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -98,7 +98,7 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, ds.setModificationTime(Timestamp.valueOf(dateTime)); } try { - logger.fine("Output dsv: " + new OREMap(dsv, false, OREMap.TRANSFER).getOREMap().toString()); + logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); From 1a35ed24a1c2721f66bec1c91ad17a38571abc43 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 1 Dec 2020 15:34:07 -0500 Subject: [PATCH 049/354] fix error handling FWIW: We have an error handler for the edu.harvard.iq.dataverse.util.json.JsonParseException class but not for javax.json.stream.JsonParsingException which was getting caught by the Throwable handler and returned as a 500 error with json message {} --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index b22d707c744..ccd49b78a83 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -77,6 +77,7 @@ import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.S3PackageImporter; +import edu.harvard.iq.dataverse.api.AbstractApiBean.WrappedResponse; import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; import edu.harvard.iq.dataverse.dataaccess.DataAccess; @@ -132,6 +133,7 @@ import javax.json.JsonObject; import javax.json.JsonObjectBuilder; import javax.json.JsonReader; +import javax.json.stream.JsonParsingException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; @@ -635,7 +637,9 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String } catch (WrappedResponse ex) { return ex.getResponse(); - + } catch (JsonParsingException jpe) { + logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", jsonLDBody); + return error(Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage()); } } From 7b1512e71ecb7c0482f41e62a5b5b7a3eb459259 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 1 Dec 2020 16:29:36 -0500 Subject: [PATCH 050/354] append to current terms --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 6f7b5ef8ccc..b09ad08f0b0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -139,7 +139,7 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, } fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); } - TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); + TermsOfUseAndAccess terms = dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess(); for (String key : jsonld.keySet()) { if (!key.equals("@context")) { From b45336c95ecbcee28b88a5b334c2180c86eb63fc Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 1 Dec 2020 16:58:49 -0500 Subject: [PATCH 051/354] add replace param --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 578c0447dee..20874f70653 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -615,7 +615,7 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, @Path("{id}/versions/{versionId}/metadata") @Consumes("application/json-ld") public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, - @PathParam("versionId") String versionId) { + @PathParam("versionId") String versionId, @QueryParam("replace") boolean replaceTerms) { if (!":draft".equals(versionId)) { return error(Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); @@ -625,7 +625,7 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String DataverseRequest req = createDataverseRequest(findUserOrDie()); DatasetVersion dsv = ds.getEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); - dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, true); + dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, !replaceTerms); DatasetVersion managedVersion; if (updateDraft) { From e5b54dfc6739c06bccb447103633d4dcd1f930f5 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 1 Dec 2020 16:58:49 -0500 Subject: [PATCH 052/354] add replace param --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index ccd49b78a83..a7f7979bbea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -613,7 +613,7 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, @Path("{id}/versions/{versionId}/metadata") @Consumes("application/json-ld") public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, - @PathParam("versionId") String versionId) { + @PathParam("versionId") String versionId, @QueryParam("replace") boolean replaceTerms) { if (!":draft".equals(versionId)) { return error(Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); @@ -623,7 +623,7 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String DataverseRequest req = createDataverseRequest(findUserOrDie()); DatasetVersion dsv = ds.getEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); - dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, true); + dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, !replaceTerms); DatasetVersion managedVersion; if (updateDraft) { From 578790f0b2558721f98defab8b384681246856f3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 2 Dec 2020 10:34:22 -0500 Subject: [PATCH 053/354] handle append on terms - fix cut/paste errors --- .../iq/dataverse/util/json/JSONLDUtil.java | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index b09ad08f0b0..a728263dea2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -28,6 +28,9 @@ import javax.json.JsonWriterFactory; import javax.json.stream.JsonGenerator; import javax.ws.rs.BadRequestException; + +import org.apache.commons.lang.StringUtils; + import com.apicatalog.jsonld.JsonLd; import com.apicatalog.jsonld.api.JsonLdError; import com.apicatalog.jsonld.document.JsonDocument; @@ -194,56 +197,56 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index + 1))); } - } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())) { + } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())&&(append || terms.getLicense().equals(TermsOfUseAndAccess.License.NONE))) { if (jsonld.getString(JsonLDTerm.schemaOrg("license").getUrl()) .equals("https://creativecommons.org/publicdomain/zero/1.0/")) { terms.setLicense(TermsOfUseAndAccess.defaultLicense); } else { terms.setLicense(TermsOfUseAndAccess.License.NONE); } - } else if (key.equals(JsonLDTerm.termsOfUse.getUrl())) { + } else if (key.equals(JsonLDTerm.termsOfUse.getUrl())&&(append || StringUtils.isBlank(terms.getTermsOfUse()))) { terms.setTermsOfUse(jsonld.getString(JsonLDTerm.termsOfUse.getUrl())); - } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getUrl())) { + } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getUrl())&&(append || StringUtils.isBlank(terms.getConfidentialityDeclaration()))) { terms.setConfidentialityDeclaration( jsonld.getString(JsonLDTerm.confidentialityDeclaration.getUrl())); - } else if (key.equals(JsonLDTerm.specialPermissions.getUrl())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.specialPermissions.getUrl())); - } else if (key.equals(JsonLDTerm.restrictions.getUrl())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.restrictions.getUrl())); - } else if (key.equals(JsonLDTerm.citationRequirements.getUrl())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.citationRequirements.getUrl())); - } else if (key.equals(JsonLDTerm.depositorRequirements.getUrl())) { - terms.setConfidentialityDeclaration( + } else if (key.equals(JsonLDTerm.specialPermissions.getUrl())&&(append || StringUtils.isBlank(terms.getSpecialPermissions()))) { + terms.setSpecialPermissions(jsonld.getString(JsonLDTerm.specialPermissions.getUrl())); + } else if (key.equals(JsonLDTerm.restrictions.getUrl())&&(append || StringUtils.isBlank(terms.getRestrictions()))) { + terms.setRestrictions(jsonld.getString(JsonLDTerm.restrictions.getUrl())); + } else if (key.equals(JsonLDTerm.citationRequirements.getUrl())&&(append || StringUtils.isBlank(terms.getCitationRequirements()))) { + terms.setCitationRequirements(jsonld.getString(JsonLDTerm.citationRequirements.getUrl())); + } else if (key.equals(JsonLDTerm.depositorRequirements.getUrl())&&(append || StringUtils.isBlank(terms.getDepositorRequirements()))) { + terms.setDepositorRequirements( jsonld.getString(JsonLDTerm.depositorRequirements.getUrl())); - } else if (key.equals(JsonLDTerm.conditions.getUrl())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.conditions.getUrl())); - } else if (key.equals(JsonLDTerm.disclaimer.getUrl())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.disclaimer.getUrl())); + } else if (key.equals(JsonLDTerm.conditions.getUrl())&&(append || StringUtils.isBlank(terms.getConditions()))) { + terms.setConditions(jsonld.getString(JsonLDTerm.conditions.getUrl())); + } else if (key.equals(JsonLDTerm.disclaimer.getUrl())&&(append || StringUtils.isBlank(terms.getDisclaimer()))) { + terms.setDisclaimer(jsonld.getString(JsonLDTerm.disclaimer.getUrl())); } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); - if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getUrl())&&(append || StringUtils.isBlank(terms.getTermsOfAccess()))) { terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getUrl())&&(append || !terms.isFileAccessRequest())) { terms.setFileAccessRequest(fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getUrl())&&(append || StringUtils.isBlank(terms.getDataAccessPlace()))) { terms.setDataAccessPlace(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getUrl())&&(append || StringUtils.isBlank(terms.getOriginalArchive()))) { terms.setOriginalArchive(fAccessObject.getString(JsonLDTerm.originalArchive.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getUrl())&&(append || StringUtils.isBlank(terms.getAvailabilityStatus()))) { terms.setAvailabilityStatus( fAccessObject.getString(JsonLDTerm.availabilityStatus.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getUrl())&&(append || StringUtils.isBlank(terms.getContactForAccess()))) { terms.setContactForAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getUrl())&&(append || StringUtils.isBlank(terms.getSizeOfCollection()))) { terms.setSizeOfCollection(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getUrl())&&(append || StringUtils.isBlank(terms.getStudyCompletion()))) { terms.setStudyCompletion(fAccessObject.getString(JsonLDTerm.studyCompletion.getUrl())); } } else { From acee4df8ce31d9d910766560caa459b2c15374a2 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 2 Dec 2020 11:58:50 -0500 Subject: [PATCH 054/354] fix logic --- .../iq/dataverse/util/json/JSONLDUtil.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index a728263dea2..75317f09806 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -197,56 +197,56 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index + 1))); } - } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())&&(append || terms.getLicense().equals(TermsOfUseAndAccess.License.NONE))) { + } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())&&(!append || terms.getLicense().equals(TermsOfUseAndAccess.License.NONE))) { if (jsonld.getString(JsonLDTerm.schemaOrg("license").getUrl()) .equals("https://creativecommons.org/publicdomain/zero/1.0/")) { terms.setLicense(TermsOfUseAndAccess.defaultLicense); } else { terms.setLicense(TermsOfUseAndAccess.License.NONE); } - } else if (key.equals(JsonLDTerm.termsOfUse.getUrl())&&(append || StringUtils.isBlank(terms.getTermsOfUse()))) { + } else if (key.equals(JsonLDTerm.termsOfUse.getUrl())&&(!append || StringUtils.isBlank(terms.getTermsOfUse()))) { terms.setTermsOfUse(jsonld.getString(JsonLDTerm.termsOfUse.getUrl())); - } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getUrl())&&(append || StringUtils.isBlank(terms.getConfidentialityDeclaration()))) { + } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getUrl())&&(!append || StringUtils.isBlank(terms.getConfidentialityDeclaration()))) { terms.setConfidentialityDeclaration( jsonld.getString(JsonLDTerm.confidentialityDeclaration.getUrl())); - } else if (key.equals(JsonLDTerm.specialPermissions.getUrl())&&(append || StringUtils.isBlank(terms.getSpecialPermissions()))) { + } else if (key.equals(JsonLDTerm.specialPermissions.getUrl())&&(!append || StringUtils.isBlank(terms.getSpecialPermissions()))) { terms.setSpecialPermissions(jsonld.getString(JsonLDTerm.specialPermissions.getUrl())); - } else if (key.equals(JsonLDTerm.restrictions.getUrl())&&(append || StringUtils.isBlank(terms.getRestrictions()))) { + } else if (key.equals(JsonLDTerm.restrictions.getUrl())&&(!append || StringUtils.isBlank(terms.getRestrictions()))) { terms.setRestrictions(jsonld.getString(JsonLDTerm.restrictions.getUrl())); - } else if (key.equals(JsonLDTerm.citationRequirements.getUrl())&&(append || StringUtils.isBlank(terms.getCitationRequirements()))) { + } else if (key.equals(JsonLDTerm.citationRequirements.getUrl())&&(!append || StringUtils.isBlank(terms.getCitationRequirements()))) { terms.setCitationRequirements(jsonld.getString(JsonLDTerm.citationRequirements.getUrl())); - } else if (key.equals(JsonLDTerm.depositorRequirements.getUrl())&&(append || StringUtils.isBlank(terms.getDepositorRequirements()))) { + } else if (key.equals(JsonLDTerm.depositorRequirements.getUrl())&&(!append || StringUtils.isBlank(terms.getDepositorRequirements()))) { terms.setDepositorRequirements( jsonld.getString(JsonLDTerm.depositorRequirements.getUrl())); - } else if (key.equals(JsonLDTerm.conditions.getUrl())&&(append || StringUtils.isBlank(terms.getConditions()))) { + } else if (key.equals(JsonLDTerm.conditions.getUrl())&&(!append || StringUtils.isBlank(terms.getConditions()))) { terms.setConditions(jsonld.getString(JsonLDTerm.conditions.getUrl())); - } else if (key.equals(JsonLDTerm.disclaimer.getUrl())&&(append || StringUtils.isBlank(terms.getDisclaimer()))) { + } else if (key.equals(JsonLDTerm.disclaimer.getUrl())&&(!append || StringUtils.isBlank(terms.getDisclaimer()))) { terms.setDisclaimer(jsonld.getString(JsonLDTerm.disclaimer.getUrl())); } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); - if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getUrl())&&(append || StringUtils.isBlank(terms.getTermsOfAccess()))) { + if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getUrl())&&(!append || StringUtils.isBlank(terms.getTermsOfAccess()))) { terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getUrl())&&(append || !terms.isFileAccessRequest())) { + if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getUrl())&&(!append || !terms.isFileAccessRequest())) { terms.setFileAccessRequest(fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getUrl())&&(append || StringUtils.isBlank(terms.getDataAccessPlace()))) { + if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getUrl())&&(!append || StringUtils.isBlank(terms.getDataAccessPlace()))) { terms.setDataAccessPlace(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getUrl())&&(append || StringUtils.isBlank(terms.getOriginalArchive()))) { + if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getUrl())&&(!append || StringUtils.isBlank(terms.getOriginalArchive()))) { terms.setOriginalArchive(fAccessObject.getString(JsonLDTerm.originalArchive.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getUrl())&&(append || StringUtils.isBlank(terms.getAvailabilityStatus()))) { + if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getUrl())&&(!append || StringUtils.isBlank(terms.getAvailabilityStatus()))) { terms.setAvailabilityStatus( fAccessObject.getString(JsonLDTerm.availabilityStatus.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getUrl())&&(append || StringUtils.isBlank(terms.getContactForAccess()))) { + if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getUrl())&&(!append || StringUtils.isBlank(terms.getContactForAccess()))) { terms.setContactForAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getUrl())&&(append || StringUtils.isBlank(terms.getSizeOfCollection()))) { + if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getUrl())&&(!append || StringUtils.isBlank(terms.getSizeOfCollection()))) { terms.setSizeOfCollection(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getUrl())&&(append || StringUtils.isBlank(terms.getStudyCompletion()))) { + if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getUrl())&&(!append || StringUtils.isBlank(terms.getStudyCompletion()))) { terms.setStudyCompletion(fAccessObject.getString(JsonLDTerm.studyCompletion.getUrl())); } } else { From 9185126fb88970b1b3e3a448070745f232261a11 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 2 Dec 2020 11:59:09 -0500 Subject: [PATCH 055/354] specify default --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index a7f7979bbea..bce104f81fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -138,6 +138,7 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; @@ -613,7 +614,7 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, @Path("{id}/versions/{versionId}/metadata") @Consumes("application/json-ld") public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, - @PathParam("versionId") String versionId, @QueryParam("replace") boolean replaceTerms) { + @PathParam("versionId") String versionId, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { if (!":draft".equals(versionId)) { return error(Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); From e8698dc41095d03442ebb156d5b4de03c3cec1ca Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 2 Dec 2020 12:21:56 -0500 Subject: [PATCH 056/354] make replace still append for multiple val fields --- .../edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 75317f09806..ec86abd07ac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -152,8 +152,8 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, DatasetField dsf = null; if (fieldByTypeMap.containsKey(dsft)) { dsf = fieldByTypeMap.get(dsft); - // If there's an existing field, we use it with append and remove it for !append - if (!append) { + // If there's an existing field, we use it with append and remove it for !append unless it's multiple + if (!append && !dsft.isAllowMultiples()) { dsfl.remove(dsf); } } @@ -256,8 +256,8 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, DatasetField dsf = null; if (fieldByTypeMap.containsKey(dsft)) { dsf = fieldByTypeMap.get(dsft); - // If there's an existing field, we use it with append and remove it for !append - if (!append) { + // If there's an existing field, we use it with append and remove it for !append (except if multiple, which is not the default) + if (!append && !dsft.isAllowMultiples()) { dsfl.remove(dsf); } } From 901efe863fb13d31dfdd8751373657f469090e2b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 2 Dec 2020 14:38:17 -0500 Subject: [PATCH 057/354] add migrating switch --- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../iq/dataverse/util/json/JSONLDUtil.java | 1237 +++++++++-------- 2 files changed, 677 insertions(+), 562 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index bce104f81fa..8ce029c1a35 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -624,7 +624,7 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String DataverseRequest req = createDataverseRequest(findUserOrDie()); DatasetVersion dsv = ds.getEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); - dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, !replaceTerms); + dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, !replaceTerms, false); DatasetVersion managedVersion; if (updateDraft) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index ec86abd07ac..414b3ec8a01 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -8,6 +8,7 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.LinkedList; @@ -47,6 +48,7 @@ import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess.License; import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.util.bagit.OREMap; @@ -54,568 +56,681 @@ public class JSONLDUtil { - private static final Logger logger = Logger.getLogger(JSONLDUtil.class.getCanonicalName()); - - /* - * private static Map populateContext(JsonValue json) { - * Map context = new TreeMap(); if (json - * instanceof JsonArray) { logger.warning("Array @context not yet supported"); } - * else { for (String key : ((JsonObject) json).keySet()) { - * context.putIfAbsent(key, ((JsonObject) json).getString(key)); } } return - * context; } - */ - - public static JsonObject getContext(Map contextMap) { - JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); - for (Entry e : contextMap.entrySet()) { - contextBuilder.add(e.getKey(), e.getValue()); - } - return contextBuilder.build(); - } - - public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, - MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append) { - - DatasetVersion dsv = new DatasetVersion(); - - JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); - Optional maybePid = GlobalId.parse(jsonld.getString("@id")); - if (maybePid.isPresent()) { - ds.setGlobalId(maybePid.get()); - } else { - // unparsable PID passed. Terminate. - throw new BadRequestException("Cannot parse the @id '" + jsonld.getString("@id") - + "'. Make sure it is in valid form - see Dataverse Native API documentation."); - } - - dsv = updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append); - dsv.setDataset(ds); - - List versions = new ArrayList<>(1); - versions.add(dsv); - - ds.setVersions(versions); - if (jsonld.containsKey(JsonLDTerm.schemaOrg("dateModified").getUrl())) { - String dateString = jsonld.getString(JsonLDTerm.schemaOrg("dateModified").getUrl()); - LocalDateTime dateTime = getDateTimeFrom(dateString); - ds.setModificationTime(Timestamp.valueOf(dateTime)); - } - try { - logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return ds; - } - - public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, String jsonLDBody, - MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append) { - JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); - return updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append); - } - - /** - * - * @param dsv - * @param jsonld - * @param metadataBlockSvc - * @param datasetFieldSvc - * @param append - if append, will add new top level field values for - * multi-valued fields, if true and field type isn't - * multiple, will fail. if false will replace all - * value(s) for fields found in the json-ld. - * @return - */ - public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, JsonObject jsonld, - MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append) { - - populateFieldTypeMap(metadataBlockSvc); - - // get existing ones? - List dsfl = dsv.getDatasetFields(); - Map fieldByTypeMap = new HashMap(); - for (DatasetField dsf : dsfl) { - if (fieldByTypeMap.containsKey(dsf.getDatasetFieldType())) { - // May have multiple values per field, but not multiple fields of one type? - logger.warning("Multiple fields of type " + dsf.getDatasetFieldType().getName()); - } - fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); - } - TermsOfUseAndAccess terms = dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess(); - - for (String key : jsonld.keySet()) { - if (!key.equals("@context")) { - if (dsftMap.containsKey(key)) { - - DatasetFieldType dsft = dsftMap.get(key); - DatasetField dsf = null; - if (fieldByTypeMap.containsKey(dsft)) { - dsf = fieldByTypeMap.get(dsft); - // If there's an existing field, we use it with append and remove it for !append unless it's multiple - if (!append && !dsft.isAllowMultiples()) { - dsfl.remove(dsf); - } - } - if (dsf == null) { - dsf = new DatasetField(); - dsfl.add(dsf); - dsf.setDatasetFieldType(dsft); - } - - // Todo - normalize object vs. array - JsonArray valArray = getValues(jsonld.get(key), dsft.isAllowMultiples(), dsft.getName()); - - addField(dsf, valArray, dsft, datasetFieldSvc, append); - - // assemble new terms, add to existing - // multivalue? - // compound? - // merge with existing dv metadata - // dsfl.add(dsf); - } else { - // Internal/non-metadatablock terms - // Add metadata related to the Dataset/DatasetVersion - - // ("@id", id) - check is equal to existing globalID? - // Add to 'md on original' ? - // (JsonLDTerm.schemaOrg("version").getLabel(), - // version.getFriendlyVersionNumber()) - // Citation metadata? - // (JsonLDTerm.schemaOrg("datePublished").getLabel(), - // dataset.getPublicationDateFormattedYYYYMMDD()) - // (JsonLDTerm.schemaOrg("name").getLabel()) - // (JsonLDTerm.schemaOrg("dateModified").getLabel()) - - // Todo - handle non-CC0 licenses, without terms as an alternate field. - if (key.equals(JsonLDTerm.schemaOrg("datePublished").getUrl())) { - dsv.setVersionState(VersionState.RELEASED); - } else if (key.equals(JsonLDTerm.schemaOrg("version").getUrl())) { - String friendlyVersion = jsonld.getString(JsonLDTerm.schemaOrg("version").getUrl()); - int index = friendlyVersion.indexOf("."); - if (index > 0) { - dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); - dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index + 1))); - } - } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())&&(!append || terms.getLicense().equals(TermsOfUseAndAccess.License.NONE))) { - if (jsonld.getString(JsonLDTerm.schemaOrg("license").getUrl()) - .equals("https://creativecommons.org/publicdomain/zero/1.0/")) { - terms.setLicense(TermsOfUseAndAccess.defaultLicense); - } else { - terms.setLicense(TermsOfUseAndAccess.License.NONE); - } - } else if (key.equals(JsonLDTerm.termsOfUse.getUrl())&&(!append || StringUtils.isBlank(terms.getTermsOfUse()))) { - terms.setTermsOfUse(jsonld.getString(JsonLDTerm.termsOfUse.getUrl())); - } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getUrl())&&(!append || StringUtils.isBlank(terms.getConfidentialityDeclaration()))) { - terms.setConfidentialityDeclaration( - jsonld.getString(JsonLDTerm.confidentialityDeclaration.getUrl())); - } else if (key.equals(JsonLDTerm.specialPermissions.getUrl())&&(!append || StringUtils.isBlank(terms.getSpecialPermissions()))) { - terms.setSpecialPermissions(jsonld.getString(JsonLDTerm.specialPermissions.getUrl())); - } else if (key.equals(JsonLDTerm.restrictions.getUrl())&&(!append || StringUtils.isBlank(terms.getRestrictions()))) { - terms.setRestrictions(jsonld.getString(JsonLDTerm.restrictions.getUrl())); - } else if (key.equals(JsonLDTerm.citationRequirements.getUrl())&&(!append || StringUtils.isBlank(terms.getCitationRequirements()))) { - terms.setCitationRequirements(jsonld.getString(JsonLDTerm.citationRequirements.getUrl())); - } else if (key.equals(JsonLDTerm.depositorRequirements.getUrl())&&(!append || StringUtils.isBlank(terms.getDepositorRequirements()))) { - terms.setDepositorRequirements( - jsonld.getString(JsonLDTerm.depositorRequirements.getUrl())); - } else if (key.equals(JsonLDTerm.conditions.getUrl())&&(!append || StringUtils.isBlank(terms.getConditions()))) { - terms.setConditions(jsonld.getString(JsonLDTerm.conditions.getUrl())); - } else if (key.equals(JsonLDTerm.disclaimer.getUrl())&&(!append || StringUtils.isBlank(terms.getDisclaimer()))) { - terms.setDisclaimer(jsonld.getString(JsonLDTerm.disclaimer.getUrl())); - } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { - JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); - if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getUrl())&&(!append || StringUtils.isBlank(terms.getTermsOfAccess()))) { - terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getUrl())&&(!append || !terms.isFileAccessRequest())) { - terms.setFileAccessRequest(fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getUrl())&&(!append || StringUtils.isBlank(terms.getDataAccessPlace()))) { - terms.setDataAccessPlace(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getUrl())&&(!append || StringUtils.isBlank(terms.getOriginalArchive()))) { - terms.setOriginalArchive(fAccessObject.getString(JsonLDTerm.originalArchive.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getUrl())&&(!append || StringUtils.isBlank(terms.getAvailabilityStatus()))) { - terms.setAvailabilityStatus( - fAccessObject.getString(JsonLDTerm.availabilityStatus.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getUrl())&&(!append || StringUtils.isBlank(terms.getContactForAccess()))) { - terms.setContactForAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getUrl())&&(!append || StringUtils.isBlank(terms.getSizeOfCollection()))) { - terms.setSizeOfCollection(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getUrl())&&(!append || StringUtils.isBlank(terms.getStudyCompletion()))) { - terms.setStudyCompletion(fAccessObject.getString(JsonLDTerm.studyCompletion.getUrl())); - } - } else { - if (dsftMap.containsKey(JsonLDTerm.metadataOnOrig.getUrl())) { - DatasetFieldType dsft = dsftMap.get(JsonLDTerm.metadataOnOrig.getUrl()); - - DatasetField dsf = null; - if (fieldByTypeMap.containsKey(dsft)) { - dsf = fieldByTypeMap.get(dsft); - // If there's an existing field, we use it with append and remove it for !append (except if multiple, which is not the default) - if (!append && !dsft.isAllowMultiples()) { - dsfl.remove(dsf); - } - } - if (dsf == null) { - dsf = new DatasetField(); - dsfl.add(dsf); - dsf.setDatasetFieldType(dsft); - } - - List vals = dsf.getDatasetFieldValues(); - - JsonObject currentValue = null; - DatasetFieldValue datasetFieldValue = null; - if (vals.isEmpty()) { - datasetFieldValue = new DatasetFieldValue(); - vals.add(datasetFieldValue); - datasetFieldValue.setDatasetField(dsf); - dsf.setDatasetFieldValues(vals); - - currentValue = Json.createObjectBuilder().build(); - } else { - datasetFieldValue = vals.get(0); - JsonObject currentVal = decontextualizeJsonLD(datasetFieldValue.getValueForEdit()); - - } - currentValue.put(key, jsonld.get(key)); - JsonObject newValue = recontextualizeJsonLD(currentValue, metadataBlockSvc); - datasetFieldValue.setValue(prettyPrint(newValue)); - } - } - dsv.setTermsOfUseAndAccess(terms); - // move to new dataverse? - // aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), - // dataset.getDataverseContext().getDisplayName()); - - } - - } - } - - dsv.setDatasetFields(dsfl); - - return dsv; - } - - private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldType dsft, - DatasetFieldServiceBean datasetFieldSvc, boolean append) { - - if (append && !dsft.isAllowMultiples()) { - if ((dsft.isCompound() && !dsf.getDatasetFieldCompoundValues().isEmpty()) - || (dsft.isAllowControlledVocabulary() && !dsf.getControlledVocabularyValues().isEmpty()) - || !dsf.getDatasetFieldValues().isEmpty()) { - throw new BadRequestException( - "Can't append to a single-value field that already has a value: " + dsft.getName()); - } - } - logger.fine("Name: " + dsft.getName()); - logger.fine("val: " + valArray.toString()); - logger.fine("Compound: " + dsft.isCompound()); - logger.fine("CV: " + dsft.isAllowControlledVocabulary()); - - if (dsft.isCompound()) { - /* - * List vals = parseCompoundValue(type, - * jsonld.get(key),testType); for (DatasetFieldCompoundValue dsfcv : vals) { - * dsfcv.setParentDatasetField(ret); } dsf.setDatasetFieldCompoundValues(vals); - */ - List cvList = dsf.getDatasetFieldCompoundValues(); - if (!cvList.isEmpty()) { - if (!append) { - cvList.clear(); - } else if (!dsft.isAllowMultiples() && cvList.size() == 1) { - // Trying to append but only a single value is allowed (and there already is - // one) - // (and we don't currently support appending new fields within a compound value) - throw new BadRequestException( - "Append with compound field with single value not yet supported: " + dsft.getDisplayName()); - } - } - - List vals = new LinkedList<>(); - for (JsonValue val : valArray) { - if (!(val instanceof JsonObject)) { - throw new BadRequestException( - "Compound field values must be JSON objects, field: " + dsft.getName()); - } - DatasetFieldCompoundValue cv = null; - - cv = new DatasetFieldCompoundValue(); - cv.setDisplayOrder(cvList.size()); - cvList.add(cv); - cv.setParentDatasetField(dsf); - - JsonObject obj = (JsonObject) val; - for (String childKey : obj.keySet()) { - if (dsftMap.containsKey(childKey)) { - DatasetFieldType childft = dsftMap.get(childKey); - if (!dsft.getChildDatasetFieldTypes().contains(childft)) { - throw new BadRequestException( - "Compound field " + dsft.getName() + "can't include term " + childKey); - } - DatasetField childDsf = new DatasetField(); - cv.getChildDatasetFields().add(childDsf); - childDsf.setDatasetFieldType(childft); - childDsf.setParentDatasetFieldCompoundValue(cv); - - JsonArray childValArray = getValues(obj.get(childKey), childft.isAllowMultiples(), - childft.getName()); - addField(childDsf, childValArray, childft, datasetFieldSvc, append); - } - } - } - - } else if (dsft.isControlledVocabulary()) { - - List vals = dsf.getControlledVocabularyValues(); - for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { - String strValue = strVal.getString(); - ControlledVocabularyValue cvv = datasetFieldSvc - .findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, strValue, true); - if (cvv == null) { - throw new BadRequestException( - "Unknown value for Controlled Vocab Field: " + dsft.getName() + " : " + strValue); - } - // Only add value to the list if it is not a duplicate - if (strValue.equals("Other")) { - System.out.println("vals = " + vals + ", contains: " + vals.contains(cvv)); - } - if (!vals.contains(cvv)) { - if (vals.size() > 0) { - cvv.setDisplayOrder(vals.size()); - } - vals.add(cvv); - cvv.setDatasetFieldType(dsft); - } - } - dsf.setControlledVocabularyValues(vals); - - } else { - List vals = dsf.getDatasetFieldValues(); - - for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { - String strValue = strVal.getString(); - DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); - - datasetFieldValue.setDisplayOrder(vals.size()); - datasetFieldValue.setValue(strValue.trim()); - vals.add(datasetFieldValue); - datasetFieldValue.setDatasetField(dsf); - - } - dsf.setDatasetFieldValues(vals); - } - } - - private static JsonArray getValues(JsonValue val, boolean allowMultiples, String name) { - JsonArray valArray = null; - if (val instanceof JsonArray) { - if ((((JsonArray) val).size() > 1) && !allowMultiples) { - throw new BadRequestException("Array for single value notsupported: " + name); - } else { - valArray = (JsonArray) val; - } - } else { - valArray = Json.createArrayBuilder().add(val).build(); - } - return valArray; - } - - static Map localContext = new TreeMap(); - static Map dsftMap = new TreeMap(); - - private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockSvc) { - if (dsftMap.isEmpty()) { - - List mdbList = metadataBlockSvc.listMetadataBlocks(); - - for (MetadataBlock mdb : mdbList) { - boolean blockHasUri = mdb.getNamespaceUri() != null; - for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { - if (dsft.getUri() != null) { - dsftMap.put(dsft.getUri(), dsft); - } - if (blockHasUri) { - if (dsft.getParentDatasetFieldType() != null) { - // ToDo - why not getName for child type? Would have to fix in ORE generation - // code and handle legacy bags - dsftMap.put(mdb.getNamespaceUri() + dsft.getParentDatasetFieldType().getName() + "#" - + dsft.getTitle(), dsft); - } else { - dsftMap.put(mdb.getNamespaceUri() + dsft.getTitle(), dsft); - } - } - } - } - logger.fine("DSFT Map: " + String.join(", ", dsftMap.keySet())); - } - } - - private static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { - if (localContext.isEmpty()) { - - // Add namespaces corresponding to core terms - localContext.put(JsonLDNamespace.dcterms.getPrefix(), JsonLDNamespace.dcterms.getUrl()); - localContext.put(JsonLDNamespace.dvcore.getPrefix(), JsonLDNamespace.dvcore.getUrl()); - localContext.put(JsonLDNamespace.schema.getPrefix(), JsonLDNamespace.schema.getUrl()); - - List mdbList = metadataBlockSvc.listMetadataBlocks(); - - for (MetadataBlock mdb : mdbList) { - boolean blockHasUri = mdb.getNamespaceUri() != null; - if (blockHasUri) { - localContext.putIfAbsent(mdb.getName(), mdb.getNamespaceUri()); - - } - for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { - if (dsft.getUri() != null) { - localContext.putIfAbsent(dsft.getName(), dsft.getUri()); - } - } - } - logger.fine("LocalContext keys: " + String.join(", ", localContext.keySet())); - } - } - - public static JsonObject decontextualizeJsonLD(String jsonLDString) { - logger.fine(jsonLDString); - try (StringReader rdr = new StringReader(jsonLDString)) { - - // Use JsonLd to expand/compact to localContext - JsonObject jsonld = Json.createReader(rdr).readObject(); - JsonDocument doc = JsonDocument.of(jsonld); - JsonArray array = null; - try { - array = JsonLd.expand(doc).get(); - jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(Json.createObjectBuilder().build())) - .get(); - // jsonld = array.getJsonObject(0); - logger.fine("Decontextualized object: " + jsonld); - return jsonld; - } catch (JsonLdError e) { - System.out.println(e.getMessage()); - return null; - } - } - } - - private static JsonObject recontextualizeJsonLD(JsonObject jsonldObj, MetadataBlockServiceBean metadataBlockSvc) { - - populateContext(metadataBlockSvc); - - // Use JsonLd to expand/compact to localContext - JsonDocument doc = JsonDocument.of(jsonldObj); - JsonArray array = null; - try { - array = JsonLd.expand(doc).get(); - - jsonldObj = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(JSONLDUtil.getContext(localContext))) - .get(); - logger.fine("Compacted: " + jsonldObj.toString()); - return jsonldObj; - } catch (JsonLdError e) { - System.out.println(e.getMessage()); - return null; - } - } - - public static String prettyPrint(JsonValue val) { - StringWriter sw = new StringWriter(); - Map properties = new HashMap<>(1); - properties.put(JsonGenerator.PRETTY_PRINTING, true); - JsonWriterFactory writerFactory = Json.createWriterFactory(properties); - JsonWriter jsonWriter = writerFactory.createWriter(sw); - jsonWriter.write(val); - jsonWriter.close(); - return sw.toString(); - } + private static final Logger logger = Logger.getLogger(JSONLDUtil.class.getCanonicalName()); + + /* + * private static Map populateContext(JsonValue json) { + * Map context = new TreeMap(); if (json + * instanceof JsonArray) { logger.warning("Array @context not yet supported"); } + * else { for (String key : ((JsonObject) json).keySet()) { + * context.putIfAbsent(key, ((JsonObject) json).getString(key)); } } return + * context; } + */ + + public static JsonObject getContext(Map contextMap) { + JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); + for (Entry e : contextMap.entrySet()) { + contextBuilder.add(e.getKey(), e.getValue()); + } + return contextBuilder.build(); + } + + public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { + + DatasetVersion dsv = new DatasetVersion(); + + JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); + Optional maybePid = GlobalId.parse(jsonld.getString("@id")); + if (maybePid.isPresent()) { + ds.setGlobalId(maybePid.get()); + } else { + // unparsable PID passed. Terminate. + throw new BadRequestException("Cannot parse the @id '" + jsonld.getString("@id") + + "'. Make sure it is in valid form - see Dataverse Native API documentation."); + } + + dsv = updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); + dsv.setDataset(ds); + + List versions = new ArrayList<>(1); + versions.add(dsv); + + ds.setVersions(versions); + if (jsonld.containsKey(JsonLDTerm.schemaOrg("dateModified").getUrl())) { + String dateString = jsonld.getString(JsonLDTerm.schemaOrg("dateModified").getUrl()); + LocalDateTime dateTime = getDateTimeFrom(dateString); + ds.setModificationTime(Timestamp.valueOf(dateTime)); + } + try { + logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return ds; + } + + public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, String jsonLDBody, + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { + JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); + return updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); + } + + /** + * + * @param dsv + * @param jsonld + * @param metadataBlockSvc + * @param datasetFieldSvc + * @param append - if append, will add new top level field values for + * multi-valued fields, if true and field type isn't + * multiple, will fail. if false will replace all + * value(s) for fields found in the json-ld. + * @return + */ + public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, JsonObject jsonld, + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { + + populateFieldTypeMap(metadataBlockSvc); + + // get existing ones? + List dsfl = dsv.getDatasetFields(); + Map fieldByTypeMap = new HashMap(); + for (DatasetField dsf : dsfl) { + if (fieldByTypeMap.containsKey(dsf.getDatasetFieldType())) { + // May have multiple values per field, but not multiple fields of one type? + logger.warning("Multiple fields of type " + dsf.getDatasetFieldType().getName()); + } + fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); + } + TermsOfUseAndAccess terms = dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess(); + + for (String key : jsonld.keySet()) { + if (!key.equals("@context")) { + if (dsftMap.containsKey(key)) { + + DatasetFieldType dsft = dsftMap.get(key); + DatasetField dsf = null; + if (fieldByTypeMap.containsKey(dsft)) { + dsf = fieldByTypeMap.get(dsft); + // If there's an existing field, we use it with append and remove it for !append + // unless it's multiple + if (!append && !dsft.isAllowMultiples()) { + dsfl.remove(dsf); + } + } + if (dsf == null) { + dsf = new DatasetField(); + dsfl.add(dsf); + dsf.setDatasetFieldType(dsft); + } + + // Todo - normalize object vs. array + JsonArray valArray = getValues(jsonld.get(key), dsft.isAllowMultiples(), dsft.getName()); + + addField(dsf, valArray, dsft, datasetFieldSvc, append); + + // assemble new terms, add to existing + // multivalue? + // compound? + // merge with existing dv metadata + // dsfl.add(dsf); + } else { + // Internal/non-metadatablock terms + // Add metadata related to the Dataset/DatasetVersion + + // ("@id", id) - check is equal to existing globalID? + // Add to 'md on original' ? + // (JsonLDTerm.schemaOrg("version").getLabel(), + // version.getFriendlyVersionNumber()) + // Citation metadata? + // (JsonLDTerm.schemaOrg("datePublished").getLabel(), + // dataset.getPublicationDateFormattedYYYYMMDD()) + // (JsonLDTerm.schemaOrg("name").getLabel()) + // (JsonLDTerm.schemaOrg("dateModified").getLabel()) + + if (key.equals(JsonLDTerm.schemaOrg("datePublished").getUrl())&& migrating && !append) { + dsv.setVersionState(VersionState.RELEASED); + } else if (key.equals(JsonLDTerm.schemaOrg("version").getUrl())&& migrating && !append) { + String friendlyVersion = jsonld.getString(JsonLDTerm.schemaOrg("version").getUrl()); + int index = friendlyVersion.indexOf("."); + if (index > 0) { + dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); + dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index + 1))); + } + } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())) { + if (!append || !isSet(terms, key)) { + // Mirror rules from SwordServiceBean + if (jsonld.containsKey(JsonLDTerm.termsOfUse.getUrl())) { + throw new BadRequestException( + "Cannot specify " + JsonLDTerm.schemaOrg("license").getUrl() + " and " + + JsonLDTerm.termsOfUse.getUrl()); + } + setSemTerm(terms, key, TermsOfUseAndAccess.defaultLicense); + } else { + throw new BadRequestException( + "Can't append to a single-value field that already has a value: " + + JsonLDTerm.schemaOrg("license").getUrl()); + } + + } else if (datasetTerms.contains(key)) { + if (!append || !isSet(terms, key)) { + // Other Dataset-level TermsOfUseAndAccess + setSemTerm(terms, key, jsonld.getString(JsonLDTerm.termsOfUse.getUrl())); + } else { + throw new BadRequestException( + "Can't append to a single-value field that already has a value: " + key); + } + } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { + JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); + for (String fileKey : fAccessObject.keySet()) { + if (datafileTerms.contains(fileKey)) { + if (!append || !isSet(terms, fileKey)) { + // Other DataFile-level TermsOfUseAndAccess + if (fileKey.equals(JsonLDTerm.fileRequestAccess.getUrl())) { + setSemTerm(terms, fileKey, fAccessObject.getBoolean(fileKey)); + } else { + setSemTerm(terms, fileKey, fAccessObject.getString(fileKey)); + } + } else { + throw new BadRequestException( + "Can't append to a single-value field that already has a value: " + + fileKey); + } + } + } + } else { + if (dsftMap.containsKey(JsonLDTerm.metadataOnOrig.getUrl())) { + DatasetFieldType dsft = dsftMap.get(JsonLDTerm.metadataOnOrig.getUrl()); + + DatasetField dsf = null; + if (fieldByTypeMap.containsKey(dsft)) { + dsf = fieldByTypeMap.get(dsft); + // If there's an existing field, we use it with append and remove it for !append + // (except if multiple, which is not the default) + if (!append && !dsft.isAllowMultiples()) { + dsfl.remove(dsf); + } + } + if (dsf == null) { + dsf = new DatasetField(); + dsfl.add(dsf); + dsf.setDatasetFieldType(dsft); + } + + List vals = dsf.getDatasetFieldValues(); + + JsonObject currentValue = null; + DatasetFieldValue datasetFieldValue = null; + if (vals.isEmpty()) { + datasetFieldValue = new DatasetFieldValue(); + vals.add(datasetFieldValue); + datasetFieldValue.setDatasetField(dsf); + dsf.setDatasetFieldValues(vals); + + currentValue = Json.createObjectBuilder().build(); + } else { + datasetFieldValue = vals.get(0); + JsonObject currentVal = decontextualizeJsonLD(datasetFieldValue.getValueForEdit()); + + } + currentValue.put(key, jsonld.get(key)); + JsonObject newValue = recontextualizeJsonLD(currentValue, metadataBlockSvc); + datasetFieldValue.setValue(prettyPrint(newValue)); + } + } + dsv.setTermsOfUseAndAccess(terms); + // move to new dataverse? + // aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), + // dataset.getDataverseContext().getDisplayName()); + + } + + } + } + + dsv.setDatasetFields(dsfl); + + return dsv; + } + + private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldType dsft, + DatasetFieldServiceBean datasetFieldSvc, boolean append) { + + if (append && !dsft.isAllowMultiples()) { + if ((dsft.isCompound() && !dsf.getDatasetFieldCompoundValues().isEmpty()) + || (dsft.isAllowControlledVocabulary() && !dsf.getControlledVocabularyValues().isEmpty()) + || !dsf.getDatasetFieldValues().isEmpty()) { + throw new BadRequestException( + "Can't append to a single-value field that already has a value: " + dsft.getName()); + } + } + logger.fine("Name: " + dsft.getName()); + logger.fine("val: " + valArray.toString()); + logger.fine("Compound: " + dsft.isCompound()); + logger.fine("CV: " + dsft.isAllowControlledVocabulary()); + + if (dsft.isCompound()) { + /* + * List vals = parseCompoundValue(type, + * jsonld.get(key),testType); for (DatasetFieldCompoundValue dsfcv : vals) { + * dsfcv.setParentDatasetField(ret); } dsf.setDatasetFieldCompoundValues(vals); + */ + List cvList = dsf.getDatasetFieldCompoundValues(); + if (!cvList.isEmpty()) { + if (!append) { + cvList.clear(); + } else if (!dsft.isAllowMultiples() && cvList.size() == 1) { + // Trying to append but only a single value is allowed (and there already is + // one) + // (and we don't currently support appending new fields within a compound value) + throw new BadRequestException( + "Append with compound field with single value not yet supported: " + dsft.getDisplayName()); + } + } + + List vals = new LinkedList<>(); + for (JsonValue val : valArray) { + if (!(val instanceof JsonObject)) { + throw new BadRequestException( + "Compound field values must be JSON objects, field: " + dsft.getName()); + } + DatasetFieldCompoundValue cv = null; + + cv = new DatasetFieldCompoundValue(); + cv.setDisplayOrder(cvList.size()); + cvList.add(cv); + cv.setParentDatasetField(dsf); + + JsonObject obj = (JsonObject) val; + for (String childKey : obj.keySet()) { + if (dsftMap.containsKey(childKey)) { + DatasetFieldType childft = dsftMap.get(childKey); + if (!dsft.getChildDatasetFieldTypes().contains(childft)) { + throw new BadRequestException( + "Compound field " + dsft.getName() + "can't include term " + childKey); + } + DatasetField childDsf = new DatasetField(); + cv.getChildDatasetFields().add(childDsf); + childDsf.setDatasetFieldType(childft); + childDsf.setParentDatasetFieldCompoundValue(cv); + + JsonArray childValArray = getValues(obj.get(childKey), childft.isAllowMultiples(), + childft.getName()); + addField(childDsf, childValArray, childft, datasetFieldSvc, append); + } + } + } + + } else if (dsft.isControlledVocabulary()) { + + List vals = dsf.getControlledVocabularyValues(); + for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + ControlledVocabularyValue cvv = datasetFieldSvc + .findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, strValue, true); + if (cvv == null) { + throw new BadRequestException( + "Unknown value for Controlled Vocab Field: " + dsft.getName() + " : " + strValue); + } + // Only add value to the list if it is not a duplicate + if (strValue.equals("Other")) { + System.out.println("vals = " + vals + ", contains: " + vals.contains(cvv)); + } + if (!vals.contains(cvv)) { + if (vals.size() > 0) { + cvv.setDisplayOrder(vals.size()); + } + vals.add(cvv); + cvv.setDatasetFieldType(dsft); + } + } + dsf.setControlledVocabularyValues(vals); + + } else { + List vals = dsf.getDatasetFieldValues(); + + for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); + + datasetFieldValue.setDisplayOrder(vals.size()); + datasetFieldValue.setValue(strValue.trim()); + vals.add(datasetFieldValue); + datasetFieldValue.setDatasetField(dsf); + + } + dsf.setDatasetFieldValues(vals); + } + } + + private static JsonArray getValues(JsonValue val, boolean allowMultiples, String name) { + JsonArray valArray = null; + if (val instanceof JsonArray) { + if ((((JsonArray) val).size() > 1) && !allowMultiples) { + throw new BadRequestException("Array for single value notsupported: " + name); + } else { + valArray = (JsonArray) val; + } + } else { + valArray = Json.createArrayBuilder().add(val).build(); + } + return valArray; + } + + static Map localContext = new TreeMap(); + static Map dsftMap = new TreeMap(); + + private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockSvc) { + if (dsftMap.isEmpty()) { + + List mdbList = metadataBlockSvc.listMetadataBlocks(); + + for (MetadataBlock mdb : mdbList) { + boolean blockHasUri = mdb.getNamespaceUri() != null; + for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { + if (dsft.getUri() != null) { + dsftMap.put(dsft.getUri(), dsft); + } + if (blockHasUri) { + if (dsft.getParentDatasetFieldType() != null) { + // ToDo - why not getName for child type? Would have to fix in ORE generation + // code and handle legacy bags + dsftMap.put(mdb.getNamespaceUri() + dsft.getParentDatasetFieldType().getName() + "#" + + dsft.getTitle(), dsft); + } else { + dsftMap.put(mdb.getNamespaceUri() + dsft.getTitle(), dsft); + } + } + } + } + logger.fine("DSFT Map: " + String.join(", ", dsftMap.keySet())); + } + } + + private static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { + if (localContext.isEmpty()) { + + // Add namespaces corresponding to core terms + localContext.put(JsonLDNamespace.dcterms.getPrefix(), JsonLDNamespace.dcterms.getUrl()); + localContext.put(JsonLDNamespace.dvcore.getPrefix(), JsonLDNamespace.dvcore.getUrl()); + localContext.put(JsonLDNamespace.schema.getPrefix(), JsonLDNamespace.schema.getUrl()); + + List mdbList = metadataBlockSvc.listMetadataBlocks(); + + for (MetadataBlock mdb : mdbList) { + boolean blockHasUri = mdb.getNamespaceUri() != null; + if (blockHasUri) { + localContext.putIfAbsent(mdb.getName(), mdb.getNamespaceUri()); + + } + for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { + if (dsft.getUri() != null) { + localContext.putIfAbsent(dsft.getName(), dsft.getUri()); + } + } + } + logger.fine("LocalContext keys: " + String.join(", ", localContext.keySet())); + } + } + + public static JsonObject decontextualizeJsonLD(String jsonLDString) { + logger.fine(jsonLDString); + try (StringReader rdr = new StringReader(jsonLDString)) { + + // Use JsonLd to expand/compact to localContext + JsonObject jsonld = Json.createReader(rdr).readObject(); + JsonDocument doc = JsonDocument.of(jsonld); + JsonArray array = null; + try { + array = JsonLd.expand(doc).get(); + jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(Json.createObjectBuilder().build())) + .get(); + // jsonld = array.getJsonObject(0); + logger.fine("Decontextualized object: " + jsonld); + return jsonld; + } catch (JsonLdError e) { + System.out.println(e.getMessage()); + return null; + } + } + } + + private static JsonObject recontextualizeJsonLD(JsonObject jsonldObj, MetadataBlockServiceBean metadataBlockSvc) { + + populateContext(metadataBlockSvc); + + // Use JsonLd to expand/compact to localContext + JsonDocument doc = JsonDocument.of(jsonldObj); + JsonArray array = null; + try { + array = JsonLd.expand(doc).get(); + + jsonldObj = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(JSONLDUtil.getContext(localContext))) + .get(); + logger.fine("Compacted: " + jsonldObj.toString()); + return jsonldObj; + } catch (JsonLdError e) { + System.out.println(e.getMessage()); + return null; + } + } + + public static String prettyPrint(JsonValue val) { + StringWriter sw = new StringWriter(); + Map properties = new HashMap<>(1); + properties.put(JsonGenerator.PRETTY_PRINTING, true); + JsonWriterFactory writerFactory = Json.createWriterFactory(properties); + JsonWriter jsonWriter = writerFactory.createWriter(sw); + jsonWriter.write(val); + jsonWriter.close(); + return sw.toString(); + } //Modified from https://stackoverflow.com/questions/3389348/parse-any-date-in-java - private static final Map DATE_FORMAT_REGEXPS = new HashMap() { - { - put("^\\d{8}$", "yyyyMMdd"); - put("^\\d{1,2}-\\d{1,2}-\\d{4}$", "dd-MM-yyyy"); - put("^\\d{4}-\\d{1,2}-\\d{1,2}$", "yyyy-MM-dd"); - put("^\\d{1,2}/\\d{1,2}/\\d{4}$", "MM/dd/yyyy"); - put("^\\d{4}/\\d{1,2}/\\d{1,2}$", "yyyy/MM/dd"); - put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}$", "dd MMM yyyy"); - put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}$", "dd MMMM yyyy"); - } - }; - - private static final Map DATETIME_FORMAT_REGEXPS = new HashMap() { - { - put("^\\d{12}$", "yyyyMMddHHmm"); - put("^\\d{8}\\s\\d{4}$", "yyyyMMdd HHmm"); - put("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}$", "dd-MM-yyyy HH:mm"); - put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}$", "yyyy-MM-dd HH:mm"); - put("^\\d{1,2}/\\d{1,2}/\\d{4}\\s\\d{1,2}:\\d{2}$", "MM/dd/yyyy HH:mm"); - put("^\\d{4}/\\d{1,2}/\\d{1,2}\\s\\d{1,2}:\\d{2}$", "yyyy/MM/dd HH:mm"); - put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}$", "dd MMM yyyy HH:mm"); - put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}$", "dd MMMM yyyy HH:mm"); - put("^\\d{14}$", "yyyyMMddHHmmss"); - put("^\\d{8}\\s\\d{6}$", "yyyyMMdd HHmmss"); - put("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd-MM-yyyy HH:mm:ss"); - put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}$", "yyyy-MM-dd HH:mm:ss"); - put("^\\d{1,2}/\\d{1,2}/\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "MM/dd/yyyy HH:mm:ss"); - put("^\\d{4}/\\d{1,2}/\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}$", "yyyy/MM/dd HH:mm:ss"); - put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMM yyyy HH:mm:ss"); - put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMMM yyyy HH:mm:ss"); - put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\.\\d{3}$", "yyyy-MM-dd HH:mm:ss.SSS"); - put("^[a-z,A-Z]{3}\\s[a-z,A-Z]{3}\\s\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\s[a-z,A-Z]{3}\\s\\d{4}$", - "EEE MMM dd HH:mm:ss zzz yyyy"); // Wed Sep 23 19:33:46 UTC 2020 - - } - }; - - /** - * Determine DateTimeFormatter pattern matching with the given date string. - * Returns null if format is unknown. You can simply extend DateUtil with more - * formats if needed. - * - * @param dateString The date string to determine the SimpleDateFormat pattern - * for. - * @return The matching SimpleDateFormat pattern, or null if format is unknown. - * @see SimpleDateFormat - */ - public static DateTimeFormatter determineDateTimeFormat(String dateString) { - for (String regexp : DATETIME_FORMAT_REGEXPS.keySet()) { - if (dateString.toLowerCase().matches(regexp)) { - return DateTimeFormatter.ofPattern(DATETIME_FORMAT_REGEXPS.get(regexp)); - } - } - logger.warning("Unknown datetime format: " + dateString); - return null; // Unknown format. - } - - public static DateTimeFormatter determineDateFormat(String dateString) { - for (String regexp : DATE_FORMAT_REGEXPS.keySet()) { - if (dateString.toLowerCase().matches(regexp)) { - return DateTimeFormatter.ofPattern(DATE_FORMAT_REGEXPS.get(regexp)); - } - } - logger.warning("Unknown date format: " + dateString); - return null; // Unknown format. - } - - public static LocalDateTime getDateTimeFrom(String dateString) { - DateTimeFormatter dtf = determineDateTimeFormat(dateString); - if (dtf != null) { - return LocalDateTime.parse(dateString, dtf); - } else { - dtf = determineDateFormat(dateString); - if (dtf != null) { - return LocalDate.parse(dateString, dtf).atStartOfDay(); - } - } - - return null; - } + private static final Map DATE_FORMAT_REGEXPS = new HashMap() { + { + put("^\\d{8}$", "yyyyMMdd"); + put("^\\d{1,2}-\\d{1,2}-\\d{4}$", "dd-MM-yyyy"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}$", "yyyy-MM-dd"); + put("^\\d{1,2}/\\d{1,2}/\\d{4}$", "MM/dd/yyyy"); + put("^\\d{4}/\\d{1,2}/\\d{1,2}$", "yyyy/MM/dd"); + put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}$", "dd MMM yyyy"); + put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}$", "dd MMMM yyyy"); + } + }; + + private static final Map DATETIME_FORMAT_REGEXPS = new HashMap() { + { + put("^\\d{12}$", "yyyyMMddHHmm"); + put("^\\d{8}\\s\\d{4}$", "yyyyMMdd HHmm"); + put("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}$", "dd-MM-yyyy HH:mm"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}$", "yyyy-MM-dd HH:mm"); + put("^\\d{1,2}/\\d{1,2}/\\d{4}\\s\\d{1,2}:\\d{2}$", "MM/dd/yyyy HH:mm"); + put("^\\d{4}/\\d{1,2}/\\d{1,2}\\s\\d{1,2}:\\d{2}$", "yyyy/MM/dd HH:mm"); + put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}$", "dd MMM yyyy HH:mm"); + put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}$", "dd MMMM yyyy HH:mm"); + put("^\\d{14}$", "yyyyMMddHHmmss"); + put("^\\d{8}\\s\\d{6}$", "yyyyMMdd HHmmss"); + put("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd-MM-yyyy HH:mm:ss"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}$", "yyyy-MM-dd HH:mm:ss"); + put("^\\d{1,2}/\\d{1,2}/\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "MM/dd/yyyy HH:mm:ss"); + put("^\\d{4}/\\d{1,2}/\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}$", "yyyy/MM/dd HH:mm:ss"); + put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMM yyyy HH:mm:ss"); + put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMMM yyyy HH:mm:ss"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\.\\d{3}$", "yyyy-MM-dd HH:mm:ss.SSS"); + put("^[a-z,A-Z]{3}\\s[a-z,A-Z]{3}\\s\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\s[a-z,A-Z]{3}\\s\\d{4}$", + "EEE MMM dd HH:mm:ss zzz yyyy"); // Wed Sep 23 19:33:46 UTC 2020 + + } + }; + + /** + * Determine DateTimeFormatter pattern matching with the given date string. + * Returns null if format is unknown. You can simply extend DateUtil with more + * formats if needed. + * + * @param dateString The date string to determine the SimpleDateFormat pattern + * for. + * @return The matching SimpleDateFormat pattern, or null if format is unknown. + * @see SimpleDateFormat + */ + public static DateTimeFormatter determineDateTimeFormat(String dateString) { + for (String regexp : DATETIME_FORMAT_REGEXPS.keySet()) { + if (dateString.toLowerCase().matches(regexp)) { + return DateTimeFormatter.ofPattern(DATETIME_FORMAT_REGEXPS.get(regexp)); + } + } + logger.warning("Unknown datetime format: " + dateString); + return null; // Unknown format. + } + + public static DateTimeFormatter determineDateFormat(String dateString) { + for (String regexp : DATE_FORMAT_REGEXPS.keySet()) { + if (dateString.toLowerCase().matches(regexp)) { + return DateTimeFormatter.ofPattern(DATE_FORMAT_REGEXPS.get(regexp)); + } + } + logger.warning("Unknown date format: " + dateString); + return null; // Unknown format. + } + + public static LocalDateTime getDateTimeFrom(String dateString) { + DateTimeFormatter dtf = determineDateTimeFormat(dateString); + if (dtf != null) { + return LocalDateTime.parse(dateString, dtf); + } else { + dtf = determineDateFormat(dateString); + if (dtf != null) { + return LocalDate.parse(dateString, dtf).atStartOfDay(); + } + } + + return null; + } + + // Convenience methods for TermsOfUseAndAccess + + public static final List datasetTerms = new ArrayList(Arrays.asList( + "https://dataverse.org/schema/core#termsOfUse", + "https://dataverse.org/schema/core#confidentialityDeclaration", + "https://dataverse.org/schema/core#specialPermissions", "https://dataverse.org/schema/core#restrictions", + "https://dataverse.org/schema/core#citationRequirements", + "https://dataverse.org/schema/core#depositorRequirements", "https://dataverse.org/schema/core#conditions", + "https://dataverse.org/schema/core#disclaimer")); + public static final List datafileTerms = new ArrayList(Arrays.asList( + "https://dataverse.org/schema/core#termsOfAccess", "https://dataverse.org/schema/core#fileRequestAccess", + "https://dataverse.org/schema/core#dataAccessPlace", "https://dataverse.org/schema/core#originalArchive", + "https://dataverse.org/schema/core#availabilityStatus", + "https://dataverse.org/schema/core#contactForAccess", "https://dataverse.org/schema/core#sizeOfCollection", + "https://dataverse.org/schema/core#studyCompletion")); + + public static boolean isSet(TermsOfUseAndAccess terms, String semterm) { + switch (semterm) { + case "http://schema.org/license": + return !terms.getLicense().equals(TermsOfUseAndAccess.License.NONE); + case "https://dataverse.org/schema/core#termsOfUse": + return StringUtils.isBlank(terms.getTermsOfUse()); + case "https://dataverse.org/schema/core#confidentialityDeclaration": + return StringUtils.isBlank(terms.getConfidentialityDeclaration()); + case "https://dataverse.org/schema/core#specialPermissions": + return StringUtils.isBlank(terms.getSpecialPermissions()); + case "https://dataverse.org/schema/core#restrictions": + return StringUtils.isBlank(terms.getRestrictions()); + case "https://dataverse.org/schema/core#citationRequirements": + return StringUtils.isBlank(terms.getCitationRequirements()); + case "https://dataverse.org/schema/core#depositorRequirements": + return StringUtils.isBlank(terms.getDepositorRequirements()); + case "https://dataverse.org/schema/core#conditions": + return StringUtils.isBlank(terms.getConditions()); + case "https://dataverse.org/schema/core#disclaimer": + return StringUtils.isBlank(terms.getDisclaimer()); + case "https://dataverse.org/schema/core#termsOfAccess": + return StringUtils.isBlank(terms.getTermsOfAccess()); + case "https://dataverse.org/schema/core#fileRequestAccess": + return !terms.isFileAccessRequest(); + case "https://dataverse.org/schema/core#dataAccessPlace": + return StringUtils.isBlank(terms.getDataAccessPlace()); + case "https://dataverse.org/schema/core#originalArchive": + return StringUtils.isBlank(terms.getOriginalArchive()); + case "https://dataverse.org/schema/core#availabilityStatus": + return StringUtils.isBlank(terms.getAvailabilityStatus()); + case "https://dataverse.org/schema/core#contactForAccess": + return StringUtils.isBlank(terms.getContactForAccess()); + case "https://dataverse.org/schema/core#sizeOfCollection": + return StringUtils.isBlank(terms.getSizeOfCollection()); + case "https://dataverse.org/schema/core#studyCompletion": + return StringUtils.isBlank(terms.getStudyCompletion()); + default: + logger.warning("isSet called for " + semterm); + return false; + } + } + + public static void setSemTerm(TermsOfUseAndAccess terms, String semterm, Object value) { + switch (semterm) { + case "http://schema.org/license": + // Mirror rules from SwordServiceBean + if (((License) value).equals(TermsOfUseAndAccess.defaultLicense)) { + terms.setLicense(TermsOfUseAndAccess.defaultLicense); + } else { + throw new BadRequestException("The only allowed value for " + JsonLDTerm.schemaOrg("license").getUrl() + + " is " + "https://creativecommons.org/publicdomain/zero/1.0/"); + } + break; + case "https://dataverse.org/schema/core#termsOfUse": + terms.setTermsOfUse((String) value); + break; + case "https://dataverse.org/schema/core#confidentialityDeclaration": + terms.setConfidentialityDeclaration((String) value); + break; + case "https://dataverse.org/schema/core#specialPermissions": + terms.setSpecialPermissions((String) value); + break; + case "https://dataverse.org/schema/core#restrictions": + terms.setRestrictions((String) value); + break; + case "https://dataverse.org/schema/core#citationRequirements": + terms.setCitationRequirements((String) value); + break; + case "https://dataverse.org/schema/core#depositorRequirements": + terms.setDepositorRequirements((String) value); + break; + case "https://dataverse.org/schema/core#conditions": + terms.setConditions((String) value); + break; + case "https://dataverse.org/schema/core#disclaimer": + terms.setDisclaimer((String) value); + break; + case "https://dataverse.org/schema/core#termsOfAccess": + terms.setTermsOfAccess((String) value); + break; + case "https://dataverse.org/schema/core#fileRequestAccess": + terms.setFileAccessRequest((boolean) value); + break; + case "https://dataverse.org/schema/core#dataAccessPlace": + terms.setDataAccessPlace((String) value); + break; + case "https://dataverse.org/schema/core#originalArchive": + terms.setOriginalArchive((String) value); + break; + case "https://dataverse.org/schema/core#availabilityStatus": + terms.setAvailabilityStatus((String) value); + break; + case "https://dataverse.org/schema/core#contactForAccess": + terms.setContactForAccess((String) value); + break; + case "https://dataverse.org/schema/core#sizeOfCollection": + terms.setSizeOfCollection((String) value); + break; + case "https://dataverse.org/schema/core#studyCompletion": + terms.setStudyCompletion((String) value); + break; + default: + logger.warning("setSemTerm called for " + semterm); + break; + } + } } From 34a28a37180c3c1a698ad359c88ff945a5ed0f53 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 3 Dec 2020 13:13:28 -0500 Subject: [PATCH 058/354] expose uri in datasetField api --- .../edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java index d02e8f72838..c945ba4df56 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java @@ -137,6 +137,7 @@ public Response getByName(@PathParam("name") String name) { String solrFieldSearchable = dsf.getSolrField().getNameSearchable(); String solrFieldFacetable = dsf.getSolrField().getNameFacetable(); String metadataBlock = dsf.getMetadataBlock().getName(); + String uri=dsf.getUri(); boolean hasParent = dsf.isHasParent(); boolean allowsMultiples = dsf.isAllowMultiples(); boolean isRequired = dsf.isRequired(); @@ -168,7 +169,8 @@ public Response getByName(@PathParam("name") String name) { .add("parentAllowsMultiples", parentAllowsMultiplesDisplay) .add("solrFieldSearchable", solrFieldSearchable) .add("solrFieldFacetable", solrFieldFacetable) - .add("isRequired", isRequired)); + .add("isRequired", isRequired) + .add("uri", uri)); } catch ( NoResultException nre ) { return notFound(name); From 1b98b2c959b2f8ad92c020e2ffb05efc5a645551 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 3 Dec 2020 14:27:14 -0500 Subject: [PATCH 059/354] track defined namespaces and avoid having contexts with specific entries for terms that are in a namespace already --- .../iq/dataverse/util/bagit/OREMap.java | 4 +-- .../iq/dataverse/util/json/JSONLDUtil.java | 12 +++---- .../dataverse/util/json/JsonLDNamespace.java | 34 ++++++++++++++++++- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 5520de3954e..caca48d73d4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -341,7 +341,7 @@ private JsonLDTerm getTermFor(DatasetFieldType dsft) { namespaceUri = SystemConfig.getDataverseSiteUrlStatic() + "/schema/" + dsft.getMetadataBlock().getName() + "#"; } - JsonLDNamespace blockNamespace = new JsonLDNamespace(dsft.getMetadataBlock().getName(), namespaceUri); + JsonLDNamespace blockNamespace = JsonLDNamespace.defineNamespace(dsft.getMetadataBlock().getName(), namespaceUri); return new JsonLDTerm(blockNamespace, dsft.getTitle()); } } @@ -357,7 +357,7 @@ private JsonLDTerm getTermFor(DatasetFieldType dfType, DatasetFieldType dsft) { + dfType.getMetadataBlock().getName() + "/"; } subFieldNamespaceUri = subFieldNamespaceUri + dfType.getName() + "#"; - JsonLDNamespace fieldNamespace = new JsonLDNamespace(dfType.getName(), subFieldNamespaceUri); + JsonLDNamespace fieldNamespace = JsonLDNamespace.defineNamespace(dfType.getName(), subFieldNamespaceUri); return new JsonLDTerm(fieldNamespace, dsft.getTitle()); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 414b3ec8a01..4f451f01ca5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -452,25 +452,21 @@ private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockS private static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { if (localContext.isEmpty()) { - // Add namespaces corresponding to core terms - localContext.put(JsonLDNamespace.dcterms.getPrefix(), JsonLDNamespace.dcterms.getUrl()); - localContext.put(JsonLDNamespace.dvcore.getPrefix(), JsonLDNamespace.dvcore.getUrl()); - localContext.put(JsonLDNamespace.schema.getPrefix(), JsonLDNamespace.schema.getUrl()); - List mdbList = metadataBlockSvc.listMetadataBlocks(); for (MetadataBlock mdb : mdbList) { boolean blockHasUri = mdb.getNamespaceUri() != null; if (blockHasUri) { - localContext.putIfAbsent(mdb.getName(), mdb.getNamespaceUri()); - + JsonLDNamespace.defineNamespace(mdb.getName(), mdb.getNamespaceUri()); } for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { - if (dsft.getUri() != null) { + if ((dsft.getUri() != null) && !JsonLDNamespace.isInNamespace(dsft.getUri())) { + //Add term if uri exists and it's not in one of the namespaces already defined localContext.putIfAbsent(dsft.getName(), dsft.getUri()); } } } + JsonLDNamespace.addNamespacesToContext(localContext); logger.fine("LocalContext keys: " + String.join(", ", localContext.keySet())); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java index bda4a55d623..2c870edc106 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java @@ -1,5 +1,10 @@ package edu.harvard.iq.dataverse.util.json; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + public class JsonLDNamespace { String prefix; @@ -12,7 +17,34 @@ public class JsonLDNamespace { public static JsonLDNamespace ore = new JsonLDNamespace("ore","http://www.openarchives.org/ore/terms/"); public static JsonLDNamespace schema = new JsonLDNamespace("schema","http://schema.org/"); - public JsonLDNamespace(String prefix, String url) { + private static List namespaces = new ArrayList(Arrays.asList(dvcore, dcterms, ore, schema)); + + public static JsonLDNamespace defineNamespace(String prefix, String url) { + JsonLDNamespace ns = new JsonLDNamespace(prefix, url); + namespaces.add(ns); + return ns; + } + + public static void deleteNamespace(JsonLDNamespace ns) { + namespaces.remove(ns); + } + + public static boolean isInNamespace(String url) { + for(JsonLDNamespace ns: namespaces) { + if(url.startsWith(ns.getUrl())) { + return true; + } + } + return false; + } + + public static void addNamespacesToContext(Map context) { + for(JsonLDNamespace ns: namespaces) { + context.putIfAbsent(ns.getPrefix(), ns.getUrl()); + }; + } + + private JsonLDNamespace(String prefix, String url) { this.prefix = prefix; this.url = url; } From 55a8b303b124b1d9982c1aba620c443a499628c0 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 3 Dec 2020 15:22:37 -0500 Subject: [PATCH 060/354] define equals, avoid duplicates in list --- .../dataverse/util/json/JsonLDNamespace.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java index 2c870edc106..904419775c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java @@ -4,6 +4,9 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; + +import edu.harvard.iq.dataverse.DataFile; public class JsonLDNamespace { @@ -20,9 +23,15 @@ public class JsonLDNamespace { private static List namespaces = new ArrayList(Arrays.asList(dvcore, dcterms, ore, schema)); public static JsonLDNamespace defineNamespace(String prefix, String url) { + JsonLDNamespace ns = new JsonLDNamespace(prefix, url); - namespaces.add(ns); - return ns; + int i = namespaces.indexOf(ns); + if(i>=0) { + return namespaces.get(i); + } else { + namespaces.add(ns); + return ns; + } } public static void deleteNamespace(JsonLDNamespace ns) { @@ -56,5 +65,14 @@ public String getPrefix() { public String getUrl() { return url; } + + @Override + public boolean equals(Object object) { + if (!(object instanceof JsonLDNamespace)) { + return false; + } + JsonLDNamespace other = (JsonLDNamespace) object; + return (other.getPrefix().equals(getPrefix()) && other.getUrl().equals(getUrl())); + } } From 966394ad33ded6939f9d606d0ca4682cb32f25a8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 4 Dec 2020 15:27:43 -0500 Subject: [PATCH 061/354] replace string with const --- .../java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java | 2 +- .../iq/dataverse/export/openaire/OpenAireExportUtil.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java b/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java index ad6775d6efd..30eae110102 100644 --- a/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java @@ -280,7 +280,7 @@ public enum License { * API use? See also https://github.com/IQSS/dataverse/issues/1385 */ public static TermsOfUseAndAccess.License defaultLicense = TermsOfUseAndAccess.License.CC0; - + public final static String CC0_URI = "https://creativecommons.org/publicdomain/zero/1.0/"; @Override public int hashCode() { int hash = 0; diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java index fe0c15969ca..f972ae2a983 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java @@ -19,6 +19,7 @@ import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.api.dto.DatasetDTO; import edu.harvard.iq.dataverse.api.dto.DatasetVersionDTO; import edu.harvard.iq.dataverse.api.dto.FieldDTO; @@ -1135,7 +1136,7 @@ public static void writeAccessRightsElement(XMLStreamWriter xmlw, DatasetVersion writeRightsHeader(xmlw, language); if (StringUtils.isNotBlank(datasetVersionDTO.getLicense())) { if (StringUtils.containsIgnoreCase(datasetVersionDTO.getLicense(), "cc0")) { - xmlw.writeAttribute("rightsURI", "https://creativecommons.org/publicdomain/zero/1.0/"); + xmlw.writeAttribute("rightsURI", TermsOfUseAndAccess.CC0_URI); if (StringUtils.isNotBlank(datasetVersionDTO.getTermsOfUse())) { xmlw.writeCharacters(datasetVersionDTO.getTermsOfUse()); } From b83f7b2f134ce87e5cf2b1d7af8182b756f1faf0 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 4 Dec 2020 15:28:14 -0500 Subject: [PATCH 062/354] constant for CC0_URI --- src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 83152d1a75f..53a58b4228f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1856,7 +1856,7 @@ public String getJsonLd() { JsonObjectBuilder license = Json.createObjectBuilder().add("@type", "Dataset"); if (TermsOfUseAndAccess.License.CC0.equals(terms.getLicense())) { - license.add("text", "CC0").add("url", "https://creativecommons.org/publicdomain/zero/1.0/"); + license.add("text", "CC0").add("url", TermsOfUseAndAccess.CC0_URI); } else { String termsOfUse = terms.getTermsOfUse(); // Terms of use can be null if you create the dataset with JSON. From 1e08f102e617fcfcdeea7482d03e5aeae8270c11 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 4 Dec 2020 15:28:32 -0500 Subject: [PATCH 063/354] GET/DELETE endpoints --- .../iq/dataverse/TermsOfUseAndAccess.java | 2 +- .../harvard/iq/dataverse/api/Datasets.java | 73 +++++- .../iq/dataverse/util/bagit/OREMap.java | 179 +++++++------- .../iq/dataverse/util/json/JSONLDUtil.java | 220 +++++++++++++++++- 4 files changed, 374 insertions(+), 100 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java b/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java index 30eae110102..72f4ab54ee8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java @@ -280,7 +280,7 @@ public enum License { * API use? See also https://github.com/IQSS/dataverse/issues/1385 */ public static TermsOfUseAndAccess.License defaultLicense = TermsOfUseAndAccess.License.CC0; - public final static String CC0_URI = "https://creativecommons.org/publicdomain/zero/1.0/"; + public static String CC0_URI = "https://creativecommons.org/publicdomain/zero/1.0/"; @Override public int hashCode() { int hash = 0; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 8ce029c1a35..35f17f0b602 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -101,6 +101,7 @@ import edu.harvard.iq.dataverse.util.EjbUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.util.json.JSONLDUtil; import edu.harvard.iq.dataverse.util.json.JsonParseException; import edu.harvard.iq.dataverse.search.IndexServiceBean; @@ -609,22 +610,80 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, } } + + @GET + @Path("{id}/versions/{versionid}/metadata") + @Produces("application/json-ld") + public Response getVersionMetadata(@PathParam("id") String id, @PathParam("versionId") String versionId) { + try { + Dataset ds = findDatasetOrDie(id); + DataverseRequest req = createDataverseRequest(findUserOrDie()); + DatasetVersion dsv = ds.getEditVersion(); + OREMap ore = new OREMap(dsv, + settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)); + return ok(JSONLDUtil.prettyPrint(ore.getOREMap(true))); + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } catch (Exception jpe) { + logger.log(Level.SEVERE, "Error getting jsonld metadata for dsv: ", jpe.getLocalizedMessage()); + return error(Response.Status.INTERNAL_SERVER_ERROR, jpe.getLocalizedMessage()); + } + } + + @GET + @Path("{id}/metadata") + @Produces("application/json-ld") + public Response getVersionMetadata(@PathParam("id") String id) { + return getVersionMetadata(id, ":draft"); + } + + @PUT + @Path("{id}/metadata") + @Consumes("application/json-ld") + public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, + @PathParam("versionId") String versionId, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { + + if (!":draft".equals(versionId)) { + return error(Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); + } + try { + Dataset ds = findDatasetOrDie(id); + DataverseRequest req = createDataverseRequest(findUserOrDie()); + DatasetVersion dsv = ds.getEditVersion(); + boolean updateDraft = ds.getLatestVersion().isDraft(); + dsv = JSONLDUtil.updateDatasetVersionMDFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, !replaceTerms, false); + + DatasetVersion managedVersion; + if (updateDraft) { + Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); + managedVersion = managedDataset.getEditVersion(); + } else { + managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); + } + String info = updateDraft ? "Version Updated" : "Version Created"; + return ok(Json.createObjectBuilder().add(info, managedVersion.getVersionDate())); + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } catch (JsonParsingException jpe) { + logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", jsonLDBody); + return error(Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage()); + } + } - @PUT - @Path("{id}/versions/{versionId}/metadata") + @DELETE + @Path("{id}/metadata") @Consumes("application/json-ld") - public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, + public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id, @PathParam("versionId") String versionId, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { - if (!":draft".equals(versionId)) { - return error(Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); - } try { Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); DatasetVersion dsv = ds.getEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); - dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, !replaceTerms, false); + dsv = JSONLDUtil.deleteDatasetVersionMDFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc); DatasetVersion managedVersion; if (updateDraft) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index caca48d73d4..657997ef642 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -37,7 +37,7 @@ public class OREMap { private Map localContext = new TreeMap(); private DatasetVersion version; private boolean excludeEmail = false; - + public OREMap(DatasetVersion version, boolean excludeEmail) { this.version = version; this.excludeEmail = excludeEmail; @@ -49,6 +49,10 @@ public void writeOREMap(OutputStream outputStream) throws Exception { } public JsonObject getOREMap() throws Exception { + return getOREMap(false); + } + + public JsonObject getOREMap(boolean aggregationOnly) throws Exception { // Add namespaces we'll definitely use to Context // Additional namespaces are added as needed below @@ -67,7 +71,7 @@ public JsonObject getOREMap() throws Exception { for (DatasetField field : fields) { if (!field.isEmpty()) { DatasetFieldType dfType = field.getDatasetFieldType(); - if(excludeEmail && DatasetFieldType.FieldType.EMAIL.equals(dfType.getFieldType())) { + if (excludeEmail && DatasetFieldType.FieldType.EMAIL.equals(dfType.getFieldType())) { continue; } JsonLDTerm fieldName = getTermFor(dfType); @@ -89,13 +93,13 @@ public JsonObject getOREMap() throws Exception { for (DatasetField dsf : dscv.getChildDatasetFields()) { DatasetFieldType dsft = dsf.getDatasetFieldType(); - if(excludeEmail && DatasetFieldType.FieldType.EMAIL.equals(dsft.getFieldType())) { + if (excludeEmail && DatasetFieldType.FieldType.EMAIL.equals(dsft.getFieldType())) { continue; } // which may have multiple values if (!dsf.isEmpty()) { - // Add context entry - //ToDo - also needs to recurse here? + // Add context entry + // ToDo - also needs to recurse here? JsonLDTerm subFieldName = getTermFor(dfType, dsft); if (subFieldName.inNamespace()) { localContext.putIfAbsent(subFieldName.getNamespace().getPrefix(), @@ -138,7 +142,7 @@ public JsonObject getOREMap() throws Exception { TermsOfUseAndAccess terms = version.getTermsOfUseAndAccess(); if (terms.getLicense() == TermsOfUseAndAccess.License.CC0) { aggBuilder.add(JsonLDTerm.schemaOrg("license").getLabel(), - "https://creativecommons.org/publicdomain/zero/1.0/"); + TermsOfUseAndAccess.CC0_URI); } else { addIfNotNull(aggBuilder, JsonLDTerm.termsOfUse, terms.getTermsOfUse()); } @@ -171,96 +175,101 @@ public JsonObject getOREMap() throws Exception { // The aggregation aggregates aggregatedresources (Datafiles) which each have // their own entry and metadata JsonArrayBuilder aggResArrayBuilder = Json.createArrayBuilder(); + if (!aggregationOnly) { - for (FileMetadata fmd : version.getFileMetadatas()) { - DataFile df = fmd.getDataFile(); - JsonObjectBuilder aggRes = Json.createObjectBuilder(); + for (FileMetadata fmd : version.getFileMetadatas()) { + DataFile df = fmd.getDataFile(); + JsonObjectBuilder aggRes = Json.createObjectBuilder(); - if (fmd.getDescription() != null) { - aggRes.add(JsonLDTerm.schemaOrg("description").getLabel(), fmd.getDescription()); - } else { - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("description"), df.getDescription()); - } - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("name"), fmd.getLabel()); // "label" is the filename - addIfNotNull(aggRes, JsonLDTerm.restricted, fmd.isRestricted()); - addIfNotNull(aggRes, JsonLDTerm.directoryLabel, fmd.getDirectoryLabel()); - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("version"), fmd.getVersion()); - addIfNotNull(aggRes, JsonLDTerm.datasetVersionId, fmd.getDatasetVersion().getId()); - JsonArray catArray = null; - if (fmd != null) { - List categories = fmd.getCategoriesByName(); - if (categories.size() > 0) { - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (String s : categories) { - jab.add(s); + if (fmd.getDescription() != null) { + aggRes.add(JsonLDTerm.schemaOrg("description").getLabel(), fmd.getDescription()); + } else { + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("description"), df.getDescription()); + } + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("name"), fmd.getLabel()); // "label" is the filename + addIfNotNull(aggRes, JsonLDTerm.restricted, fmd.isRestricted()); + addIfNotNull(aggRes, JsonLDTerm.directoryLabel, fmd.getDirectoryLabel()); + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("version"), fmd.getVersion()); + addIfNotNull(aggRes, JsonLDTerm.datasetVersionId, fmd.getDatasetVersion().getId()); + JsonArray catArray = null; + if (fmd != null) { + List categories = fmd.getCategoriesByName(); + if (categories.size() > 0) { + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (String s : categories) { + jab.add(s); + } + catArray = jab.build(); } - catArray = jab.build(); } + addIfNotNull(aggRes, JsonLDTerm.categories, catArray); + // File DOI if it exists + String fileId = null; + String fileSameAs = null; + if (df.getGlobalId().asString().length() != 0) { + fileId = df.getGlobalId().asString(); + fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + + "/api/access/datafile/:persistentId?persistentId=" + fileId; + } else { + fileId = SystemConfig.getDataverseSiteUrlStatic() + "/file.xhtml?fileId=" + df.getId(); + fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId(); + } + aggRes.add("@id", fileId); + aggRes.add(JsonLDTerm.schemaOrg("sameAs").getLabel(), fileSameAs); + fileArray.add(fileId); + + aggRes.add("@type", JsonLDTerm.ore("AggregatedResource").getLabel()); + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), df.getContentType()); + addIfNotNull(aggRes, JsonLDTerm.filesize, df.getFilesize()); + addIfNotNull(aggRes, JsonLDTerm.storageIdentifier, df.getStorageIdentifier()); + addIfNotNull(aggRes, JsonLDTerm.originalFileFormat, df.getOriginalFileFormat()); + addIfNotNull(aggRes, JsonLDTerm.originalFormatLabel, df.getOriginalFormatLabel()); + addIfNotNull(aggRes, JsonLDTerm.UNF, df.getUnf()); + addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId()); + addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId()); + JsonObject checksum = null; + // Add checksum. RDA recommends SHA-512 + if (df.getChecksumType() != null && df.getChecksumValue() != null) { + checksum = Json.createObjectBuilder().add("@type", df.getChecksumType().toString()) + .add("@value", df.getChecksumValue()).build(); + aggRes.add(JsonLDTerm.checksum.getLabel(), checksum); + } + JsonArray tabTags = null; + JsonArrayBuilder jab = JsonPrinter.getTabularFileTags(df); + if (jab != null) { + tabTags = jab.build(); + } + addIfNotNull(aggRes, JsonLDTerm.tabularTags, tabTags); + // Add latest resource to the array + aggResArrayBuilder.add(aggRes.build()); } - addIfNotNull(aggRes, JsonLDTerm.categories, catArray); - // File DOI if it exists - String fileId = null; - String fileSameAs = null; - if (df.getGlobalId().asString().length() != 0) { - fileId = df.getGlobalId().asString(); - fileSameAs = SystemConfig.getDataverseSiteUrlStatic() - + "/api/access/datafile/:persistentId?persistentId=" + fileId; - } else { - fileId = SystemConfig.getDataverseSiteUrlStatic() + "/file.xhtml?fileId=" + df.getId(); - fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId(); - } - aggRes.add("@id", fileId); - aggRes.add(JsonLDTerm.schemaOrg("sameAs").getLabel(), fileSameAs); - fileArray.add(fileId); - - aggRes.add("@type", JsonLDTerm.ore("AggregatedResource").getLabel()); - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), df.getContentType()); - addIfNotNull(aggRes, JsonLDTerm.filesize, df.getFilesize()); - addIfNotNull(aggRes, JsonLDTerm.storageIdentifier, df.getStorageIdentifier()); - addIfNotNull(aggRes, JsonLDTerm.originalFileFormat, df.getOriginalFileFormat()); - addIfNotNull(aggRes, JsonLDTerm.originalFormatLabel, df.getOriginalFormatLabel()); - addIfNotNull(aggRes, JsonLDTerm.UNF, df.getUnf()); - addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId()); - addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId()); - JsonObject checksum = null; - // Add checksum. RDA recommends SHA-512 - if (df.getChecksumType() != null && df.getChecksumValue() != null) { - checksum = Json.createObjectBuilder().add("@type", df.getChecksumType().toString()) - .add("@value", df.getChecksumValue()).build(); - aggRes.add(JsonLDTerm.checksum.getLabel(), checksum); - } - JsonArray tabTags = null; - JsonArrayBuilder jab = JsonPrinter.getTabularFileTags(df); - if (jab != null) { - tabTags = jab.build(); - } - addIfNotNull(aggRes, JsonLDTerm.tabularTags, tabTags); - //Add latest resource to the array - aggResArrayBuilder.add(aggRes.build()); } // Build the '@context' object for json-ld based on the localContext entries JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); for (Entry e : localContext.entrySet()) { contextBuilder.add(e.getKey(), e.getValue()); } - // Now create the overall map object with it's metadata - JsonObject oremap = Json.createObjectBuilder() - .add(JsonLDTerm.dcTerms("modified").getLabel(), LocalDate.now().toString()) - .add(JsonLDTerm.dcTerms("creator").getLabel(), - BundleUtil.getStringFromBundle("institution.name")) - .add("@type", JsonLDTerm.ore("ResourceMap").getLabel()) - // Define an id for the map itself (separate from the @id of the dataset being - // described - .add("@id", - SystemConfig.getDataverseSiteUrlStatic() + "/api/datasets/export?exporter=" - + OAI_OREExporter.NAME + "&persistentId=" + id) - // Add the aggregation (Dataset) itself to the map. - .add(JsonLDTerm.ore("describes").getLabel(), - aggBuilder.add(JsonLDTerm.ore("aggregates").getLabel(), aggResArrayBuilder.build()) - .add(JsonLDTerm.schemaOrg("hasPart").getLabel(), fileArray.build()).build()) - // and finally add the context - .add("@context", contextBuilder.build()).build(); - return oremap; + if (aggregationOnly) { + return aggBuilder.add("@context", contextBuilder.build()).build(); + } else { + // Now create the overall map object with it's metadata + JsonObject oremap = Json.createObjectBuilder() + .add(JsonLDTerm.dcTerms("modified").getLabel(), LocalDate.now().toString()) + .add(JsonLDTerm.dcTerms("creator").getLabel(), BundleUtil.getStringFromBundle("institution.name")) + .add("@type", JsonLDTerm.ore("ResourceMap").getLabel()) + // Define an id for the map itself (separate from the @id of the dataset being + // described + .add("@id", + SystemConfig.getDataverseSiteUrlStatic() + "/api/datasets/export?exporter=" + + OAI_OREExporter.NAME + "&persistentId=" + id) + // Add the aggregation (Dataset) itself to the map. + .add(JsonLDTerm.ore("describes").getLabel(), + aggBuilder.add(JsonLDTerm.ore("aggregates").getLabel(), aggResArrayBuilder.build()) + .add(JsonLDTerm.schemaOrg("hasPart").getLabel(), fileArray.build()).build()) + // and finally add the context + .add("@context", contextBuilder.build()).build(); + return oremap; + } } /* diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 4f451f01ca5..822a5f4008d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -75,7 +75,7 @@ public static JsonObject getContext(Map contextMap) { return contextBuilder.build(); } - public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, + public static Dataset updateDatasetMDFromJsonLD(Dataset ds, String jsonLDBody, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { DatasetVersion dsv = new DatasetVersion(); @@ -90,7 +90,7 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, + "'. Make sure it is in valid form - see Dataverse Native API documentation."); } - dsv = updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); + dsv = updateDatasetVersionMDFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); dsv.setDataset(ds); List versions = new ArrayList<>(1); @@ -111,10 +111,10 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, return ds; } - public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, String jsonLDBody, + public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv, String jsonLDBody, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); - return updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); + return updateDatasetVersionMDFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); } /** @@ -129,7 +129,7 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, * value(s) for fields found in the json-ld. * @return */ - public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, JsonObject jsonld, + public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv, JsonObject jsonld, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { populateFieldTypeMap(metadataBlockSvc); @@ -294,6 +294,104 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, return dsv; } + /** + * + * @param dsv + * @param jsonLDBody + * @param metadataBlockService + * @param datasetFieldSvc + * @param b + * @param c + * @return + */ + public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv, String jsonLDBody, + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { + + JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); + //All terms are now URIs + //Setup dsftMap - URI to datasetFieldType map + populateFieldTypeMap(metadataBlockSvc); + + + //Another map - from datasetFieldType to an existing field in the dataset + List dsfl = dsv.getDatasetFields(); + Map fieldByTypeMap = new HashMap(); + for (DatasetField dsf : dsfl) { + if (fieldByTypeMap.containsKey(dsf.getDatasetFieldType())) { + // May have multiple values per field, but not multiple fields of one type? + logger.warning("Multiple fields of type " + dsf.getDatasetFieldType().getName()); + } + fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); + } + + TermsOfUseAndAccess terms = dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess(); + + //Iterate through input json + for (String key : jsonld.keySet()) { + //Skip context (shouldn't be present with decontextualize used above) + if (!key.equals("@context")) { + if (dsftMap.containsKey(key)) { + //THere's a field type with theis URI + DatasetFieldType dsft = dsftMap.get(key); + DatasetField dsf = null; + if (fieldByTypeMap.containsKey(dsft)) { + //There's a field of this type + dsf = fieldByTypeMap.get(dsft); + + // Todo - normalize object vs. array + JsonArray valArray = getValues(jsonld.get(key), dsft.isAllowMultiples(), dsft.getName()); + + DatasetField dsf2 = getReplacementField(dsf, valArray); + if(dsf2 == null) { + //Exact match - remove the field + dsfl.remove(dsf); + } else if(!dsf2.equals(dsf)) { + //Partial match - some values of a multivalue field match, so keep the remaining values + dsfl.remove(dsf); + dsfl.add(dsf2); + } + } + } else { + // Internal/non-metadatablock terms + if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())) { + if(jsonld.getString(key).equals(TermsOfUseAndAccess.CC0_URI)) { + setSemTerm(terms, key, TermsOfUseAndAccess.License.NONE); + } else { + throw new BadRequestException( + "Term: " + key + " with value: " + jsonld.getString(key) + " not found."); + } + } else if (datasetTerms.contains(key)) { + if(!deleteIfSemTermMatches(terms, key, jsonld.get(key))) { + throw new BadRequestException( + "Term: " + key + " with value: " + jsonld.getString(key) + " not found."); + } + } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { + JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); + for (String fileKey : fAccessObject.keySet()) { + if (datafileTerms.contains(fileKey)) { + if(!deleteIfSemTermMatches(terms, key, jsonld.get(key))) { + throw new BadRequestException( + "Term: " + key + " with value: " + jsonld.getString(key) + " not found."); + } + } + } + } else { + throw new BadRequestException( + "Term: " + key + " not found."); + } + + dsv.setTermsOfUseAndAccess(terms); + } + } + } + dsv.setDatasetFields(dsfl); + return dsv; + } + + private static DatasetField getReplacementField(DatasetField dsf, JsonArray valArray) { + // TODO Auto-generated method stub + return null; + } private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldType dsft, DatasetFieldServiceBean datasetFieldSvc, boolean append) { @@ -449,7 +547,7 @@ private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockS } } - private static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { + public static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { if (localContext.isEmpty()) { List mdbList = metadataBlockSvc.listMetadataBlocks(); @@ -673,7 +771,7 @@ public static void setSemTerm(TermsOfUseAndAccess terms, String semterm, Object terms.setLicense(TermsOfUseAndAccess.defaultLicense); } else { throw new BadRequestException("The only allowed value for " + JsonLDTerm.schemaOrg("license").getUrl() - + " is " + "https://creativecommons.org/publicdomain/zero/1.0/"); + + " is " + TermsOfUseAndAccess.CC0_URI); } break; case "https://dataverse.org/schema/core#termsOfUse": @@ -729,4 +827,112 @@ public static void setSemTerm(TermsOfUseAndAccess terms, String semterm, Object break; } } + + private static boolean deleteIfSemTermMatches(TermsOfUseAndAccess terms, String semterm, JsonValue jsonValue) { + boolean foundTerm=false; + switch (semterm) { + + case "https://dataverse.org/schema/core#termsOfUse": + if(terms.getTermsOfUse().equals(jsonValue.toString())) { + terms.setTermsOfAccess(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#confidentialityDeclaration": + if(terms.getConfidentialityDeclaration().equals(jsonValue.toString())) { + terms.setConfidentialityDeclaration(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#specialPermissions": + if(terms.getSpecialPermissions().equals(jsonValue.toString())) { + terms.setSpecialPermissions(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#restrictions": + if(terms.getRestrictions().equals(jsonValue.toString())) { + terms.setRestrictions(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#citationRequirements": + if(terms.getCitationRequirements().equals(jsonValue.toString())) { + terms.setCitationRequirements(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#depositorRequirements": + if(terms.getDepositorRequirements().equals(jsonValue.toString())) { + terms.setDepositorRequirements(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#conditions": + if(terms.getConditions().equals(jsonValue.toString())) { + terms.setConditions(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#disclaimer": + if(terms.getDisclaimer().equals(jsonValue.toString())) { + terms.setDisclaimer(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#termsOfAccess": + if(terms.getTermsOfAccess().equals(jsonValue.toString())) { + terms.setTermsOfAccess(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#fileRequestAccess": + if(terms.isFileAccessRequest() && (jsonValue.equals(JsonValue.TRUE))) { + terms.setFileAccessRequest(false); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#dataAccessPlace": + if(terms.getDataAccessPlace().equals(jsonValue.toString())) { + terms.setDataAccessPlace(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#originalArchive": + if(terms.getOriginalArchive().equals(jsonValue.toString())) { + terms.setOriginalArchive(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#availabilityStatus": + if(terms.getAvailabilityStatus().equals(jsonValue.toString())) { + terms.setAvailabilityStatus(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#contactForAccess": + if(terms.getContactForAccess().equals(jsonValue.toString())) { + terms.setContactForAccess(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#sizeOfCollection": + if(terms.getSizeOfCollection().equals(jsonValue.toString())) { + terms.setSizeOfCollection(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#studyCompletion": + if(terms.getStudyCompletion().equals(jsonValue.toString())) { + terms.setStudyCompletion(null); + foundTerm=true; + } + break; + default: + logger.warning("deleteIfSemTermMatches called for " + semterm); + break; + } + return foundTerm; + } + } From 780630f3a953c736a7a6278db98e64c04371c5f9 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 8 Dec 2020 12:58:16 -0500 Subject: [PATCH 064/354] 7130-handle missing contact name --- .../iq/dataverse/util/bagit/BagGenerator.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java index 1f8d2f4f617..25ee9978335 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java @@ -764,8 +764,10 @@ private String generateInfoFile() { info.append(CRLF); } else { - info.append(((JsonObject) person).get(contactNameTerm.getLabel()).getAsString()); - info.append(CRLF); + if(contactNameTerm != null) { + info.append(((JsonObject) person).get(contactNameTerm.getLabel()).getAsString()); + info.append(CRLF); + } if ((contactEmailTerm!=null) &&((JsonObject) person).has(contactEmailTerm.getLabel())) { info.append("Contact-Email: "); info.append(((JsonObject) person).get(contactEmailTerm.getLabel()).getAsString()); @@ -782,9 +784,10 @@ private String generateInfoFile() { } else { JsonObject person = contacts.getAsJsonObject(); - - info.append(person.get(contactNameTerm.getLabel()).getAsString()); - info.append(CRLF); + if(contactNameTerm != null) { + info.append(person.get(contactNameTerm.getLabel()).getAsString()); + info.append(CRLF); + } if ((contactEmailTerm!=null) && (person.has(contactEmailTerm.getLabel()))) { info.append("Contact-Email: "); info.append(person.get(contactEmailTerm.getLabel()).getAsString()); From e0ea36ea18d121229ed10620f4030f27017d1775 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 13:16:08 -0500 Subject: [PATCH 065/354] Fix multiple description logic for info file --- .../iq/dataverse/util/bagit/BagGenerator.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java index 25ee9978335..e1f169f408c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java @@ -825,7 +825,7 @@ private String generateInfoFile() { } else { info.append( // FixMe - handle description having subfields better - WordUtils.wrap(getSingleValue(aggregation.getAsJsonObject(descriptionTerm.getLabel()), + WordUtils.wrap(getSingleValue(aggregation.get(descriptionTerm.getLabel()), descriptionTextTerm.getLabel()), 78, CRLF + " ", true)); info.append(CRLF); @@ -871,22 +871,24 @@ private String generateInfoFile() { * - the key to find a value(s) for * @return - a single string */ - String getSingleValue(JsonObject jsonObject, String key) { + String getSingleValue(JsonElement jsonElement, String key) { String val = ""; - if (jsonObject.get(key).isJsonPrimitive()) { + if(jsonElement.isJsonObject()) { + JsonObject jsonObject=jsonElement.getAsJsonObject(); val = jsonObject.get(key).getAsString(); - } else if (jsonObject.get(key).isJsonArray()) { - Iterator iter = jsonObject.getAsJsonArray(key).iterator(); + } else if (jsonElement.isJsonArray()) { + + Iterator iter = jsonElement.getAsJsonArray().iterator(); ArrayList stringArray = new ArrayList(); while (iter.hasNext()) { - stringArray.add(iter.next().getAsString()); + stringArray.add(iter.next().getAsJsonObject().getAsJsonPrimitive(key).getAsString()); } if (stringArray.size() > 1) { val = StringUtils.join((String[]) stringArray.toArray(), ","); } else { val = stringArray.get(0); } - logger.warning("Multiple values found for: " + key + ": " + val); + logger.fine("Multiple values found for: " + key + ": " + val); } return val; } From 6d0c6153f2215815b6d9be201e66fe191ed447e0 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 13:44:51 -0500 Subject: [PATCH 066/354] put is always for :draft version --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 35f17f0b602..9a6b6d67748 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -641,12 +641,8 @@ public Response getVersionMetadata(@PathParam("id") String id) { @PUT @Path("{id}/metadata") @Consumes("application/json-ld") - public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, - @PathParam("versionId") String versionId, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { + public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { - if (!":draft".equals(versionId)) { - return error(Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); - } try { Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); From 51f8f78cbf6c30a142c481587d1bb93ed1f797a1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 15:17:22 -0500 Subject: [PATCH 067/354] don't cast to String[] --- .../java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java index e1f169f408c..445883e846e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java @@ -884,7 +884,7 @@ String getSingleValue(JsonElement jsonElement, String key) { stringArray.add(iter.next().getAsJsonObject().getAsJsonPrimitive(key).getAsString()); } if (stringArray.size() > 1) { - val = StringUtils.join((String[]) stringArray.toArray(), ","); + val = StringUtils.join(stringArray.toArray(), ","); } else { val = stringArray.get(0); } From 353644a0e0bf560839841ba29bb3911ae0c03e70 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 15:17:52 -0500 Subject: [PATCH 068/354] add more logging --- .../edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java b/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java index d3c6fd2df50..ab61fec82fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java @@ -217,6 +217,7 @@ public JsonResponseBuilder log(Logger logger, Level level, Optional e if (ex.isPresent()) { metadata.append("|"); logger.log(level, metadata.toString(), ex); + ex.get().printStackTrace(); } else { logger.log(level, metadata.toString()); } From 2382fef9211623ba35f56c0f59accfb06c4bafa0 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 15:34:10 -0500 Subject: [PATCH 069/354] handle unpublished versions --- src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 657997ef642..69ce31c98ef 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -135,9 +135,9 @@ public JsonObject getOREMap(boolean aggregationOnly) throws Exception { Json.createArrayBuilder().add(JsonLDTerm.ore("Aggregation").getLabel()) .add(JsonLDTerm.schemaOrg("Dataset").getLabel())) .add(JsonLDTerm.schemaOrg("version").getLabel(), version.getFriendlyVersionNumber()) - .add(JsonLDTerm.schemaOrg("datePublished").getLabel(), dataset.getPublicationDateFormattedYYYYMMDD()) .add(JsonLDTerm.schemaOrg("name").getLabel(), version.getTitle()) .add(JsonLDTerm.schemaOrg("dateModified").getLabel(), version.getLastUpdateTime().toString()); + addIfNotNull(aggBuilder, JsonLDTerm.schemaOrg("datePublished"), dataset.getPublicationDateFormattedYYYYMMDD()); TermsOfUseAndAccess terms = version.getTermsOfUseAndAccess(); if (terms.getLicense() == TermsOfUseAndAccess.License.CC0) { From 243769aee31fe5c49ca93a1f168aa8ac083d493d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 17:27:05 -0500 Subject: [PATCH 070/354] add method that can return JsonObjectBuilder which can be used with existing AbstractApiBean.ok() --- .../harvard/iq/dataverse/util/bagit/OREMap.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 69ce31c98ef..5b56fe60ac6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -51,8 +51,12 @@ public void writeOREMap(OutputStream outputStream) throws Exception { public JsonObject getOREMap() throws Exception { return getOREMap(false); } - + public JsonObject getOREMap(boolean aggregationOnly) throws Exception { + return getOREMapBuilder(aggregationOnly).build(); + } + + public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) throws Exception { // Add namespaces we'll definitely use to Context // Additional namespaces are added as needed below @@ -250,10 +254,10 @@ public JsonObject getOREMap(boolean aggregationOnly) throws Exception { contextBuilder.add(e.getKey(), e.getValue()); } if (aggregationOnly) { - return aggBuilder.add("@context", contextBuilder.build()).build(); + return aggBuilder.add("@context", contextBuilder.build()); } else { // Now create the overall map object with it's metadata - JsonObject oremap = Json.createObjectBuilder() + JsonObjectBuilder oremapBuilder = Json.createObjectBuilder() .add(JsonLDTerm.dcTerms("modified").getLabel(), LocalDate.now().toString()) .add(JsonLDTerm.dcTerms("creator").getLabel(), BundleUtil.getStringFromBundle("institution.name")) .add("@type", JsonLDTerm.ore("ResourceMap").getLabel()) @@ -267,8 +271,8 @@ public JsonObject getOREMap(boolean aggregationOnly) throws Exception { aggBuilder.add(JsonLDTerm.ore("aggregates").getLabel(), aggResArrayBuilder.build()) .add(JsonLDTerm.schemaOrg("hasPart").getLabel(), fileArray.build()).build()) // and finally add the context - .add("@context", contextBuilder.build()).build(); - return oremap; + .add("@context", contextBuilder.build()); + return oremapBuilder; } } From 9bfa7c3d2f6c492b361fccd13f0bbc109d325353 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 17:27:24 -0500 Subject: [PATCH 071/354] log details on failure --- .../engine/command/impl/LocalSubmitToArchiveCommand.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java index 0a1de25bed0..d87c3011c15 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java @@ -84,6 +84,7 @@ public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken t } } catch (Exception e) { logger.warning(e.getLocalizedMessage() + "here"); + e.printStackTrace(); } return WorkflowStepResult.OK; } From 60f8a99676e1b9e2c35e7f624fc4f3da58d6675d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 17:27:55 -0500 Subject: [PATCH 072/354] multiple updates/fixes, added logging --- .../harvard/iq/dataverse/api/Datasets.java | 28 ++++++++++--------- .../iq/dataverse/util/json/JSONLDUtil.java | 13 +++++++-- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 9a6b6d67748..662a04d6d2c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -612,21 +612,22 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, } @GET - @Path("{id}/versions/{versionid}/metadata") + @Path("{id}/versions/{versionId}/metadata") @Produces("application/json-ld") - public Response getVersionMetadata(@PathParam("id") String id, @PathParam("versionId") String versionId) { + public Response getVersionJsonLDMetadata(@PathParam("id") String id, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) { try { - Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); - DatasetVersion dsv = ds.getEditVersion(); + DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(id), uriInfo, headers); OREMap ore = new OREMap(dsv, settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)); - return ok(JSONLDUtil.prettyPrint(ore.getOREMap(true))); + return ok(ore.getOREMapBuilder(true)); } catch (WrappedResponse ex) { + ex.printStackTrace(); return ex.getResponse(); } catch (Exception jpe) { logger.log(Level.SEVERE, "Error getting jsonld metadata for dsv: ", jpe.getLocalizedMessage()); + jpe.printStackTrace(); return error(Response.Status.INTERNAL_SERVER_ERROR, jpe.getLocalizedMessage()); } } @@ -634,8 +635,8 @@ public Response getVersionMetadata(@PathParam("id") String id, @PathParam("versi @GET @Path("{id}/metadata") @Produces("application/json-ld") - public Response getVersionMetadata(@PathParam("id") String id) { - return getVersionMetadata(id, ":draft"); + public Response getVersionJsonLDMetadata(@PathParam("id") String id, @Context UriInfo uriInfo, @Context HttpHeaders headers) { + return getVersionJsonLDMetadata(id, ":draft", uriInfo, headers); } @PUT @@ -668,19 +669,18 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String } } - @DELETE - @Path("{id}/metadata") + @PUT + @Path("{id}/metadata/delete") @Consumes("application/json-ld") - public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id, - @PathParam("versionId") String versionId, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { - + public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id) { + logger.info("In delteMetadata"); try { Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); DatasetVersion dsv = ds.getEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); dsv = JSONLDUtil.deleteDatasetVersionMDFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc); - + logger.info("Updating ver"); DatasetVersion managedVersion; if (updateDraft) { Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); @@ -692,9 +692,11 @@ public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id, return ok(Json.createObjectBuilder().add(info, managedVersion.getVersionDate())); } catch (WrappedResponse ex) { + ex.printStackTrace(); return ex.getResponse(); } catch (JsonParsingException jpe) { logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", jsonLDBody); + jpe.printStackTrace(); return error(Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage()); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 822a5f4008d..a8b12a85828 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -158,6 +158,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv // unless it's multiple if (!append && !dsft.isAllowMultiples()) { dsfl.remove(dsf); + dsf=null; } } if (dsf == null) { @@ -306,7 +307,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv */ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv, String jsonLDBody, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { - +logger.info("deleteDatasetVersionMD"); JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); //All terms are now URIs //Setup dsftMap - URI to datasetFieldType map @@ -340,12 +341,12 @@ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv // Todo - normalize object vs. array JsonArray valArray = getValues(jsonld.get(key), dsft.isAllowMultiples(), dsft.getName()); - +logger.info("Deleting: " + key + " : " + valArray.toString()); DatasetField dsf2 = getReplacementField(dsf, valArray); if(dsf2 == null) { //Exact match - remove the field dsfl.remove(dsf); - } else if(!dsf2.equals(dsf)) { + } else { //Partial match - some values of a multivalue field match, so keep the remaining values dsfl.remove(dsf); dsfl.add(dsf2); @@ -388,6 +389,12 @@ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv return dsv; } + /** + * + * @param dsf + * @param valArray + * @return null if exact match, otherwise return a field without the value to be deleted + */ private static DatasetField getReplacementField(DatasetField dsf, JsonArray valArray) { // TODO Auto-generated method stub return null; From 83c8794fb344bfa139061e0c8bb5c063f5f2a54e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 1 Dec 2020 12:43:56 -0500 Subject: [PATCH 073/354] remove OREMap parameter --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index f83826e7607..6f7b5ef8ccc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -98,7 +98,7 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, ds.setModificationTime(Timestamp.valueOf(dateTime)); } try { - logger.fine("Output dsv: " + new OREMap(dsv, false, OREMap.TRANSFER).getOREMap().toString()); + logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); From 6d537ce15b5989abb1bacd85a48ae17b447cd1cd Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 1 Dec 2020 15:34:07 -0500 Subject: [PATCH 074/354] fix error handling FWIW: We have an error handler for the edu.harvard.iq.dataverse.util.json.JsonParseException class but not for javax.json.stream.JsonParsingException which was getting caught by the Throwable handler and returned as a 500 error with json message {} --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 20874f70653..a376e663e99 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -78,6 +78,7 @@ import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.S3PackageImporter; +import edu.harvard.iq.dataverse.api.AbstractApiBean.WrappedResponse; import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; import edu.harvard.iq.dataverse.dataaccess.DataAccess; @@ -135,6 +136,7 @@ import javax.json.JsonObject; import javax.json.JsonObjectBuilder; import javax.json.JsonReader; +import javax.json.stream.JsonParsingException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.BadRequestException; @@ -639,7 +641,9 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String } catch (WrappedResponse ex) { return ex.getResponse(); - + } catch (JsonParsingException jpe) { + logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", jsonLDBody); + return error(Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage()); } } From 6ee749945b32e9f017ae910de1abbdba30e6d958 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 1 Dec 2020 16:29:36 -0500 Subject: [PATCH 075/354] append to current terms --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 6f7b5ef8ccc..b09ad08f0b0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -139,7 +139,7 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, } fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); } - TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); + TermsOfUseAndAccess terms = dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess(); for (String key : jsonld.keySet()) { if (!key.equals("@context")) { From 8827a0c05b8083dce6230eb144592e5af90bb722 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 2 Dec 2020 10:34:22 -0500 Subject: [PATCH 076/354] handle append on terms - fix cut/paste errors --- .../iq/dataverse/util/json/JSONLDUtil.java | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index b09ad08f0b0..a728263dea2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -28,6 +28,9 @@ import javax.json.JsonWriterFactory; import javax.json.stream.JsonGenerator; import javax.ws.rs.BadRequestException; + +import org.apache.commons.lang.StringUtils; + import com.apicatalog.jsonld.JsonLd; import com.apicatalog.jsonld.api.JsonLdError; import com.apicatalog.jsonld.document.JsonDocument; @@ -194,56 +197,56 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index + 1))); } - } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())) { + } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())&&(append || terms.getLicense().equals(TermsOfUseAndAccess.License.NONE))) { if (jsonld.getString(JsonLDTerm.schemaOrg("license").getUrl()) .equals("https://creativecommons.org/publicdomain/zero/1.0/")) { terms.setLicense(TermsOfUseAndAccess.defaultLicense); } else { terms.setLicense(TermsOfUseAndAccess.License.NONE); } - } else if (key.equals(JsonLDTerm.termsOfUse.getUrl())) { + } else if (key.equals(JsonLDTerm.termsOfUse.getUrl())&&(append || StringUtils.isBlank(terms.getTermsOfUse()))) { terms.setTermsOfUse(jsonld.getString(JsonLDTerm.termsOfUse.getUrl())); - } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getUrl())) { + } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getUrl())&&(append || StringUtils.isBlank(terms.getConfidentialityDeclaration()))) { terms.setConfidentialityDeclaration( jsonld.getString(JsonLDTerm.confidentialityDeclaration.getUrl())); - } else if (key.equals(JsonLDTerm.specialPermissions.getUrl())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.specialPermissions.getUrl())); - } else if (key.equals(JsonLDTerm.restrictions.getUrl())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.restrictions.getUrl())); - } else if (key.equals(JsonLDTerm.citationRequirements.getUrl())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.citationRequirements.getUrl())); - } else if (key.equals(JsonLDTerm.depositorRequirements.getUrl())) { - terms.setConfidentialityDeclaration( + } else if (key.equals(JsonLDTerm.specialPermissions.getUrl())&&(append || StringUtils.isBlank(terms.getSpecialPermissions()))) { + terms.setSpecialPermissions(jsonld.getString(JsonLDTerm.specialPermissions.getUrl())); + } else if (key.equals(JsonLDTerm.restrictions.getUrl())&&(append || StringUtils.isBlank(terms.getRestrictions()))) { + terms.setRestrictions(jsonld.getString(JsonLDTerm.restrictions.getUrl())); + } else if (key.equals(JsonLDTerm.citationRequirements.getUrl())&&(append || StringUtils.isBlank(terms.getCitationRequirements()))) { + terms.setCitationRequirements(jsonld.getString(JsonLDTerm.citationRequirements.getUrl())); + } else if (key.equals(JsonLDTerm.depositorRequirements.getUrl())&&(append || StringUtils.isBlank(terms.getDepositorRequirements()))) { + terms.setDepositorRequirements( jsonld.getString(JsonLDTerm.depositorRequirements.getUrl())); - } else if (key.equals(JsonLDTerm.conditions.getUrl())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.conditions.getUrl())); - } else if (key.equals(JsonLDTerm.disclaimer.getUrl())) { - terms.setConfidentialityDeclaration(jsonld.getString(JsonLDTerm.disclaimer.getUrl())); + } else if (key.equals(JsonLDTerm.conditions.getUrl())&&(append || StringUtils.isBlank(terms.getConditions()))) { + terms.setConditions(jsonld.getString(JsonLDTerm.conditions.getUrl())); + } else if (key.equals(JsonLDTerm.disclaimer.getUrl())&&(append || StringUtils.isBlank(terms.getDisclaimer()))) { + terms.setDisclaimer(jsonld.getString(JsonLDTerm.disclaimer.getUrl())); } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); - if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getUrl())&&(append || StringUtils.isBlank(terms.getTermsOfAccess()))) { terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getUrl())&&(append || !terms.isFileAccessRequest())) { terms.setFileAccessRequest(fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getUrl())&&(append || StringUtils.isBlank(terms.getDataAccessPlace()))) { terms.setDataAccessPlace(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getUrl())&&(append || StringUtils.isBlank(terms.getOriginalArchive()))) { terms.setOriginalArchive(fAccessObject.getString(JsonLDTerm.originalArchive.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getUrl())&&(append || StringUtils.isBlank(terms.getAvailabilityStatus()))) { terms.setAvailabilityStatus( fAccessObject.getString(JsonLDTerm.availabilityStatus.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getUrl())&&(append || StringUtils.isBlank(terms.getContactForAccess()))) { terms.setContactForAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getUrl())&&(append || StringUtils.isBlank(terms.getSizeOfCollection()))) { terms.setSizeOfCollection(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getUrl())) { + if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getUrl())&&(append || StringUtils.isBlank(terms.getStudyCompletion()))) { terms.setStudyCompletion(fAccessObject.getString(JsonLDTerm.studyCompletion.getUrl())); } } else { From 92328cb51cd012639d188006e879c6ee6821ddb1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 2 Dec 2020 11:58:50 -0500 Subject: [PATCH 077/354] fix logic --- .../iq/dataverse/util/json/JSONLDUtil.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index a728263dea2..75317f09806 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -197,56 +197,56 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index + 1))); } - } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())&&(append || terms.getLicense().equals(TermsOfUseAndAccess.License.NONE))) { + } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())&&(!append || terms.getLicense().equals(TermsOfUseAndAccess.License.NONE))) { if (jsonld.getString(JsonLDTerm.schemaOrg("license").getUrl()) .equals("https://creativecommons.org/publicdomain/zero/1.0/")) { terms.setLicense(TermsOfUseAndAccess.defaultLicense); } else { terms.setLicense(TermsOfUseAndAccess.License.NONE); } - } else if (key.equals(JsonLDTerm.termsOfUse.getUrl())&&(append || StringUtils.isBlank(terms.getTermsOfUse()))) { + } else if (key.equals(JsonLDTerm.termsOfUse.getUrl())&&(!append || StringUtils.isBlank(terms.getTermsOfUse()))) { terms.setTermsOfUse(jsonld.getString(JsonLDTerm.termsOfUse.getUrl())); - } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getUrl())&&(append || StringUtils.isBlank(terms.getConfidentialityDeclaration()))) { + } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getUrl())&&(!append || StringUtils.isBlank(terms.getConfidentialityDeclaration()))) { terms.setConfidentialityDeclaration( jsonld.getString(JsonLDTerm.confidentialityDeclaration.getUrl())); - } else if (key.equals(JsonLDTerm.specialPermissions.getUrl())&&(append || StringUtils.isBlank(terms.getSpecialPermissions()))) { + } else if (key.equals(JsonLDTerm.specialPermissions.getUrl())&&(!append || StringUtils.isBlank(terms.getSpecialPermissions()))) { terms.setSpecialPermissions(jsonld.getString(JsonLDTerm.specialPermissions.getUrl())); - } else if (key.equals(JsonLDTerm.restrictions.getUrl())&&(append || StringUtils.isBlank(terms.getRestrictions()))) { + } else if (key.equals(JsonLDTerm.restrictions.getUrl())&&(!append || StringUtils.isBlank(terms.getRestrictions()))) { terms.setRestrictions(jsonld.getString(JsonLDTerm.restrictions.getUrl())); - } else if (key.equals(JsonLDTerm.citationRequirements.getUrl())&&(append || StringUtils.isBlank(terms.getCitationRequirements()))) { + } else if (key.equals(JsonLDTerm.citationRequirements.getUrl())&&(!append || StringUtils.isBlank(terms.getCitationRequirements()))) { terms.setCitationRequirements(jsonld.getString(JsonLDTerm.citationRequirements.getUrl())); - } else if (key.equals(JsonLDTerm.depositorRequirements.getUrl())&&(append || StringUtils.isBlank(terms.getDepositorRequirements()))) { + } else if (key.equals(JsonLDTerm.depositorRequirements.getUrl())&&(!append || StringUtils.isBlank(terms.getDepositorRequirements()))) { terms.setDepositorRequirements( jsonld.getString(JsonLDTerm.depositorRequirements.getUrl())); - } else if (key.equals(JsonLDTerm.conditions.getUrl())&&(append || StringUtils.isBlank(terms.getConditions()))) { + } else if (key.equals(JsonLDTerm.conditions.getUrl())&&(!append || StringUtils.isBlank(terms.getConditions()))) { terms.setConditions(jsonld.getString(JsonLDTerm.conditions.getUrl())); - } else if (key.equals(JsonLDTerm.disclaimer.getUrl())&&(append || StringUtils.isBlank(terms.getDisclaimer()))) { + } else if (key.equals(JsonLDTerm.disclaimer.getUrl())&&(!append || StringUtils.isBlank(terms.getDisclaimer()))) { terms.setDisclaimer(jsonld.getString(JsonLDTerm.disclaimer.getUrl())); } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); - if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getUrl())&&(append || StringUtils.isBlank(terms.getTermsOfAccess()))) { + if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getUrl())&&(!append || StringUtils.isBlank(terms.getTermsOfAccess()))) { terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getUrl())&&(append || !terms.isFileAccessRequest())) { + if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getUrl())&&(!append || !terms.isFileAccessRequest())) { terms.setFileAccessRequest(fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getUrl())&&(append || StringUtils.isBlank(terms.getDataAccessPlace()))) { + if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getUrl())&&(!append || StringUtils.isBlank(terms.getDataAccessPlace()))) { terms.setDataAccessPlace(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getUrl())&&(append || StringUtils.isBlank(terms.getOriginalArchive()))) { + if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getUrl())&&(!append || StringUtils.isBlank(terms.getOriginalArchive()))) { terms.setOriginalArchive(fAccessObject.getString(JsonLDTerm.originalArchive.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getUrl())&&(append || StringUtils.isBlank(terms.getAvailabilityStatus()))) { + if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getUrl())&&(!append || StringUtils.isBlank(terms.getAvailabilityStatus()))) { terms.setAvailabilityStatus( fAccessObject.getString(JsonLDTerm.availabilityStatus.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getUrl())&&(append || StringUtils.isBlank(terms.getContactForAccess()))) { + if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getUrl())&&(!append || StringUtils.isBlank(terms.getContactForAccess()))) { terms.setContactForAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getUrl())&&(append || StringUtils.isBlank(terms.getSizeOfCollection()))) { + if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getUrl())&&(!append || StringUtils.isBlank(terms.getSizeOfCollection()))) { terms.setSizeOfCollection(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getUrl())); } - if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getUrl())&&(append || StringUtils.isBlank(terms.getStudyCompletion()))) { + if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getUrl())&&(!append || StringUtils.isBlank(terms.getStudyCompletion()))) { terms.setStudyCompletion(fAccessObject.getString(JsonLDTerm.studyCompletion.getUrl())); } } else { From 2a83a701cd1721647316d9cbb149f3c42573c628 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 2 Dec 2020 11:59:09 -0500 Subject: [PATCH 078/354] specify default --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index a376e663e99..e2d5a22cc9b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -142,6 +142,7 @@ import javax.ws.rs.BadRequestException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; @@ -617,7 +618,7 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, @Path("{id}/versions/{versionId}/metadata") @Consumes("application/json-ld") public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, - @PathParam("versionId") String versionId, @QueryParam("replace") boolean replaceTerms) { + @PathParam("versionId") String versionId, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { if (!":draft".equals(versionId)) { return error(Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); From d75fb78fb27eae84093fc739bce8994747e16e01 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 2 Dec 2020 12:21:56 -0500 Subject: [PATCH 079/354] make replace still append for multiple val fields --- .../edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 75317f09806..ec86abd07ac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -152,8 +152,8 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, DatasetField dsf = null; if (fieldByTypeMap.containsKey(dsft)) { dsf = fieldByTypeMap.get(dsft); - // If there's an existing field, we use it with append and remove it for !append - if (!append) { + // If there's an existing field, we use it with append and remove it for !append unless it's multiple + if (!append && !dsft.isAllowMultiples()) { dsfl.remove(dsf); } } @@ -256,8 +256,8 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, DatasetField dsf = null; if (fieldByTypeMap.containsKey(dsft)) { dsf = fieldByTypeMap.get(dsft); - // If there's an existing field, we use it with append and remove it for !append - if (!append) { + // If there's an existing field, we use it with append and remove it for !append (except if multiple, which is not the default) + if (!append && !dsft.isAllowMultiples()) { dsfl.remove(dsf); } } From 782d5ecfcedc0460b0ed619288651795775ddaf6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 2 Dec 2020 14:38:17 -0500 Subject: [PATCH 080/354] add migrating switch --- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../iq/dataverse/util/json/JSONLDUtil.java | 1237 +++++++++-------- 2 files changed, 677 insertions(+), 562 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index e2d5a22cc9b..527476ea59a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -628,7 +628,7 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String DataverseRequest req = createDataverseRequest(findUserOrDie()); DatasetVersion dsv = ds.getEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); - dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, !replaceTerms); + dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, !replaceTerms, false); DatasetVersion managedVersion; if (updateDraft) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index ec86abd07ac..414b3ec8a01 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -8,6 +8,7 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.LinkedList; @@ -47,6 +48,7 @@ import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess.License; import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.util.bagit.OREMap; @@ -54,568 +56,681 @@ public class JSONLDUtil { - private static final Logger logger = Logger.getLogger(JSONLDUtil.class.getCanonicalName()); - - /* - * private static Map populateContext(JsonValue json) { - * Map context = new TreeMap(); if (json - * instanceof JsonArray) { logger.warning("Array @context not yet supported"); } - * else { for (String key : ((JsonObject) json).keySet()) { - * context.putIfAbsent(key, ((JsonObject) json).getString(key)); } } return - * context; } - */ - - public static JsonObject getContext(Map contextMap) { - JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); - for (Entry e : contextMap.entrySet()) { - contextBuilder.add(e.getKey(), e.getValue()); - } - return contextBuilder.build(); - } - - public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, - MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append) { - - DatasetVersion dsv = new DatasetVersion(); - - JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); - Optional maybePid = GlobalId.parse(jsonld.getString("@id")); - if (maybePid.isPresent()) { - ds.setGlobalId(maybePid.get()); - } else { - // unparsable PID passed. Terminate. - throw new BadRequestException("Cannot parse the @id '" + jsonld.getString("@id") - + "'. Make sure it is in valid form - see Dataverse Native API documentation."); - } - - dsv = updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append); - dsv.setDataset(ds); - - List versions = new ArrayList<>(1); - versions.add(dsv); - - ds.setVersions(versions); - if (jsonld.containsKey(JsonLDTerm.schemaOrg("dateModified").getUrl())) { - String dateString = jsonld.getString(JsonLDTerm.schemaOrg("dateModified").getUrl()); - LocalDateTime dateTime = getDateTimeFrom(dateString); - ds.setModificationTime(Timestamp.valueOf(dateTime)); - } - try { - logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return ds; - } - - public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, String jsonLDBody, - MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append) { - JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); - return updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append); - } - - /** - * - * @param dsv - * @param jsonld - * @param metadataBlockSvc - * @param datasetFieldSvc - * @param append - if append, will add new top level field values for - * multi-valued fields, if true and field type isn't - * multiple, will fail. if false will replace all - * value(s) for fields found in the json-ld. - * @return - */ - public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, JsonObject jsonld, - MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append) { - - populateFieldTypeMap(metadataBlockSvc); - - // get existing ones? - List dsfl = dsv.getDatasetFields(); - Map fieldByTypeMap = new HashMap(); - for (DatasetField dsf : dsfl) { - if (fieldByTypeMap.containsKey(dsf.getDatasetFieldType())) { - // May have multiple values per field, but not multiple fields of one type? - logger.warning("Multiple fields of type " + dsf.getDatasetFieldType().getName()); - } - fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); - } - TermsOfUseAndAccess terms = dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess(); - - for (String key : jsonld.keySet()) { - if (!key.equals("@context")) { - if (dsftMap.containsKey(key)) { - - DatasetFieldType dsft = dsftMap.get(key); - DatasetField dsf = null; - if (fieldByTypeMap.containsKey(dsft)) { - dsf = fieldByTypeMap.get(dsft); - // If there's an existing field, we use it with append and remove it for !append unless it's multiple - if (!append && !dsft.isAllowMultiples()) { - dsfl.remove(dsf); - } - } - if (dsf == null) { - dsf = new DatasetField(); - dsfl.add(dsf); - dsf.setDatasetFieldType(dsft); - } - - // Todo - normalize object vs. array - JsonArray valArray = getValues(jsonld.get(key), dsft.isAllowMultiples(), dsft.getName()); - - addField(dsf, valArray, dsft, datasetFieldSvc, append); - - // assemble new terms, add to existing - // multivalue? - // compound? - // merge with existing dv metadata - // dsfl.add(dsf); - } else { - // Internal/non-metadatablock terms - // Add metadata related to the Dataset/DatasetVersion - - // ("@id", id) - check is equal to existing globalID? - // Add to 'md on original' ? - // (JsonLDTerm.schemaOrg("version").getLabel(), - // version.getFriendlyVersionNumber()) - // Citation metadata? - // (JsonLDTerm.schemaOrg("datePublished").getLabel(), - // dataset.getPublicationDateFormattedYYYYMMDD()) - // (JsonLDTerm.schemaOrg("name").getLabel()) - // (JsonLDTerm.schemaOrg("dateModified").getLabel()) - - // Todo - handle non-CC0 licenses, without terms as an alternate field. - if (key.equals(JsonLDTerm.schemaOrg("datePublished").getUrl())) { - dsv.setVersionState(VersionState.RELEASED); - } else if (key.equals(JsonLDTerm.schemaOrg("version").getUrl())) { - String friendlyVersion = jsonld.getString(JsonLDTerm.schemaOrg("version").getUrl()); - int index = friendlyVersion.indexOf("."); - if (index > 0) { - dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); - dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index + 1))); - } - } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())&&(!append || terms.getLicense().equals(TermsOfUseAndAccess.License.NONE))) { - if (jsonld.getString(JsonLDTerm.schemaOrg("license").getUrl()) - .equals("https://creativecommons.org/publicdomain/zero/1.0/")) { - terms.setLicense(TermsOfUseAndAccess.defaultLicense); - } else { - terms.setLicense(TermsOfUseAndAccess.License.NONE); - } - } else if (key.equals(JsonLDTerm.termsOfUse.getUrl())&&(!append || StringUtils.isBlank(terms.getTermsOfUse()))) { - terms.setTermsOfUse(jsonld.getString(JsonLDTerm.termsOfUse.getUrl())); - } else if (key.equals(JsonLDTerm.confidentialityDeclaration.getUrl())&&(!append || StringUtils.isBlank(terms.getConfidentialityDeclaration()))) { - terms.setConfidentialityDeclaration( - jsonld.getString(JsonLDTerm.confidentialityDeclaration.getUrl())); - } else if (key.equals(JsonLDTerm.specialPermissions.getUrl())&&(!append || StringUtils.isBlank(terms.getSpecialPermissions()))) { - terms.setSpecialPermissions(jsonld.getString(JsonLDTerm.specialPermissions.getUrl())); - } else if (key.equals(JsonLDTerm.restrictions.getUrl())&&(!append || StringUtils.isBlank(terms.getRestrictions()))) { - terms.setRestrictions(jsonld.getString(JsonLDTerm.restrictions.getUrl())); - } else if (key.equals(JsonLDTerm.citationRequirements.getUrl())&&(!append || StringUtils.isBlank(terms.getCitationRequirements()))) { - terms.setCitationRequirements(jsonld.getString(JsonLDTerm.citationRequirements.getUrl())); - } else if (key.equals(JsonLDTerm.depositorRequirements.getUrl())&&(!append || StringUtils.isBlank(terms.getDepositorRequirements()))) { - terms.setDepositorRequirements( - jsonld.getString(JsonLDTerm.depositorRequirements.getUrl())); - } else if (key.equals(JsonLDTerm.conditions.getUrl())&&(!append || StringUtils.isBlank(terms.getConditions()))) { - terms.setConditions(jsonld.getString(JsonLDTerm.conditions.getUrl())); - } else if (key.equals(JsonLDTerm.disclaimer.getUrl())&&(!append || StringUtils.isBlank(terms.getDisclaimer()))) { - terms.setDisclaimer(jsonld.getString(JsonLDTerm.disclaimer.getUrl())); - } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { - JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); - if (fAccessObject.containsKey(JsonLDTerm.termsOfAccess.getUrl())&&(!append || StringUtils.isBlank(terms.getTermsOfAccess()))) { - terms.setTermsOfAccess(fAccessObject.getString(JsonLDTerm.termsOfAccess.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.fileRequestAccess.getUrl())&&(!append || !terms.isFileAccessRequest())) { - terms.setFileAccessRequest(fAccessObject.getBoolean(JsonLDTerm.fileRequestAccess.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.dataAccessPlace.getUrl())&&(!append || StringUtils.isBlank(terms.getDataAccessPlace()))) { - terms.setDataAccessPlace(fAccessObject.getString(JsonLDTerm.dataAccessPlace.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.originalArchive.getUrl())&&(!append || StringUtils.isBlank(terms.getOriginalArchive()))) { - terms.setOriginalArchive(fAccessObject.getString(JsonLDTerm.originalArchive.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.availabilityStatus.getUrl())&&(!append || StringUtils.isBlank(terms.getAvailabilityStatus()))) { - terms.setAvailabilityStatus( - fAccessObject.getString(JsonLDTerm.availabilityStatus.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.contactForAccess.getUrl())&&(!append || StringUtils.isBlank(terms.getContactForAccess()))) { - terms.setContactForAccess(fAccessObject.getString(JsonLDTerm.contactForAccess.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.sizeOfCollection.getUrl())&&(!append || StringUtils.isBlank(terms.getSizeOfCollection()))) { - terms.setSizeOfCollection(fAccessObject.getString(JsonLDTerm.sizeOfCollection.getUrl())); - } - if (fAccessObject.containsKey(JsonLDTerm.studyCompletion.getUrl())&&(!append || StringUtils.isBlank(terms.getStudyCompletion()))) { - terms.setStudyCompletion(fAccessObject.getString(JsonLDTerm.studyCompletion.getUrl())); - } - } else { - if (dsftMap.containsKey(JsonLDTerm.metadataOnOrig.getUrl())) { - DatasetFieldType dsft = dsftMap.get(JsonLDTerm.metadataOnOrig.getUrl()); - - DatasetField dsf = null; - if (fieldByTypeMap.containsKey(dsft)) { - dsf = fieldByTypeMap.get(dsft); - // If there's an existing field, we use it with append and remove it for !append (except if multiple, which is not the default) - if (!append && !dsft.isAllowMultiples()) { - dsfl.remove(dsf); - } - } - if (dsf == null) { - dsf = new DatasetField(); - dsfl.add(dsf); - dsf.setDatasetFieldType(dsft); - } - - List vals = dsf.getDatasetFieldValues(); - - JsonObject currentValue = null; - DatasetFieldValue datasetFieldValue = null; - if (vals.isEmpty()) { - datasetFieldValue = new DatasetFieldValue(); - vals.add(datasetFieldValue); - datasetFieldValue.setDatasetField(dsf); - dsf.setDatasetFieldValues(vals); - - currentValue = Json.createObjectBuilder().build(); - } else { - datasetFieldValue = vals.get(0); - JsonObject currentVal = decontextualizeJsonLD(datasetFieldValue.getValueForEdit()); - - } - currentValue.put(key, jsonld.get(key)); - JsonObject newValue = recontextualizeJsonLD(currentValue, metadataBlockSvc); - datasetFieldValue.setValue(prettyPrint(newValue)); - } - } - dsv.setTermsOfUseAndAccess(terms); - // move to new dataverse? - // aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), - // dataset.getDataverseContext().getDisplayName()); - - } - - } - } - - dsv.setDatasetFields(dsfl); - - return dsv; - } - - private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldType dsft, - DatasetFieldServiceBean datasetFieldSvc, boolean append) { - - if (append && !dsft.isAllowMultiples()) { - if ((dsft.isCompound() && !dsf.getDatasetFieldCompoundValues().isEmpty()) - || (dsft.isAllowControlledVocabulary() && !dsf.getControlledVocabularyValues().isEmpty()) - || !dsf.getDatasetFieldValues().isEmpty()) { - throw new BadRequestException( - "Can't append to a single-value field that already has a value: " + dsft.getName()); - } - } - logger.fine("Name: " + dsft.getName()); - logger.fine("val: " + valArray.toString()); - logger.fine("Compound: " + dsft.isCompound()); - logger.fine("CV: " + dsft.isAllowControlledVocabulary()); - - if (dsft.isCompound()) { - /* - * List vals = parseCompoundValue(type, - * jsonld.get(key),testType); for (DatasetFieldCompoundValue dsfcv : vals) { - * dsfcv.setParentDatasetField(ret); } dsf.setDatasetFieldCompoundValues(vals); - */ - List cvList = dsf.getDatasetFieldCompoundValues(); - if (!cvList.isEmpty()) { - if (!append) { - cvList.clear(); - } else if (!dsft.isAllowMultiples() && cvList.size() == 1) { - // Trying to append but only a single value is allowed (and there already is - // one) - // (and we don't currently support appending new fields within a compound value) - throw new BadRequestException( - "Append with compound field with single value not yet supported: " + dsft.getDisplayName()); - } - } - - List vals = new LinkedList<>(); - for (JsonValue val : valArray) { - if (!(val instanceof JsonObject)) { - throw new BadRequestException( - "Compound field values must be JSON objects, field: " + dsft.getName()); - } - DatasetFieldCompoundValue cv = null; - - cv = new DatasetFieldCompoundValue(); - cv.setDisplayOrder(cvList.size()); - cvList.add(cv); - cv.setParentDatasetField(dsf); - - JsonObject obj = (JsonObject) val; - for (String childKey : obj.keySet()) { - if (dsftMap.containsKey(childKey)) { - DatasetFieldType childft = dsftMap.get(childKey); - if (!dsft.getChildDatasetFieldTypes().contains(childft)) { - throw new BadRequestException( - "Compound field " + dsft.getName() + "can't include term " + childKey); - } - DatasetField childDsf = new DatasetField(); - cv.getChildDatasetFields().add(childDsf); - childDsf.setDatasetFieldType(childft); - childDsf.setParentDatasetFieldCompoundValue(cv); - - JsonArray childValArray = getValues(obj.get(childKey), childft.isAllowMultiples(), - childft.getName()); - addField(childDsf, childValArray, childft, datasetFieldSvc, append); - } - } - } - - } else if (dsft.isControlledVocabulary()) { - - List vals = dsf.getControlledVocabularyValues(); - for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { - String strValue = strVal.getString(); - ControlledVocabularyValue cvv = datasetFieldSvc - .findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, strValue, true); - if (cvv == null) { - throw new BadRequestException( - "Unknown value for Controlled Vocab Field: " + dsft.getName() + " : " + strValue); - } - // Only add value to the list if it is not a duplicate - if (strValue.equals("Other")) { - System.out.println("vals = " + vals + ", contains: " + vals.contains(cvv)); - } - if (!vals.contains(cvv)) { - if (vals.size() > 0) { - cvv.setDisplayOrder(vals.size()); - } - vals.add(cvv); - cvv.setDatasetFieldType(dsft); - } - } - dsf.setControlledVocabularyValues(vals); - - } else { - List vals = dsf.getDatasetFieldValues(); - - for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { - String strValue = strVal.getString(); - DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); - - datasetFieldValue.setDisplayOrder(vals.size()); - datasetFieldValue.setValue(strValue.trim()); - vals.add(datasetFieldValue); - datasetFieldValue.setDatasetField(dsf); - - } - dsf.setDatasetFieldValues(vals); - } - } - - private static JsonArray getValues(JsonValue val, boolean allowMultiples, String name) { - JsonArray valArray = null; - if (val instanceof JsonArray) { - if ((((JsonArray) val).size() > 1) && !allowMultiples) { - throw new BadRequestException("Array for single value notsupported: " + name); - } else { - valArray = (JsonArray) val; - } - } else { - valArray = Json.createArrayBuilder().add(val).build(); - } - return valArray; - } - - static Map localContext = new TreeMap(); - static Map dsftMap = new TreeMap(); - - private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockSvc) { - if (dsftMap.isEmpty()) { - - List mdbList = metadataBlockSvc.listMetadataBlocks(); - - for (MetadataBlock mdb : mdbList) { - boolean blockHasUri = mdb.getNamespaceUri() != null; - for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { - if (dsft.getUri() != null) { - dsftMap.put(dsft.getUri(), dsft); - } - if (blockHasUri) { - if (dsft.getParentDatasetFieldType() != null) { - // ToDo - why not getName for child type? Would have to fix in ORE generation - // code and handle legacy bags - dsftMap.put(mdb.getNamespaceUri() + dsft.getParentDatasetFieldType().getName() + "#" - + dsft.getTitle(), dsft); - } else { - dsftMap.put(mdb.getNamespaceUri() + dsft.getTitle(), dsft); - } - } - } - } - logger.fine("DSFT Map: " + String.join(", ", dsftMap.keySet())); - } - } - - private static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { - if (localContext.isEmpty()) { - - // Add namespaces corresponding to core terms - localContext.put(JsonLDNamespace.dcterms.getPrefix(), JsonLDNamespace.dcterms.getUrl()); - localContext.put(JsonLDNamespace.dvcore.getPrefix(), JsonLDNamespace.dvcore.getUrl()); - localContext.put(JsonLDNamespace.schema.getPrefix(), JsonLDNamespace.schema.getUrl()); - - List mdbList = metadataBlockSvc.listMetadataBlocks(); - - for (MetadataBlock mdb : mdbList) { - boolean blockHasUri = mdb.getNamespaceUri() != null; - if (blockHasUri) { - localContext.putIfAbsent(mdb.getName(), mdb.getNamespaceUri()); - - } - for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { - if (dsft.getUri() != null) { - localContext.putIfAbsent(dsft.getName(), dsft.getUri()); - } - } - } - logger.fine("LocalContext keys: " + String.join(", ", localContext.keySet())); - } - } - - public static JsonObject decontextualizeJsonLD(String jsonLDString) { - logger.fine(jsonLDString); - try (StringReader rdr = new StringReader(jsonLDString)) { - - // Use JsonLd to expand/compact to localContext - JsonObject jsonld = Json.createReader(rdr).readObject(); - JsonDocument doc = JsonDocument.of(jsonld); - JsonArray array = null; - try { - array = JsonLd.expand(doc).get(); - jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(Json.createObjectBuilder().build())) - .get(); - // jsonld = array.getJsonObject(0); - logger.fine("Decontextualized object: " + jsonld); - return jsonld; - } catch (JsonLdError e) { - System.out.println(e.getMessage()); - return null; - } - } - } - - private static JsonObject recontextualizeJsonLD(JsonObject jsonldObj, MetadataBlockServiceBean metadataBlockSvc) { - - populateContext(metadataBlockSvc); - - // Use JsonLd to expand/compact to localContext - JsonDocument doc = JsonDocument.of(jsonldObj); - JsonArray array = null; - try { - array = JsonLd.expand(doc).get(); - - jsonldObj = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(JSONLDUtil.getContext(localContext))) - .get(); - logger.fine("Compacted: " + jsonldObj.toString()); - return jsonldObj; - } catch (JsonLdError e) { - System.out.println(e.getMessage()); - return null; - } - } - - public static String prettyPrint(JsonValue val) { - StringWriter sw = new StringWriter(); - Map properties = new HashMap<>(1); - properties.put(JsonGenerator.PRETTY_PRINTING, true); - JsonWriterFactory writerFactory = Json.createWriterFactory(properties); - JsonWriter jsonWriter = writerFactory.createWriter(sw); - jsonWriter.write(val); - jsonWriter.close(); - return sw.toString(); - } + private static final Logger logger = Logger.getLogger(JSONLDUtil.class.getCanonicalName()); + + /* + * private static Map populateContext(JsonValue json) { + * Map context = new TreeMap(); if (json + * instanceof JsonArray) { logger.warning("Array @context not yet supported"); } + * else { for (String key : ((JsonObject) json).keySet()) { + * context.putIfAbsent(key, ((JsonObject) json).getString(key)); } } return + * context; } + */ + + public static JsonObject getContext(Map contextMap) { + JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); + for (Entry e : contextMap.entrySet()) { + contextBuilder.add(e.getKey(), e.getValue()); + } + return contextBuilder.build(); + } + + public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { + + DatasetVersion dsv = new DatasetVersion(); + + JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); + Optional maybePid = GlobalId.parse(jsonld.getString("@id")); + if (maybePid.isPresent()) { + ds.setGlobalId(maybePid.get()); + } else { + // unparsable PID passed. Terminate. + throw new BadRequestException("Cannot parse the @id '" + jsonld.getString("@id") + + "'. Make sure it is in valid form - see Dataverse Native API documentation."); + } + + dsv = updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); + dsv.setDataset(ds); + + List versions = new ArrayList<>(1); + versions.add(dsv); + + ds.setVersions(versions); + if (jsonld.containsKey(JsonLDTerm.schemaOrg("dateModified").getUrl())) { + String dateString = jsonld.getString(JsonLDTerm.schemaOrg("dateModified").getUrl()); + LocalDateTime dateTime = getDateTimeFrom(dateString); + ds.setModificationTime(Timestamp.valueOf(dateTime)); + } + try { + logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return ds; + } + + public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, String jsonLDBody, + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { + JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); + return updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); + } + + /** + * + * @param dsv + * @param jsonld + * @param metadataBlockSvc + * @param datasetFieldSvc + * @param append - if append, will add new top level field values for + * multi-valued fields, if true and field type isn't + * multiple, will fail. if false will replace all + * value(s) for fields found in the json-ld. + * @return + */ + public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, JsonObject jsonld, + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { + + populateFieldTypeMap(metadataBlockSvc); + + // get existing ones? + List dsfl = dsv.getDatasetFields(); + Map fieldByTypeMap = new HashMap(); + for (DatasetField dsf : dsfl) { + if (fieldByTypeMap.containsKey(dsf.getDatasetFieldType())) { + // May have multiple values per field, but not multiple fields of one type? + logger.warning("Multiple fields of type " + dsf.getDatasetFieldType().getName()); + } + fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); + } + TermsOfUseAndAccess terms = dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess(); + + for (String key : jsonld.keySet()) { + if (!key.equals("@context")) { + if (dsftMap.containsKey(key)) { + + DatasetFieldType dsft = dsftMap.get(key); + DatasetField dsf = null; + if (fieldByTypeMap.containsKey(dsft)) { + dsf = fieldByTypeMap.get(dsft); + // If there's an existing field, we use it with append and remove it for !append + // unless it's multiple + if (!append && !dsft.isAllowMultiples()) { + dsfl.remove(dsf); + } + } + if (dsf == null) { + dsf = new DatasetField(); + dsfl.add(dsf); + dsf.setDatasetFieldType(dsft); + } + + // Todo - normalize object vs. array + JsonArray valArray = getValues(jsonld.get(key), dsft.isAllowMultiples(), dsft.getName()); + + addField(dsf, valArray, dsft, datasetFieldSvc, append); + + // assemble new terms, add to existing + // multivalue? + // compound? + // merge with existing dv metadata + // dsfl.add(dsf); + } else { + // Internal/non-metadatablock terms + // Add metadata related to the Dataset/DatasetVersion + + // ("@id", id) - check is equal to existing globalID? + // Add to 'md on original' ? + // (JsonLDTerm.schemaOrg("version").getLabel(), + // version.getFriendlyVersionNumber()) + // Citation metadata? + // (JsonLDTerm.schemaOrg("datePublished").getLabel(), + // dataset.getPublicationDateFormattedYYYYMMDD()) + // (JsonLDTerm.schemaOrg("name").getLabel()) + // (JsonLDTerm.schemaOrg("dateModified").getLabel()) + + if (key.equals(JsonLDTerm.schemaOrg("datePublished").getUrl())&& migrating && !append) { + dsv.setVersionState(VersionState.RELEASED); + } else if (key.equals(JsonLDTerm.schemaOrg("version").getUrl())&& migrating && !append) { + String friendlyVersion = jsonld.getString(JsonLDTerm.schemaOrg("version").getUrl()); + int index = friendlyVersion.indexOf("."); + if (index > 0) { + dsv.setVersionNumber(Long.parseLong(friendlyVersion.substring(0, index))); + dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index + 1))); + } + } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())) { + if (!append || !isSet(terms, key)) { + // Mirror rules from SwordServiceBean + if (jsonld.containsKey(JsonLDTerm.termsOfUse.getUrl())) { + throw new BadRequestException( + "Cannot specify " + JsonLDTerm.schemaOrg("license").getUrl() + " and " + + JsonLDTerm.termsOfUse.getUrl()); + } + setSemTerm(terms, key, TermsOfUseAndAccess.defaultLicense); + } else { + throw new BadRequestException( + "Can't append to a single-value field that already has a value: " + + JsonLDTerm.schemaOrg("license").getUrl()); + } + + } else if (datasetTerms.contains(key)) { + if (!append || !isSet(terms, key)) { + // Other Dataset-level TermsOfUseAndAccess + setSemTerm(terms, key, jsonld.getString(JsonLDTerm.termsOfUse.getUrl())); + } else { + throw new BadRequestException( + "Can't append to a single-value field that already has a value: " + key); + } + } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { + JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); + for (String fileKey : fAccessObject.keySet()) { + if (datafileTerms.contains(fileKey)) { + if (!append || !isSet(terms, fileKey)) { + // Other DataFile-level TermsOfUseAndAccess + if (fileKey.equals(JsonLDTerm.fileRequestAccess.getUrl())) { + setSemTerm(terms, fileKey, fAccessObject.getBoolean(fileKey)); + } else { + setSemTerm(terms, fileKey, fAccessObject.getString(fileKey)); + } + } else { + throw new BadRequestException( + "Can't append to a single-value field that already has a value: " + + fileKey); + } + } + } + } else { + if (dsftMap.containsKey(JsonLDTerm.metadataOnOrig.getUrl())) { + DatasetFieldType dsft = dsftMap.get(JsonLDTerm.metadataOnOrig.getUrl()); + + DatasetField dsf = null; + if (fieldByTypeMap.containsKey(dsft)) { + dsf = fieldByTypeMap.get(dsft); + // If there's an existing field, we use it with append and remove it for !append + // (except if multiple, which is not the default) + if (!append && !dsft.isAllowMultiples()) { + dsfl.remove(dsf); + } + } + if (dsf == null) { + dsf = new DatasetField(); + dsfl.add(dsf); + dsf.setDatasetFieldType(dsft); + } + + List vals = dsf.getDatasetFieldValues(); + + JsonObject currentValue = null; + DatasetFieldValue datasetFieldValue = null; + if (vals.isEmpty()) { + datasetFieldValue = new DatasetFieldValue(); + vals.add(datasetFieldValue); + datasetFieldValue.setDatasetField(dsf); + dsf.setDatasetFieldValues(vals); + + currentValue = Json.createObjectBuilder().build(); + } else { + datasetFieldValue = vals.get(0); + JsonObject currentVal = decontextualizeJsonLD(datasetFieldValue.getValueForEdit()); + + } + currentValue.put(key, jsonld.get(key)); + JsonObject newValue = recontextualizeJsonLD(currentValue, metadataBlockSvc); + datasetFieldValue.setValue(prettyPrint(newValue)); + } + } + dsv.setTermsOfUseAndAccess(terms); + // move to new dataverse? + // aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), + // dataset.getDataverseContext().getDisplayName()); + + } + + } + } + + dsv.setDatasetFields(dsfl); + + return dsv; + } + + private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldType dsft, + DatasetFieldServiceBean datasetFieldSvc, boolean append) { + + if (append && !dsft.isAllowMultiples()) { + if ((dsft.isCompound() && !dsf.getDatasetFieldCompoundValues().isEmpty()) + || (dsft.isAllowControlledVocabulary() && !dsf.getControlledVocabularyValues().isEmpty()) + || !dsf.getDatasetFieldValues().isEmpty()) { + throw new BadRequestException( + "Can't append to a single-value field that already has a value: " + dsft.getName()); + } + } + logger.fine("Name: " + dsft.getName()); + logger.fine("val: " + valArray.toString()); + logger.fine("Compound: " + dsft.isCompound()); + logger.fine("CV: " + dsft.isAllowControlledVocabulary()); + + if (dsft.isCompound()) { + /* + * List vals = parseCompoundValue(type, + * jsonld.get(key),testType); for (DatasetFieldCompoundValue dsfcv : vals) { + * dsfcv.setParentDatasetField(ret); } dsf.setDatasetFieldCompoundValues(vals); + */ + List cvList = dsf.getDatasetFieldCompoundValues(); + if (!cvList.isEmpty()) { + if (!append) { + cvList.clear(); + } else if (!dsft.isAllowMultiples() && cvList.size() == 1) { + // Trying to append but only a single value is allowed (and there already is + // one) + // (and we don't currently support appending new fields within a compound value) + throw new BadRequestException( + "Append with compound field with single value not yet supported: " + dsft.getDisplayName()); + } + } + + List vals = new LinkedList<>(); + for (JsonValue val : valArray) { + if (!(val instanceof JsonObject)) { + throw new BadRequestException( + "Compound field values must be JSON objects, field: " + dsft.getName()); + } + DatasetFieldCompoundValue cv = null; + + cv = new DatasetFieldCompoundValue(); + cv.setDisplayOrder(cvList.size()); + cvList.add(cv); + cv.setParentDatasetField(dsf); + + JsonObject obj = (JsonObject) val; + for (String childKey : obj.keySet()) { + if (dsftMap.containsKey(childKey)) { + DatasetFieldType childft = dsftMap.get(childKey); + if (!dsft.getChildDatasetFieldTypes().contains(childft)) { + throw new BadRequestException( + "Compound field " + dsft.getName() + "can't include term " + childKey); + } + DatasetField childDsf = new DatasetField(); + cv.getChildDatasetFields().add(childDsf); + childDsf.setDatasetFieldType(childft); + childDsf.setParentDatasetFieldCompoundValue(cv); + + JsonArray childValArray = getValues(obj.get(childKey), childft.isAllowMultiples(), + childft.getName()); + addField(childDsf, childValArray, childft, datasetFieldSvc, append); + } + } + } + + } else if (dsft.isControlledVocabulary()) { + + List vals = dsf.getControlledVocabularyValues(); + for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + ControlledVocabularyValue cvv = datasetFieldSvc + .findControlledVocabularyValueByDatasetFieldTypeAndStrValue(dsft, strValue, true); + if (cvv == null) { + throw new BadRequestException( + "Unknown value for Controlled Vocab Field: " + dsft.getName() + " : " + strValue); + } + // Only add value to the list if it is not a duplicate + if (strValue.equals("Other")) { + System.out.println("vals = " + vals + ", contains: " + vals.contains(cvv)); + } + if (!vals.contains(cvv)) { + if (vals.size() > 0) { + cvv.setDisplayOrder(vals.size()); + } + vals.add(cvv); + cvv.setDatasetFieldType(dsft); + } + } + dsf.setControlledVocabularyValues(vals); + + } else { + List vals = dsf.getDatasetFieldValues(); + + for (JsonString strVal : valArray.getValuesAs(JsonString.class)) { + String strValue = strVal.getString(); + DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); + + datasetFieldValue.setDisplayOrder(vals.size()); + datasetFieldValue.setValue(strValue.trim()); + vals.add(datasetFieldValue); + datasetFieldValue.setDatasetField(dsf); + + } + dsf.setDatasetFieldValues(vals); + } + } + + private static JsonArray getValues(JsonValue val, boolean allowMultiples, String name) { + JsonArray valArray = null; + if (val instanceof JsonArray) { + if ((((JsonArray) val).size() > 1) && !allowMultiples) { + throw new BadRequestException("Array for single value notsupported: " + name); + } else { + valArray = (JsonArray) val; + } + } else { + valArray = Json.createArrayBuilder().add(val).build(); + } + return valArray; + } + + static Map localContext = new TreeMap(); + static Map dsftMap = new TreeMap(); + + private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockSvc) { + if (dsftMap.isEmpty()) { + + List mdbList = metadataBlockSvc.listMetadataBlocks(); + + for (MetadataBlock mdb : mdbList) { + boolean blockHasUri = mdb.getNamespaceUri() != null; + for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { + if (dsft.getUri() != null) { + dsftMap.put(dsft.getUri(), dsft); + } + if (blockHasUri) { + if (dsft.getParentDatasetFieldType() != null) { + // ToDo - why not getName for child type? Would have to fix in ORE generation + // code and handle legacy bags + dsftMap.put(mdb.getNamespaceUri() + dsft.getParentDatasetFieldType().getName() + "#" + + dsft.getTitle(), dsft); + } else { + dsftMap.put(mdb.getNamespaceUri() + dsft.getTitle(), dsft); + } + } + } + } + logger.fine("DSFT Map: " + String.join(", ", dsftMap.keySet())); + } + } + + private static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { + if (localContext.isEmpty()) { + + // Add namespaces corresponding to core terms + localContext.put(JsonLDNamespace.dcterms.getPrefix(), JsonLDNamespace.dcterms.getUrl()); + localContext.put(JsonLDNamespace.dvcore.getPrefix(), JsonLDNamespace.dvcore.getUrl()); + localContext.put(JsonLDNamespace.schema.getPrefix(), JsonLDNamespace.schema.getUrl()); + + List mdbList = metadataBlockSvc.listMetadataBlocks(); + + for (MetadataBlock mdb : mdbList) { + boolean blockHasUri = mdb.getNamespaceUri() != null; + if (blockHasUri) { + localContext.putIfAbsent(mdb.getName(), mdb.getNamespaceUri()); + + } + for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { + if (dsft.getUri() != null) { + localContext.putIfAbsent(dsft.getName(), dsft.getUri()); + } + } + } + logger.fine("LocalContext keys: " + String.join(", ", localContext.keySet())); + } + } + + public static JsonObject decontextualizeJsonLD(String jsonLDString) { + logger.fine(jsonLDString); + try (StringReader rdr = new StringReader(jsonLDString)) { + + // Use JsonLd to expand/compact to localContext + JsonObject jsonld = Json.createReader(rdr).readObject(); + JsonDocument doc = JsonDocument.of(jsonld); + JsonArray array = null; + try { + array = JsonLd.expand(doc).get(); + jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(Json.createObjectBuilder().build())) + .get(); + // jsonld = array.getJsonObject(0); + logger.fine("Decontextualized object: " + jsonld); + return jsonld; + } catch (JsonLdError e) { + System.out.println(e.getMessage()); + return null; + } + } + } + + private static JsonObject recontextualizeJsonLD(JsonObject jsonldObj, MetadataBlockServiceBean metadataBlockSvc) { + + populateContext(metadataBlockSvc); + + // Use JsonLd to expand/compact to localContext + JsonDocument doc = JsonDocument.of(jsonldObj); + JsonArray array = null; + try { + array = JsonLd.expand(doc).get(); + + jsonldObj = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(JSONLDUtil.getContext(localContext))) + .get(); + logger.fine("Compacted: " + jsonldObj.toString()); + return jsonldObj; + } catch (JsonLdError e) { + System.out.println(e.getMessage()); + return null; + } + } + + public static String prettyPrint(JsonValue val) { + StringWriter sw = new StringWriter(); + Map properties = new HashMap<>(1); + properties.put(JsonGenerator.PRETTY_PRINTING, true); + JsonWriterFactory writerFactory = Json.createWriterFactory(properties); + JsonWriter jsonWriter = writerFactory.createWriter(sw); + jsonWriter.write(val); + jsonWriter.close(); + return sw.toString(); + } //Modified from https://stackoverflow.com/questions/3389348/parse-any-date-in-java - private static final Map DATE_FORMAT_REGEXPS = new HashMap() { - { - put("^\\d{8}$", "yyyyMMdd"); - put("^\\d{1,2}-\\d{1,2}-\\d{4}$", "dd-MM-yyyy"); - put("^\\d{4}-\\d{1,2}-\\d{1,2}$", "yyyy-MM-dd"); - put("^\\d{1,2}/\\d{1,2}/\\d{4}$", "MM/dd/yyyy"); - put("^\\d{4}/\\d{1,2}/\\d{1,2}$", "yyyy/MM/dd"); - put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}$", "dd MMM yyyy"); - put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}$", "dd MMMM yyyy"); - } - }; - - private static final Map DATETIME_FORMAT_REGEXPS = new HashMap() { - { - put("^\\d{12}$", "yyyyMMddHHmm"); - put("^\\d{8}\\s\\d{4}$", "yyyyMMdd HHmm"); - put("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}$", "dd-MM-yyyy HH:mm"); - put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}$", "yyyy-MM-dd HH:mm"); - put("^\\d{1,2}/\\d{1,2}/\\d{4}\\s\\d{1,2}:\\d{2}$", "MM/dd/yyyy HH:mm"); - put("^\\d{4}/\\d{1,2}/\\d{1,2}\\s\\d{1,2}:\\d{2}$", "yyyy/MM/dd HH:mm"); - put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}$", "dd MMM yyyy HH:mm"); - put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}$", "dd MMMM yyyy HH:mm"); - put("^\\d{14}$", "yyyyMMddHHmmss"); - put("^\\d{8}\\s\\d{6}$", "yyyyMMdd HHmmss"); - put("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd-MM-yyyy HH:mm:ss"); - put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}$", "yyyy-MM-dd HH:mm:ss"); - put("^\\d{1,2}/\\d{1,2}/\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "MM/dd/yyyy HH:mm:ss"); - put("^\\d{4}/\\d{1,2}/\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}$", "yyyy/MM/dd HH:mm:ss"); - put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMM yyyy HH:mm:ss"); - put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMMM yyyy HH:mm:ss"); - put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\.\\d{3}$", "yyyy-MM-dd HH:mm:ss.SSS"); - put("^[a-z,A-Z]{3}\\s[a-z,A-Z]{3}\\s\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\s[a-z,A-Z]{3}\\s\\d{4}$", - "EEE MMM dd HH:mm:ss zzz yyyy"); // Wed Sep 23 19:33:46 UTC 2020 - - } - }; - - /** - * Determine DateTimeFormatter pattern matching with the given date string. - * Returns null if format is unknown. You can simply extend DateUtil with more - * formats if needed. - * - * @param dateString The date string to determine the SimpleDateFormat pattern - * for. - * @return The matching SimpleDateFormat pattern, or null if format is unknown. - * @see SimpleDateFormat - */ - public static DateTimeFormatter determineDateTimeFormat(String dateString) { - for (String regexp : DATETIME_FORMAT_REGEXPS.keySet()) { - if (dateString.toLowerCase().matches(regexp)) { - return DateTimeFormatter.ofPattern(DATETIME_FORMAT_REGEXPS.get(regexp)); - } - } - logger.warning("Unknown datetime format: " + dateString); - return null; // Unknown format. - } - - public static DateTimeFormatter determineDateFormat(String dateString) { - for (String regexp : DATE_FORMAT_REGEXPS.keySet()) { - if (dateString.toLowerCase().matches(regexp)) { - return DateTimeFormatter.ofPattern(DATE_FORMAT_REGEXPS.get(regexp)); - } - } - logger.warning("Unknown date format: " + dateString); - return null; // Unknown format. - } - - public static LocalDateTime getDateTimeFrom(String dateString) { - DateTimeFormatter dtf = determineDateTimeFormat(dateString); - if (dtf != null) { - return LocalDateTime.parse(dateString, dtf); - } else { - dtf = determineDateFormat(dateString); - if (dtf != null) { - return LocalDate.parse(dateString, dtf).atStartOfDay(); - } - } - - return null; - } + private static final Map DATE_FORMAT_REGEXPS = new HashMap() { + { + put("^\\d{8}$", "yyyyMMdd"); + put("^\\d{1,2}-\\d{1,2}-\\d{4}$", "dd-MM-yyyy"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}$", "yyyy-MM-dd"); + put("^\\d{1,2}/\\d{1,2}/\\d{4}$", "MM/dd/yyyy"); + put("^\\d{4}/\\d{1,2}/\\d{1,2}$", "yyyy/MM/dd"); + put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}$", "dd MMM yyyy"); + put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}$", "dd MMMM yyyy"); + } + }; + + private static final Map DATETIME_FORMAT_REGEXPS = new HashMap() { + { + put("^\\d{12}$", "yyyyMMddHHmm"); + put("^\\d{8}\\s\\d{4}$", "yyyyMMdd HHmm"); + put("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}$", "dd-MM-yyyy HH:mm"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}$", "yyyy-MM-dd HH:mm"); + put("^\\d{1,2}/\\d{1,2}/\\d{4}\\s\\d{1,2}:\\d{2}$", "MM/dd/yyyy HH:mm"); + put("^\\d{4}/\\d{1,2}/\\d{1,2}\\s\\d{1,2}:\\d{2}$", "yyyy/MM/dd HH:mm"); + put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}$", "dd MMM yyyy HH:mm"); + put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}$", "dd MMMM yyyy HH:mm"); + put("^\\d{14}$", "yyyyMMddHHmmss"); + put("^\\d{8}\\s\\d{6}$", "yyyyMMdd HHmmss"); + put("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd-MM-yyyy HH:mm:ss"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}$", "yyyy-MM-dd HH:mm:ss"); + put("^\\d{1,2}/\\d{1,2}/\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "MM/dd/yyyy HH:mm:ss"); + put("^\\d{4}/\\d{1,2}/\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}$", "yyyy/MM/dd HH:mm:ss"); + put("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMM yyyy HH:mm:ss"); + put("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$", "dd MMMM yyyy HH:mm:ss"); + put("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\.\\d{3}$", "yyyy-MM-dd HH:mm:ss.SSS"); + put("^[a-z,A-Z]{3}\\s[a-z,A-Z]{3}\\s\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\s[a-z,A-Z]{3}\\s\\d{4}$", + "EEE MMM dd HH:mm:ss zzz yyyy"); // Wed Sep 23 19:33:46 UTC 2020 + + } + }; + + /** + * Determine DateTimeFormatter pattern matching with the given date string. + * Returns null if format is unknown. You can simply extend DateUtil with more + * formats if needed. + * + * @param dateString The date string to determine the SimpleDateFormat pattern + * for. + * @return The matching SimpleDateFormat pattern, or null if format is unknown. + * @see SimpleDateFormat + */ + public static DateTimeFormatter determineDateTimeFormat(String dateString) { + for (String regexp : DATETIME_FORMAT_REGEXPS.keySet()) { + if (dateString.toLowerCase().matches(regexp)) { + return DateTimeFormatter.ofPattern(DATETIME_FORMAT_REGEXPS.get(regexp)); + } + } + logger.warning("Unknown datetime format: " + dateString); + return null; // Unknown format. + } + + public static DateTimeFormatter determineDateFormat(String dateString) { + for (String regexp : DATE_FORMAT_REGEXPS.keySet()) { + if (dateString.toLowerCase().matches(regexp)) { + return DateTimeFormatter.ofPattern(DATE_FORMAT_REGEXPS.get(regexp)); + } + } + logger.warning("Unknown date format: " + dateString); + return null; // Unknown format. + } + + public static LocalDateTime getDateTimeFrom(String dateString) { + DateTimeFormatter dtf = determineDateTimeFormat(dateString); + if (dtf != null) { + return LocalDateTime.parse(dateString, dtf); + } else { + dtf = determineDateFormat(dateString); + if (dtf != null) { + return LocalDate.parse(dateString, dtf).atStartOfDay(); + } + } + + return null; + } + + // Convenience methods for TermsOfUseAndAccess + + public static final List datasetTerms = new ArrayList(Arrays.asList( + "https://dataverse.org/schema/core#termsOfUse", + "https://dataverse.org/schema/core#confidentialityDeclaration", + "https://dataverse.org/schema/core#specialPermissions", "https://dataverse.org/schema/core#restrictions", + "https://dataverse.org/schema/core#citationRequirements", + "https://dataverse.org/schema/core#depositorRequirements", "https://dataverse.org/schema/core#conditions", + "https://dataverse.org/schema/core#disclaimer")); + public static final List datafileTerms = new ArrayList(Arrays.asList( + "https://dataverse.org/schema/core#termsOfAccess", "https://dataverse.org/schema/core#fileRequestAccess", + "https://dataverse.org/schema/core#dataAccessPlace", "https://dataverse.org/schema/core#originalArchive", + "https://dataverse.org/schema/core#availabilityStatus", + "https://dataverse.org/schema/core#contactForAccess", "https://dataverse.org/schema/core#sizeOfCollection", + "https://dataverse.org/schema/core#studyCompletion")); + + public static boolean isSet(TermsOfUseAndAccess terms, String semterm) { + switch (semterm) { + case "http://schema.org/license": + return !terms.getLicense().equals(TermsOfUseAndAccess.License.NONE); + case "https://dataverse.org/schema/core#termsOfUse": + return StringUtils.isBlank(terms.getTermsOfUse()); + case "https://dataverse.org/schema/core#confidentialityDeclaration": + return StringUtils.isBlank(terms.getConfidentialityDeclaration()); + case "https://dataverse.org/schema/core#specialPermissions": + return StringUtils.isBlank(terms.getSpecialPermissions()); + case "https://dataverse.org/schema/core#restrictions": + return StringUtils.isBlank(terms.getRestrictions()); + case "https://dataverse.org/schema/core#citationRequirements": + return StringUtils.isBlank(terms.getCitationRequirements()); + case "https://dataverse.org/schema/core#depositorRequirements": + return StringUtils.isBlank(terms.getDepositorRequirements()); + case "https://dataverse.org/schema/core#conditions": + return StringUtils.isBlank(terms.getConditions()); + case "https://dataverse.org/schema/core#disclaimer": + return StringUtils.isBlank(terms.getDisclaimer()); + case "https://dataverse.org/schema/core#termsOfAccess": + return StringUtils.isBlank(terms.getTermsOfAccess()); + case "https://dataverse.org/schema/core#fileRequestAccess": + return !terms.isFileAccessRequest(); + case "https://dataverse.org/schema/core#dataAccessPlace": + return StringUtils.isBlank(terms.getDataAccessPlace()); + case "https://dataverse.org/schema/core#originalArchive": + return StringUtils.isBlank(terms.getOriginalArchive()); + case "https://dataverse.org/schema/core#availabilityStatus": + return StringUtils.isBlank(terms.getAvailabilityStatus()); + case "https://dataverse.org/schema/core#contactForAccess": + return StringUtils.isBlank(terms.getContactForAccess()); + case "https://dataverse.org/schema/core#sizeOfCollection": + return StringUtils.isBlank(terms.getSizeOfCollection()); + case "https://dataverse.org/schema/core#studyCompletion": + return StringUtils.isBlank(terms.getStudyCompletion()); + default: + logger.warning("isSet called for " + semterm); + return false; + } + } + + public static void setSemTerm(TermsOfUseAndAccess terms, String semterm, Object value) { + switch (semterm) { + case "http://schema.org/license": + // Mirror rules from SwordServiceBean + if (((License) value).equals(TermsOfUseAndAccess.defaultLicense)) { + terms.setLicense(TermsOfUseAndAccess.defaultLicense); + } else { + throw new BadRequestException("The only allowed value for " + JsonLDTerm.schemaOrg("license").getUrl() + + " is " + "https://creativecommons.org/publicdomain/zero/1.0/"); + } + break; + case "https://dataverse.org/schema/core#termsOfUse": + terms.setTermsOfUse((String) value); + break; + case "https://dataverse.org/schema/core#confidentialityDeclaration": + terms.setConfidentialityDeclaration((String) value); + break; + case "https://dataverse.org/schema/core#specialPermissions": + terms.setSpecialPermissions((String) value); + break; + case "https://dataverse.org/schema/core#restrictions": + terms.setRestrictions((String) value); + break; + case "https://dataverse.org/schema/core#citationRequirements": + terms.setCitationRequirements((String) value); + break; + case "https://dataverse.org/schema/core#depositorRequirements": + terms.setDepositorRequirements((String) value); + break; + case "https://dataverse.org/schema/core#conditions": + terms.setConditions((String) value); + break; + case "https://dataverse.org/schema/core#disclaimer": + terms.setDisclaimer((String) value); + break; + case "https://dataverse.org/schema/core#termsOfAccess": + terms.setTermsOfAccess((String) value); + break; + case "https://dataverse.org/schema/core#fileRequestAccess": + terms.setFileAccessRequest((boolean) value); + break; + case "https://dataverse.org/schema/core#dataAccessPlace": + terms.setDataAccessPlace((String) value); + break; + case "https://dataverse.org/schema/core#originalArchive": + terms.setOriginalArchive((String) value); + break; + case "https://dataverse.org/schema/core#availabilityStatus": + terms.setAvailabilityStatus((String) value); + break; + case "https://dataverse.org/schema/core#contactForAccess": + terms.setContactForAccess((String) value); + break; + case "https://dataverse.org/schema/core#sizeOfCollection": + terms.setSizeOfCollection((String) value); + break; + case "https://dataverse.org/schema/core#studyCompletion": + terms.setStudyCompletion((String) value); + break; + default: + logger.warning("setSemTerm called for " + semterm); + break; + } + } } From be978e966dd9d20e4fe517f07962a99f81472375 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 3 Dec 2020 13:13:28 -0500 Subject: [PATCH 081/354] expose uri in datasetField api --- .../edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java index d02e8f72838..c945ba4df56 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java @@ -137,6 +137,7 @@ public Response getByName(@PathParam("name") String name) { String solrFieldSearchable = dsf.getSolrField().getNameSearchable(); String solrFieldFacetable = dsf.getSolrField().getNameFacetable(); String metadataBlock = dsf.getMetadataBlock().getName(); + String uri=dsf.getUri(); boolean hasParent = dsf.isHasParent(); boolean allowsMultiples = dsf.isAllowMultiples(); boolean isRequired = dsf.isRequired(); @@ -168,7 +169,8 @@ public Response getByName(@PathParam("name") String name) { .add("parentAllowsMultiples", parentAllowsMultiplesDisplay) .add("solrFieldSearchable", solrFieldSearchable) .add("solrFieldFacetable", solrFieldFacetable) - .add("isRequired", isRequired)); + .add("isRequired", isRequired) + .add("uri", uri)); } catch ( NoResultException nre ) { return notFound(name); From faa92f6fde17f5e4cc41ec657c0433158fbc333a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 3 Dec 2020 14:27:14 -0500 Subject: [PATCH 082/354] track defined namespaces and avoid having contexts with specific entries for terms that are in a namespace already --- .../iq/dataverse/util/bagit/OREMap.java | 4 +-- .../iq/dataverse/util/json/JSONLDUtil.java | 12 +++---- .../dataverse/util/json/JsonLDNamespace.java | 34 ++++++++++++++++++- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index f6c064183c9..6dbcde153c8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -364,7 +364,7 @@ private JsonLDTerm getTermFor(DatasetFieldType dsft) { namespaceUri = SystemConfig.getDataverseSiteUrlStatic() + "/schema/" + dsft.getMetadataBlock().getName() + "#"; } - JsonLDNamespace blockNamespace = new JsonLDNamespace(dsft.getMetadataBlock().getName(), namespaceUri); + JsonLDNamespace blockNamespace = JsonLDNamespace.defineNamespace(dsft.getMetadataBlock().getName(), namespaceUri); return new JsonLDTerm(blockNamespace, dsft.getTitle()); } } @@ -380,7 +380,7 @@ private JsonLDTerm getTermFor(DatasetFieldType dfType, DatasetFieldType dsft) { + dfType.getMetadataBlock().getName() + "/"; } subFieldNamespaceUri = subFieldNamespaceUri + dfType.getName() + "#"; - JsonLDNamespace fieldNamespace = new JsonLDNamespace(dfType.getName(), subFieldNamespaceUri); + JsonLDNamespace fieldNamespace = JsonLDNamespace.defineNamespace(dfType.getName(), subFieldNamespaceUri); return new JsonLDTerm(fieldNamespace, dsft.getTitle()); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 414b3ec8a01..4f451f01ca5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -452,25 +452,21 @@ private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockS private static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { if (localContext.isEmpty()) { - // Add namespaces corresponding to core terms - localContext.put(JsonLDNamespace.dcterms.getPrefix(), JsonLDNamespace.dcterms.getUrl()); - localContext.put(JsonLDNamespace.dvcore.getPrefix(), JsonLDNamespace.dvcore.getUrl()); - localContext.put(JsonLDNamespace.schema.getPrefix(), JsonLDNamespace.schema.getUrl()); - List mdbList = metadataBlockSvc.listMetadataBlocks(); for (MetadataBlock mdb : mdbList) { boolean blockHasUri = mdb.getNamespaceUri() != null; if (blockHasUri) { - localContext.putIfAbsent(mdb.getName(), mdb.getNamespaceUri()); - + JsonLDNamespace.defineNamespace(mdb.getName(), mdb.getNamespaceUri()); } for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { - if (dsft.getUri() != null) { + if ((dsft.getUri() != null) && !JsonLDNamespace.isInNamespace(dsft.getUri())) { + //Add term if uri exists and it's not in one of the namespaces already defined localContext.putIfAbsent(dsft.getName(), dsft.getUri()); } } } + JsonLDNamespace.addNamespacesToContext(localContext); logger.fine("LocalContext keys: " + String.join(", ", localContext.keySet())); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java index bda4a55d623..2c870edc106 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java @@ -1,5 +1,10 @@ package edu.harvard.iq.dataverse.util.json; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + public class JsonLDNamespace { String prefix; @@ -12,7 +17,34 @@ public class JsonLDNamespace { public static JsonLDNamespace ore = new JsonLDNamespace("ore","http://www.openarchives.org/ore/terms/"); public static JsonLDNamespace schema = new JsonLDNamespace("schema","http://schema.org/"); - public JsonLDNamespace(String prefix, String url) { + private static List namespaces = new ArrayList(Arrays.asList(dvcore, dcterms, ore, schema)); + + public static JsonLDNamespace defineNamespace(String prefix, String url) { + JsonLDNamespace ns = new JsonLDNamespace(prefix, url); + namespaces.add(ns); + return ns; + } + + public static void deleteNamespace(JsonLDNamespace ns) { + namespaces.remove(ns); + } + + public static boolean isInNamespace(String url) { + for(JsonLDNamespace ns: namespaces) { + if(url.startsWith(ns.getUrl())) { + return true; + } + } + return false; + } + + public static void addNamespacesToContext(Map context) { + for(JsonLDNamespace ns: namespaces) { + context.putIfAbsent(ns.getPrefix(), ns.getUrl()); + }; + } + + private JsonLDNamespace(String prefix, String url) { this.prefix = prefix; this.url = url; } From fb08a3834d8432791f10c02f1fcba3dad7ab2a29 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 3 Dec 2020 15:22:37 -0500 Subject: [PATCH 083/354] define equals, avoid duplicates in list --- .../dataverse/util/json/JsonLDNamespace.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java index 2c870edc106..904419775c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDNamespace.java @@ -4,6 +4,9 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; + +import edu.harvard.iq.dataverse.DataFile; public class JsonLDNamespace { @@ -20,9 +23,15 @@ public class JsonLDNamespace { private static List namespaces = new ArrayList(Arrays.asList(dvcore, dcterms, ore, schema)); public static JsonLDNamespace defineNamespace(String prefix, String url) { + JsonLDNamespace ns = new JsonLDNamespace(prefix, url); - namespaces.add(ns); - return ns; + int i = namespaces.indexOf(ns); + if(i>=0) { + return namespaces.get(i); + } else { + namespaces.add(ns); + return ns; + } } public static void deleteNamespace(JsonLDNamespace ns) { @@ -56,5 +65,14 @@ public String getPrefix() { public String getUrl() { return url; } + + @Override + public boolean equals(Object object) { + if (!(object instanceof JsonLDNamespace)) { + return false; + } + JsonLDNamespace other = (JsonLDNamespace) object; + return (other.getPrefix().equals(getPrefix()) && other.getUrl().equals(getUrl())); + } } From ece3bffef31b0e7a4665ef5792780f53ce1a55cc Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 4 Dec 2020 15:27:43 -0500 Subject: [PATCH 084/354] replace string with const --- .../java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java | 2 +- .../iq/dataverse/export/openaire/OpenAireExportUtil.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java b/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java index ad6775d6efd..30eae110102 100644 --- a/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java @@ -280,7 +280,7 @@ public enum License { * API use? See also https://github.com/IQSS/dataverse/issues/1385 */ public static TermsOfUseAndAccess.License defaultLicense = TermsOfUseAndAccess.License.CC0; - + public final static String CC0_URI = "https://creativecommons.org/publicdomain/zero/1.0/"; @Override public int hashCode() { int hash = 0; diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java index fe0c15969ca..f972ae2a983 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java @@ -19,6 +19,7 @@ import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.api.dto.DatasetDTO; import edu.harvard.iq.dataverse.api.dto.DatasetVersionDTO; import edu.harvard.iq.dataverse.api.dto.FieldDTO; @@ -1135,7 +1136,7 @@ public static void writeAccessRightsElement(XMLStreamWriter xmlw, DatasetVersion writeRightsHeader(xmlw, language); if (StringUtils.isNotBlank(datasetVersionDTO.getLicense())) { if (StringUtils.containsIgnoreCase(datasetVersionDTO.getLicense(), "cc0")) { - xmlw.writeAttribute("rightsURI", "https://creativecommons.org/publicdomain/zero/1.0/"); + xmlw.writeAttribute("rightsURI", TermsOfUseAndAccess.CC0_URI); if (StringUtils.isNotBlank(datasetVersionDTO.getTermsOfUse())) { xmlw.writeCharacters(datasetVersionDTO.getTermsOfUse()); } From 8b737dd9046238dd6cc86d7870495cad056ef2f8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 4 Dec 2020 15:28:14 -0500 Subject: [PATCH 085/354] constant for CC0_URI --- src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 83152d1a75f..53a58b4228f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1856,7 +1856,7 @@ public String getJsonLd() { JsonObjectBuilder license = Json.createObjectBuilder().add("@type", "Dataset"); if (TermsOfUseAndAccess.License.CC0.equals(terms.getLicense())) { - license.add("text", "CC0").add("url", "https://creativecommons.org/publicdomain/zero/1.0/"); + license.add("text", "CC0").add("url", TermsOfUseAndAccess.CC0_URI); } else { String termsOfUse = terms.getTermsOfUse(); // Terms of use can be null if you create the dataset with JSON. From 63362d9f87df415198d8c11aa9367580df7bd796 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 4 Dec 2020 15:28:32 -0500 Subject: [PATCH 086/354] GET/DELETE endpoints Conflicts: src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java --- .../iq/dataverse/TermsOfUseAndAccess.java | 2 +- .../harvard/iq/dataverse/api/Datasets.java | 73 +++++- .../iq/dataverse/util/bagit/OREMap.java | 201 ++++++++-------- .../iq/dataverse/util/json/JSONLDUtil.java | 220 +++++++++++++++++- 4 files changed, 375 insertions(+), 121 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java b/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java index 30eae110102..72f4ab54ee8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java @@ -280,7 +280,7 @@ public enum License { * API use? See also https://github.com/IQSS/dataverse/issues/1385 */ public static TermsOfUseAndAccess.License defaultLicense = TermsOfUseAndAccess.License.CC0; - public final static String CC0_URI = "https://creativecommons.org/publicdomain/zero/1.0/"; + public static String CC0_URI = "https://creativecommons.org/publicdomain/zero/1.0/"; @Override public int hashCode() { int hash = 0; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 527476ea59a..6ac132e771b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -102,6 +102,7 @@ import edu.harvard.iq.dataverse.util.EjbUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.util.json.JSONLDUtil; import edu.harvard.iq.dataverse.util.json.JsonLDTerm; import edu.harvard.iq.dataverse.util.json.JsonParseException; @@ -613,22 +614,80 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, } } + + @GET + @Path("{id}/versions/{versionid}/metadata") + @Produces("application/json-ld") + public Response getVersionMetadata(@PathParam("id") String id, @PathParam("versionId") String versionId) { + try { + Dataset ds = findDatasetOrDie(id); + DataverseRequest req = createDataverseRequest(findUserOrDie()); + DatasetVersion dsv = ds.getEditVersion(); + OREMap ore = new OREMap(dsv, + settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)); + return ok(JSONLDUtil.prettyPrint(ore.getOREMap(true))); + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } catch (Exception jpe) { + logger.log(Level.SEVERE, "Error getting jsonld metadata for dsv: ", jpe.getLocalizedMessage()); + return error(Response.Status.INTERNAL_SERVER_ERROR, jpe.getLocalizedMessage()); + } + } + + @GET + @Path("{id}/metadata") + @Produces("application/json-ld") + public Response getVersionMetadata(@PathParam("id") String id) { + return getVersionMetadata(id, ":draft"); + } + + @PUT + @Path("{id}/metadata") + @Consumes("application/json-ld") + public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, + @PathParam("versionId") String versionId, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { + + if (!":draft".equals(versionId)) { + return error(Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); + } + try { + Dataset ds = findDatasetOrDie(id); + DataverseRequest req = createDataverseRequest(findUserOrDie()); + DatasetVersion dsv = ds.getEditVersion(); + boolean updateDraft = ds.getLatestVersion().isDraft(); + dsv = JSONLDUtil.updateDatasetVersionMDFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, !replaceTerms, false); + + DatasetVersion managedVersion; + if (updateDraft) { + Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); + managedVersion = managedDataset.getEditVersion(); + } else { + managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); + } + String info = updateDraft ? "Version Updated" : "Version Created"; + return ok(Json.createObjectBuilder().add(info, managedVersion.getVersionDate())); + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } catch (JsonParsingException jpe) { + logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", jsonLDBody); + return error(Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage()); + } + } - @PUT - @Path("{id}/versions/{versionId}/metadata") + @DELETE + @Path("{id}/metadata") @Consumes("application/json-ld") - public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, + public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id, @PathParam("versionId") String versionId, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { - if (!":draft".equals(versionId)) { - return error(Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); - } try { Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); DatasetVersion dsv = ds.getEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); - dsv = JSONLDUtil.updateDatasetVersionFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, !replaceTerms, false); + dsv = JSONLDUtil.deleteDatasetVersionMDFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc); DatasetVersion managedVersion; if (updateDraft) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 6dbcde153c8..a65ab2e1455 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -9,7 +9,6 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; -import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand; import edu.harvard.iq.dataverse.export.OAI_OREExporter; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -34,18 +33,13 @@ public class OREMap { public static final String NAME = "OREMap"; - public static final String TRANSFER = "transfer"; - public static final String ARCHIVE = "archive"; - private Map localContext = new TreeMap(); private DatasetVersion version; - private String type; private boolean excludeEmail = false; - - public OREMap(DatasetVersion version, boolean excludeEmail, String type) { + + public OREMap(DatasetVersion version, boolean excludeEmail) { this.version = version; this.excludeEmail = excludeEmail; - this.type=type; } public void writeOREMap(OutputStream outputStream) throws Exception { @@ -54,6 +48,10 @@ public void writeOREMap(OutputStream outputStream) throws Exception { } public JsonObject getOREMap() throws Exception { + return getOREMap(false); + } + + public JsonObject getOREMap(boolean aggregationOnly) throws Exception { // Add namespaces we'll definitely use to Context // Additional namespaces are added as needed below @@ -72,7 +70,7 @@ public JsonObject getOREMap() throws Exception { for (DatasetField field : fields) { if (!field.isEmpty()) { DatasetFieldType dfType = field.getDatasetFieldType(); - if(excludeEmail && DatasetFieldType.FieldType.EMAIL.equals(dfType.getFieldType())) { + if (excludeEmail && DatasetFieldType.FieldType.EMAIL.equals(dfType.getFieldType())) { continue; } JsonLDTerm fieldName = getTermFor(dfType); @@ -94,13 +92,13 @@ public JsonObject getOREMap() throws Exception { for (DatasetField dsf : dscv.getChildDatasetFields()) { DatasetFieldType dsft = dsf.getDatasetFieldType(); - if(excludeEmail && DatasetFieldType.FieldType.EMAIL.equals(dsft.getFieldType())) { + if (excludeEmail && DatasetFieldType.FieldType.EMAIL.equals(dsft.getFieldType())) { continue; } // which may have multiple values if (!dsf.isEmpty()) { - // Add context entry - //ToDo - also needs to recurse here? + // Add context entry + // ToDo - also needs to recurse here? JsonLDTerm subFieldName = getTermFor(dfType, dsft); if (subFieldName.inNamespace()) { localContext.putIfAbsent(subFieldName.getNamespace().getPrefix(), @@ -147,7 +145,7 @@ public JsonObject getOREMap() throws Exception { TermsOfUseAndAccess terms = version.getTermsOfUseAndAccess(); if (terms.getLicense() == TermsOfUseAndAccess.License.CC0) { aggBuilder.add(JsonLDTerm.schemaOrg("license").getLabel(), - "https://creativecommons.org/publicdomain/zero/1.0/"); + TermsOfUseAndAccess.CC0_URI); } else { addIfNotNull(aggBuilder, JsonLDTerm.termsOfUse, terms.getTermsOfUse()); } @@ -180,110 +178,101 @@ public JsonObject getOREMap() throws Exception { // The aggregation aggregates aggregatedresources (Datafiles) which each have // their own entry and metadata JsonArrayBuilder aggResArrayBuilder = Json.createArrayBuilder(); + if (!aggregationOnly) { - for (FileMetadata fmd : version.getFileMetadatas()) { - DataFile df = fmd.getDataFile(); - JsonObjectBuilder aggRes = Json.createObjectBuilder(); + for (FileMetadata fmd : version.getFileMetadatas()) { + DataFile df = fmd.getDataFile(); + JsonObjectBuilder aggRes = Json.createObjectBuilder(); - if (fmd.getDescription() != null) { - aggRes.add(JsonLDTerm.schemaOrg("description").getLabel(), fmd.getDescription()); - } else { - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("description"), df.getDescription()); - } - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("name"), fmd.getLabel()); // "label" is the filename - addIfNotNull(aggRes, JsonLDTerm.restricted, fmd.isRestricted()); - addIfNotNull(aggRes, JsonLDTerm.directoryLabel, fmd.getDirectoryLabel()); - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("version"), fmd.getVersion()); - addIfNotNull(aggRes, JsonLDTerm.datasetVersionId, fmd.getDatasetVersion().getId()); - JsonArray catArray = null; - if (fmd != null) { - List categories = fmd.getCategoriesByName(); - if (categories.size() > 0) { - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (String s : categories) { - jab.add(s); + if (fmd.getDescription() != null) { + aggRes.add(JsonLDTerm.schemaOrg("description").getLabel(), fmd.getDescription()); + } else { + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("description"), df.getDescription()); + } + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("name"), fmd.getLabel()); // "label" is the filename + addIfNotNull(aggRes, JsonLDTerm.restricted, fmd.isRestricted()); + addIfNotNull(aggRes, JsonLDTerm.directoryLabel, fmd.getDirectoryLabel()); + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("version"), fmd.getVersion()); + addIfNotNull(aggRes, JsonLDTerm.datasetVersionId, fmd.getDatasetVersion().getId()); + JsonArray catArray = null; + if (fmd != null) { + List categories = fmd.getCategoriesByName(); + if (categories.size() > 0) { + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (String s : categories) { + jab.add(s); + } + catArray = jab.build(); } - catArray = jab.build(); } + addIfNotNull(aggRes, JsonLDTerm.categories, catArray); + // File DOI if it exists + String fileId = null; + String fileSameAs = null; + if (df.getGlobalId().asString().length() != 0) { + fileId = df.getGlobalId().asString(); + fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + + "/api/access/datafile/:persistentId?persistentId=" + fileId; + } else { + fileId = SystemConfig.getDataverseSiteUrlStatic() + "/file.xhtml?fileId=" + df.getId(); + fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId(); + } + aggRes.add("@id", fileId); + aggRes.add(JsonLDTerm.schemaOrg("sameAs").getLabel(), fileSameAs); + fileArray.add(fileId); + + aggRes.add("@type", JsonLDTerm.ore("AggregatedResource").getLabel()); + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), df.getContentType()); + addIfNotNull(aggRes, JsonLDTerm.filesize, df.getFilesize()); + addIfNotNull(aggRes, JsonLDTerm.storageIdentifier, df.getStorageIdentifier()); + addIfNotNull(aggRes, JsonLDTerm.originalFileFormat, df.getOriginalFileFormat()); + addIfNotNull(aggRes, JsonLDTerm.originalFormatLabel, df.getOriginalFormatLabel()); + addIfNotNull(aggRes, JsonLDTerm.UNF, df.getUnf()); + addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId()); + addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId()); + JsonObject checksum = null; + // Add checksum. RDA recommends SHA-512 + if (df.getChecksumType() != null && df.getChecksumValue() != null) { + checksum = Json.createObjectBuilder().add("@type", df.getChecksumType().toString()) + .add("@value", df.getChecksumValue()).build(); + aggRes.add(JsonLDTerm.checksum.getLabel(), checksum); + } + JsonArray tabTags = null; + JsonArrayBuilder jab = JsonPrinter.getTabularFileTags(df); + if (jab != null) { + tabTags = jab.build(); + } + addIfNotNull(aggRes, JsonLDTerm.tabularTags, tabTags); + // Add latest resource to the array + aggResArrayBuilder.add(aggRes.build()); } - addIfNotNull(aggRes, JsonLDTerm.categories, catArray); - // File DOI if it exists - String fileId = null; - String fileSameAs = null; - if (df.getGlobalId().asString().length() != 0) { - fileId = df.getGlobalId().asString(); - fileSameAs = SystemConfig.getDataverseSiteUrlStatic() - + "/api/access/datafile/:persistentId?persistentId=" + fileId + (type.equals(TRANSFER) ? "&format=original" : ""); - } else { - fileId = SystemConfig.getDataverseSiteUrlStatic() + "/file.xhtml?fileId=" + df.getId(); - fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId() + (type.equals(TRANSFER) ? "?format=original" : ""); - } - aggRes.add("@id", fileId); - aggRes.add(JsonLDTerm.schemaOrg("sameAs").getLabel(), fileSameAs); - fileArray.add(fileId); - - aggRes.add("@type", JsonLDTerm.ore("AggregatedResource").getLabel()); - switch (type) { - case TRANSFER: - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), df.getOriginalFileFormat()); - addIfNotNull(aggRes, JsonLDTerm.filesize, df.getOriginalFileSize()); - addIfNotNull(aggRes, JsonLDTerm.storageIdentifier, df.getStorageIdentifier()); - addIfNotNull(aggRes, JsonLDTerm.UNF, df.getUnf()); - addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId()); - addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId()); - //Checksum not available - will be generated in Bag - - break; - case ARCHIVE: - default: - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), df.getContentType()); - addIfNotNull(aggRes, JsonLDTerm.filesize, df.getFilesize()); - addIfNotNull(aggRes, JsonLDTerm.storageIdentifier, df.getStorageIdentifier()); - addIfNotNull(aggRes, JsonLDTerm.originalFileFormat, df.getOriginalFileFormat()); - addIfNotNull(aggRes, JsonLDTerm.originalFormatLabel, df.getOriginalFormatLabel()); - addIfNotNull(aggRes, JsonLDTerm.UNF, df.getUnf()); - addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId()); - addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId()); - JsonObject checksum = null; - // Add checksum. RDA recommends SHA-512 - if (df.getChecksumType() != null && df.getChecksumValue() != null) { - checksum = Json.createObjectBuilder().add("@type", df.getChecksumType().toString()) - .add("@value", df.getChecksumValue()).build(); - aggRes.add(JsonLDTerm.checksum.getLabel(), checksum); - } - } - JsonArray tabTags = null; - JsonArrayBuilder jab = JsonPrinter.getTabularFileTags(df); - if (jab != null) { - tabTags = jab.build(); - } - addIfNotNull(aggRes, JsonLDTerm.tabularTags, tabTags); - //Add latest resource to the array - aggResArrayBuilder.add(aggRes.build()); } // Build the '@context' object for json-ld based on the localContext entries JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); for (Entry e : localContext.entrySet()) { contextBuilder.add(e.getKey(), e.getValue()); } - // Now create the overall map object with it's metadata - JsonObject oremap = Json.createObjectBuilder() - .add(JsonLDTerm.dcTerms("modified").getLabel(), LocalDate.now().toString()) - .add(JsonLDTerm.dcTerms("creator").getLabel(), - BundleUtil.getStringFromBundle("institution.name")) - .add("@type", JsonLDTerm.ore("ResourceMap").getLabel()) - // Define an id for the map itself (separate from the @id of the dataset being - // described - .add("@id", - SystemConfig.getDataverseSiteUrlStatic() + "/api/datasets/export?exporter=" - + OAI_OREExporter.NAME + "&persistentId=" + id) - // Add the aggregation (Dataset) itself to the map. - .add(JsonLDTerm.ore("describes").getLabel(), - aggBuilder.add(JsonLDTerm.ore("aggregates").getLabel(), aggResArrayBuilder.build()) - .add(JsonLDTerm.schemaOrg("hasPart").getLabel(), fileArray.build()).build()) - // and finally add the context - .add("@context", contextBuilder.build()).build(); - return oremap; + if (aggregationOnly) { + return aggBuilder.add("@context", contextBuilder.build()).build(); + } else { + // Now create the overall map object with it's metadata + JsonObject oremap = Json.createObjectBuilder() + .add(JsonLDTerm.dcTerms("modified").getLabel(), LocalDate.now().toString()) + .add(JsonLDTerm.dcTerms("creator").getLabel(), BundleUtil.getStringFromBundle("institution.name")) + .add("@type", JsonLDTerm.ore("ResourceMap").getLabel()) + // Define an id for the map itself (separate from the @id of the dataset being + // described + .add("@id", + SystemConfig.getDataverseSiteUrlStatic() + "/api/datasets/export?exporter=" + + OAI_OREExporter.NAME + "&persistentId=" + id) + // Add the aggregation (Dataset) itself to the map. + .add(JsonLDTerm.ore("describes").getLabel(), + aggBuilder.add(JsonLDTerm.ore("aggregates").getLabel(), aggResArrayBuilder.build()) + .add(JsonLDTerm.schemaOrg("hasPart").getLabel(), fileArray.build()).build()) + // and finally add the context + .add("@context", contextBuilder.build()).build(); + return oremap; + } } /* diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 4f451f01ca5..822a5f4008d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -75,7 +75,7 @@ public static JsonObject getContext(Map contextMap) { return contextBuilder.build(); } - public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, + public static Dataset updateDatasetMDFromJsonLD(Dataset ds, String jsonLDBody, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { DatasetVersion dsv = new DatasetVersion(); @@ -90,7 +90,7 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, + "'. Make sure it is in valid form - see Dataverse Native API documentation."); } - dsv = updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); + dsv = updateDatasetVersionMDFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); dsv.setDataset(ds); List versions = new ArrayList<>(1); @@ -111,10 +111,10 @@ public static Dataset updateDatasetFromJsonLD(Dataset ds, String jsonLDBody, return ds; } - public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, String jsonLDBody, + public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv, String jsonLDBody, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); - return updateDatasetVersionFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); + return updateDatasetVersionMDFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); } /** @@ -129,7 +129,7 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, * value(s) for fields found in the json-ld. * @return */ - public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, JsonObject jsonld, + public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv, JsonObject jsonld, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { populateFieldTypeMap(metadataBlockSvc); @@ -294,6 +294,104 @@ public static DatasetVersion updateDatasetVersionFromJsonLD(DatasetVersion dsv, return dsv; } + /** + * + * @param dsv + * @param jsonLDBody + * @param metadataBlockService + * @param datasetFieldSvc + * @param b + * @param c + * @return + */ + public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv, String jsonLDBody, + MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { + + JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); + //All terms are now URIs + //Setup dsftMap - URI to datasetFieldType map + populateFieldTypeMap(metadataBlockSvc); + + + //Another map - from datasetFieldType to an existing field in the dataset + List dsfl = dsv.getDatasetFields(); + Map fieldByTypeMap = new HashMap(); + for (DatasetField dsf : dsfl) { + if (fieldByTypeMap.containsKey(dsf.getDatasetFieldType())) { + // May have multiple values per field, but not multiple fields of one type? + logger.warning("Multiple fields of type " + dsf.getDatasetFieldType().getName()); + } + fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); + } + + TermsOfUseAndAccess terms = dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess(); + + //Iterate through input json + for (String key : jsonld.keySet()) { + //Skip context (shouldn't be present with decontextualize used above) + if (!key.equals("@context")) { + if (dsftMap.containsKey(key)) { + //THere's a field type with theis URI + DatasetFieldType dsft = dsftMap.get(key); + DatasetField dsf = null; + if (fieldByTypeMap.containsKey(dsft)) { + //There's a field of this type + dsf = fieldByTypeMap.get(dsft); + + // Todo - normalize object vs. array + JsonArray valArray = getValues(jsonld.get(key), dsft.isAllowMultiples(), dsft.getName()); + + DatasetField dsf2 = getReplacementField(dsf, valArray); + if(dsf2 == null) { + //Exact match - remove the field + dsfl.remove(dsf); + } else if(!dsf2.equals(dsf)) { + //Partial match - some values of a multivalue field match, so keep the remaining values + dsfl.remove(dsf); + dsfl.add(dsf2); + } + } + } else { + // Internal/non-metadatablock terms + if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())) { + if(jsonld.getString(key).equals(TermsOfUseAndAccess.CC0_URI)) { + setSemTerm(terms, key, TermsOfUseAndAccess.License.NONE); + } else { + throw new BadRequestException( + "Term: " + key + " with value: " + jsonld.getString(key) + " not found."); + } + } else if (datasetTerms.contains(key)) { + if(!deleteIfSemTermMatches(terms, key, jsonld.get(key))) { + throw new BadRequestException( + "Term: " + key + " with value: " + jsonld.getString(key) + " not found."); + } + } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { + JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); + for (String fileKey : fAccessObject.keySet()) { + if (datafileTerms.contains(fileKey)) { + if(!deleteIfSemTermMatches(terms, key, jsonld.get(key))) { + throw new BadRequestException( + "Term: " + key + " with value: " + jsonld.getString(key) + " not found."); + } + } + } + } else { + throw new BadRequestException( + "Term: " + key + " not found."); + } + + dsv.setTermsOfUseAndAccess(terms); + } + } + } + dsv.setDatasetFields(dsfl); + return dsv; + } + + private static DatasetField getReplacementField(DatasetField dsf, JsonArray valArray) { + // TODO Auto-generated method stub + return null; + } private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldType dsft, DatasetFieldServiceBean datasetFieldSvc, boolean append) { @@ -449,7 +547,7 @@ private static void populateFieldTypeMap(MetadataBlockServiceBean metadataBlockS } } - private static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { + public static void populateContext(MetadataBlockServiceBean metadataBlockSvc) { if (localContext.isEmpty()) { List mdbList = metadataBlockSvc.listMetadataBlocks(); @@ -673,7 +771,7 @@ public static void setSemTerm(TermsOfUseAndAccess terms, String semterm, Object terms.setLicense(TermsOfUseAndAccess.defaultLicense); } else { throw new BadRequestException("The only allowed value for " + JsonLDTerm.schemaOrg("license").getUrl() - + " is " + "https://creativecommons.org/publicdomain/zero/1.0/"); + + " is " + TermsOfUseAndAccess.CC0_URI); } break; case "https://dataverse.org/schema/core#termsOfUse": @@ -729,4 +827,112 @@ public static void setSemTerm(TermsOfUseAndAccess terms, String semterm, Object break; } } + + private static boolean deleteIfSemTermMatches(TermsOfUseAndAccess terms, String semterm, JsonValue jsonValue) { + boolean foundTerm=false; + switch (semterm) { + + case "https://dataverse.org/schema/core#termsOfUse": + if(terms.getTermsOfUse().equals(jsonValue.toString())) { + terms.setTermsOfAccess(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#confidentialityDeclaration": + if(terms.getConfidentialityDeclaration().equals(jsonValue.toString())) { + terms.setConfidentialityDeclaration(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#specialPermissions": + if(terms.getSpecialPermissions().equals(jsonValue.toString())) { + terms.setSpecialPermissions(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#restrictions": + if(terms.getRestrictions().equals(jsonValue.toString())) { + terms.setRestrictions(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#citationRequirements": + if(terms.getCitationRequirements().equals(jsonValue.toString())) { + terms.setCitationRequirements(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#depositorRequirements": + if(terms.getDepositorRequirements().equals(jsonValue.toString())) { + terms.setDepositorRequirements(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#conditions": + if(terms.getConditions().equals(jsonValue.toString())) { + terms.setConditions(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#disclaimer": + if(terms.getDisclaimer().equals(jsonValue.toString())) { + terms.setDisclaimer(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#termsOfAccess": + if(terms.getTermsOfAccess().equals(jsonValue.toString())) { + terms.setTermsOfAccess(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#fileRequestAccess": + if(terms.isFileAccessRequest() && (jsonValue.equals(JsonValue.TRUE))) { + terms.setFileAccessRequest(false); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#dataAccessPlace": + if(terms.getDataAccessPlace().equals(jsonValue.toString())) { + terms.setDataAccessPlace(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#originalArchive": + if(terms.getOriginalArchive().equals(jsonValue.toString())) { + terms.setOriginalArchive(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#availabilityStatus": + if(terms.getAvailabilityStatus().equals(jsonValue.toString())) { + terms.setAvailabilityStatus(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#contactForAccess": + if(terms.getContactForAccess().equals(jsonValue.toString())) { + terms.setContactForAccess(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#sizeOfCollection": + if(terms.getSizeOfCollection().equals(jsonValue.toString())) { + terms.setSizeOfCollection(null); + foundTerm=true; + } + break; + case "https://dataverse.org/schema/core#studyCompletion": + if(terms.getStudyCompletion().equals(jsonValue.toString())) { + terms.setStudyCompletion(null); + foundTerm=true; + } + break; + default: + logger.warning("deleteIfSemTermMatches called for " + semterm); + break; + } + return foundTerm; + } + } From 849f6f8913d3148eb0a46ad82e2a115286871027 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 8 Dec 2020 12:58:16 -0500 Subject: [PATCH 087/354] 7130-handle missing contact name --- .../iq/dataverse/util/bagit/BagGenerator.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java index a939a14535c..a4152e4b627 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java @@ -765,8 +765,10 @@ private String generateInfoFile() { info.append(CRLF); } else { - info.append(((JsonObject) person).get(contactNameTerm.getLabel()).getAsString()); - info.append(CRLF); + if(contactNameTerm != null) { + info.append(((JsonObject) person).get(contactNameTerm.getLabel()).getAsString()); + info.append(CRLF); + } if ((contactEmailTerm!=null) &&((JsonObject) person).has(contactEmailTerm.getLabel())) { info.append("Contact-Email: "); info.append(((JsonObject) person).get(contactEmailTerm.getLabel()).getAsString()); @@ -783,9 +785,10 @@ private String generateInfoFile() { } else { JsonObject person = contacts.getAsJsonObject(); - - info.append(person.get(contactNameTerm.getLabel()).getAsString()); - info.append(CRLF); + if(contactNameTerm != null) { + info.append(person.get(contactNameTerm.getLabel()).getAsString()); + info.append(CRLF); + } if ((contactEmailTerm!=null) && (person.has(contactEmailTerm.getLabel()))) { info.append("Contact-Email: "); info.append(person.get(contactEmailTerm.getLabel()).getAsString()); From 8321f62ae12599cc14baaf382f284dccf9ad43ce Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 13:16:08 -0500 Subject: [PATCH 088/354] Fix multiple description logic for info file --- .../iq/dataverse/util/bagit/BagGenerator.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java index a4152e4b627..d3097a84fc2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java @@ -826,7 +826,7 @@ private String generateInfoFile() { } else { info.append( // FixMe - handle description having subfields better - WordUtils.wrap(getSingleValue(aggregation.getAsJsonObject(descriptionTerm.getLabel()), + WordUtils.wrap(getSingleValue(aggregation.get(descriptionTerm.getLabel()), descriptionTextTerm.getLabel()), 78, CRLF + " ", true)); info.append(CRLF); @@ -872,22 +872,24 @@ private String generateInfoFile() { * - the key to find a value(s) for * @return - a single string */ - String getSingleValue(JsonObject jsonObject, String key) { + String getSingleValue(JsonElement jsonElement, String key) { String val = ""; - if (jsonObject.get(key).isJsonPrimitive()) { + if(jsonElement.isJsonObject()) { + JsonObject jsonObject=jsonElement.getAsJsonObject(); val = jsonObject.get(key).getAsString(); - } else if (jsonObject.get(key).isJsonArray()) { - Iterator iter = jsonObject.getAsJsonArray(key).iterator(); + } else if (jsonElement.isJsonArray()) { + + Iterator iter = jsonElement.getAsJsonArray().iterator(); ArrayList stringArray = new ArrayList(); while (iter.hasNext()) { - stringArray.add(iter.next().getAsString()); + stringArray.add(iter.next().getAsJsonObject().getAsJsonPrimitive(key).getAsString()); } if (stringArray.size() > 1) { val = StringUtils.join((String[]) stringArray.toArray(), ","); } else { val = stringArray.get(0); } - logger.warning("Multiple values found for: " + key + ": " + val); + logger.fine("Multiple values found for: " + key + ": " + val); } return val; } From 5050e9390ff8fe4679ffb58a3f3bb89106b7f8d0 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 13:44:51 -0500 Subject: [PATCH 089/354] put is always for :draft version --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 6ac132e771b..049b5ece82c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -645,12 +645,8 @@ public Response getVersionMetadata(@PathParam("id") String id) { @PUT @Path("{id}/metadata") @Consumes("application/json-ld") - public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, - @PathParam("versionId") String versionId, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { + public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { - if (!":draft".equals(versionId)) { - return error(Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); - } try { Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); From 11292c07be3f974e4143aacfd202a25af5bfc9a6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 15:17:22 -0500 Subject: [PATCH 090/354] don't cast to String[] --- .../java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java index d3097a84fc2..3eccafc634c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java @@ -885,7 +885,7 @@ String getSingleValue(JsonElement jsonElement, String key) { stringArray.add(iter.next().getAsJsonObject().getAsJsonPrimitive(key).getAsString()); } if (stringArray.size() > 1) { - val = StringUtils.join((String[]) stringArray.toArray(), ","); + val = StringUtils.join(stringArray.toArray(), ","); } else { val = stringArray.get(0); } From cbbb084a35d284e42327c6d6ae0f8e2da8df64c1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 15:17:52 -0500 Subject: [PATCH 091/354] add more logging --- .../edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java b/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java index 881d6c79872..76f08ecc03d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java @@ -227,6 +227,7 @@ public JsonResponseBuilder log(Logger logger, Level level, Optional e if (ex.isPresent()) { metadata.append("|"); logger.log(level, metadata.toString(), ex); + ex.get().printStackTrace(); } else { logger.log(level, metadata.toString()); } From fd14226fa137f6a3afb85f5be6ddcc1cca7e167f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 17:27:05 -0500 Subject: [PATCH 092/354] add method that can return JsonObjectBuilder which can be used with existing AbstractApiBean.ok() --- .../harvard/iq/dataverse/util/bagit/OREMap.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index a65ab2e1455..98e7229154c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -50,8 +50,12 @@ public void writeOREMap(OutputStream outputStream) throws Exception { public JsonObject getOREMap() throws Exception { return getOREMap(false); } - + public JsonObject getOREMap(boolean aggregationOnly) throws Exception { + return getOREMapBuilder(aggregationOnly).build(); + } + + public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) throws Exception { // Add namespaces we'll definitely use to Context // Additional namespaces are added as needed below @@ -253,10 +257,10 @@ public JsonObject getOREMap(boolean aggregationOnly) throws Exception { contextBuilder.add(e.getKey(), e.getValue()); } if (aggregationOnly) { - return aggBuilder.add("@context", contextBuilder.build()).build(); + return aggBuilder.add("@context", contextBuilder.build()); } else { // Now create the overall map object with it's metadata - JsonObject oremap = Json.createObjectBuilder() + JsonObjectBuilder oremapBuilder = Json.createObjectBuilder() .add(JsonLDTerm.dcTerms("modified").getLabel(), LocalDate.now().toString()) .add(JsonLDTerm.dcTerms("creator").getLabel(), BundleUtil.getStringFromBundle("institution.name")) .add("@type", JsonLDTerm.ore("ResourceMap").getLabel()) @@ -270,8 +274,8 @@ public JsonObject getOREMap(boolean aggregationOnly) throws Exception { aggBuilder.add(JsonLDTerm.ore("aggregates").getLabel(), aggResArrayBuilder.build()) .add(JsonLDTerm.schemaOrg("hasPart").getLabel(), fileArray.build()).build()) // and finally add the context - .add("@context", contextBuilder.build()).build(); - return oremap; + .add("@context", contextBuilder.build()); + return oremapBuilder; } } From 7fe02495015e4cd65cb8a6d0e3d280861466bbee Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 17:27:24 -0500 Subject: [PATCH 093/354] log details on failure --- .../engine/command/impl/LocalSubmitToArchiveCommand.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java index 091d5f364de..23a98b9a788 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java @@ -84,6 +84,7 @@ public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken t } } catch (Exception e) { logger.warning(e.getLocalizedMessage() + "here"); + e.printStackTrace(); } return WorkflowStepResult.OK; } From 24cfcfa08257d9245b2d6f033c679e59c4424316 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 11 Dec 2020 17:27:55 -0500 Subject: [PATCH 094/354] multiple updates/fixes, added logging --- .../harvard/iq/dataverse/api/Datasets.java | 28 ++++++++++--------- .../iq/dataverse/util/json/JSONLDUtil.java | 13 +++++++-- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 049b5ece82c..8b9118741b3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -616,21 +616,22 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, } @GET - @Path("{id}/versions/{versionid}/metadata") + @Path("{id}/versions/{versionId}/metadata") @Produces("application/json-ld") - public Response getVersionMetadata(@PathParam("id") String id, @PathParam("versionId") String versionId) { + public Response getVersionJsonLDMetadata(@PathParam("id") String id, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) { try { - Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); - DatasetVersion dsv = ds.getEditVersion(); + DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(id), uriInfo, headers); OREMap ore = new OREMap(dsv, settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)); - return ok(JSONLDUtil.prettyPrint(ore.getOREMap(true))); + return ok(ore.getOREMapBuilder(true)); } catch (WrappedResponse ex) { + ex.printStackTrace(); return ex.getResponse(); } catch (Exception jpe) { logger.log(Level.SEVERE, "Error getting jsonld metadata for dsv: ", jpe.getLocalizedMessage()); + jpe.printStackTrace(); return error(Response.Status.INTERNAL_SERVER_ERROR, jpe.getLocalizedMessage()); } } @@ -638,8 +639,8 @@ public Response getVersionMetadata(@PathParam("id") String id, @PathParam("versi @GET @Path("{id}/metadata") @Produces("application/json-ld") - public Response getVersionMetadata(@PathParam("id") String id) { - return getVersionMetadata(id, ":draft"); + public Response getVersionJsonLDMetadata(@PathParam("id") String id, @Context UriInfo uriInfo, @Context HttpHeaders headers) { + return getVersionJsonLDMetadata(id, ":draft", uriInfo, headers); } @PUT @@ -672,19 +673,18 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String } } - @DELETE - @Path("{id}/metadata") + @PUT + @Path("{id}/metadata/delete") @Consumes("application/json-ld") - public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id, - @PathParam("versionId") String versionId, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { - + public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id) { + logger.info("In delteMetadata"); try { Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); DatasetVersion dsv = ds.getEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); dsv = JSONLDUtil.deleteDatasetVersionMDFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc); - + logger.info("Updating ver"); DatasetVersion managedVersion; if (updateDraft) { Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); @@ -696,9 +696,11 @@ public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id, return ok(Json.createObjectBuilder().add(info, managedVersion.getVersionDate())); } catch (WrappedResponse ex) { + ex.printStackTrace(); return ex.getResponse(); } catch (JsonParsingException jpe) { logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", jsonLDBody); + jpe.printStackTrace(); return error(Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage()); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 822a5f4008d..a8b12a85828 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -158,6 +158,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv // unless it's multiple if (!append && !dsft.isAllowMultiples()) { dsfl.remove(dsf); + dsf=null; } } if (dsf == null) { @@ -306,7 +307,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv */ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv, String jsonLDBody, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { - +logger.info("deleteDatasetVersionMD"); JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); //All terms are now URIs //Setup dsftMap - URI to datasetFieldType map @@ -340,12 +341,12 @@ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv // Todo - normalize object vs. array JsonArray valArray = getValues(jsonld.get(key), dsft.isAllowMultiples(), dsft.getName()); - +logger.info("Deleting: " + key + " : " + valArray.toString()); DatasetField dsf2 = getReplacementField(dsf, valArray); if(dsf2 == null) { //Exact match - remove the field dsfl.remove(dsf); - } else if(!dsf2.equals(dsf)) { + } else { //Partial match - some values of a multivalue field match, so keep the remaining values dsfl.remove(dsf); dsfl.add(dsf2); @@ -388,6 +389,12 @@ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv return dsv; } + /** + * + * @param dsf + * @param valArray + * @return null if exact match, otherwise return a field without the value to be deleted + */ private static DatasetField getReplacementField(DatasetField dsf, JsonArray valArray) { // TODO Auto-generated method stub return null; From a01c4211d7b35ded94593f49c387650bdb1678ea Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 22 Dec 2020 14:36:04 -0500 Subject: [PATCH 095/354] merge issues --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 4 ++-- .../command/impl/AbstractSubmitToArchiveCommand.java | 8 +------- .../command/impl/DuraCloudSubmitToArchiveCommand.java | 6 +++--- .../engine/command/impl/LocalSubmitToArchiveCommand.java | 6 +++--- .../edu/harvard/iq/dataverse/export/OAI_OREExporter.java | 2 +- .../java/edu/harvard/iq/dataverse/util/ArchiverUtil.java | 9 ++------- .../harvard/iq/dataverse/util/bagit/BagGenerator.java | 3 +-- 7 files changed, 13 insertions(+), 25 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index dc44cb474ff..b52665a7747 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1678,7 +1678,7 @@ public Response validateDataFileHashValue(@PathParam("fileId") String fileId) { @GET @Path("/submitDataVersionToArchive/{id}/{version}") - public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, @PathParam("version") String versionNumber, @QueryParam(value = "type") String type) { + public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, @PathParam("version") String versionNumber) { try { AuthenticatedUser au = findAuthenticatedUserOrDie(); @@ -1692,7 +1692,7 @@ public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, @Pat DatasetVersion dv = datasetversionService.findByFriendlyVersionNumber(ds.getId(), versionNumber); if (dv.getArchivalCopyLocation() == null) { String className = settingsService.getValueForKey(SettingsServiceBean.Key.ArchiverClassName); - AbstractSubmitToArchiveCommand cmd = ArchiverUtil.createSubmitToArchiveCommand(className, type, dvRequestService.getDataverseRequest(), dv); + AbstractSubmitToArchiveCommand cmd = ArchiverUtil.createSubmitToArchiveCommand(className, dvRequestService.getDataverseRequest(), dv); if (cmd != null) { new Thread(new Runnable() { public void run() { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java index 75bd6b50149..77ea680598f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java @@ -22,14 +22,12 @@ public abstract class AbstractSubmitToArchiveCommand extends AbstractCommand { private final DatasetVersion version; - private final String type; private final Map requestedSettings = new HashMap(); private static final Logger logger = Logger.getLogger(AbstractSubmitToArchiveCommand.class.getName()); - public AbstractSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version, String type) { + public AbstractSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { super(aRequest, version.getDataset()); this.version = version; - this.type = type; } @Override @@ -75,8 +73,4 @@ public String describe() { + version.getFriendlyVersionNumber()+")]"; } - public String getType() { - return type; - } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java index b40ff4f7db4..468e99f24c1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java @@ -42,8 +42,8 @@ public class DuraCloudSubmitToArchiveCommand extends AbstractSubmitToArchiveComm private static final String DURACLOUD_HOST = ":DuraCloudHost"; private static final String DURACLOUD_CONTEXT = ":DuraCloudContext"; - public DuraCloudSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version, String type) { - super(aRequest, version, type); + public DuraCloudSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { + super(aRequest, version); } @Override @@ -127,7 +127,7 @@ public void run() { public void run() { try (PipedOutputStream out = new PipedOutputStream(in)){ // Generate bag - BagGenerator bagger = new BagGenerator(new OREMap(dv, false, getType()), dataciteXml, getType()); + BagGenerator bagger = new BagGenerator(new OREMap(dv, false), dataciteXml); bagger.setAuthenticationKey(token.getTokenString()); bagger.generateBag(out); } catch (Exception e) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java index 23a98b9a788..d87c3011c15 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java @@ -42,8 +42,8 @@ public class LocalSubmitToArchiveCommand extends AbstractSubmitToArchiveCommand private static final Logger logger = Logger.getLogger(LocalSubmitToArchiveCommand.class.getName()); - public LocalSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version, String type) { - super(aRequest, version, type); + public LocalSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { + super(aRequest, version); } @Override @@ -69,7 +69,7 @@ public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken t FileUtils.writeStringToFile(new File(localPath+"/"+spaceName + "-datacite.v" + dv.getFriendlyVersionNumber()+".xml"), dataciteXml); - BagGenerator bagger = new BagGenerator(new OREMap(dv, false, getType()), dataciteXml, getType()); + BagGenerator bagger = new BagGenerator(new OREMap(dv, false), dataciteXml); bagger.setAuthenticationKey(token.getTokenString()); bagger.generateBag(new FileOutputStream(localPath+"/"+spaceName + "v" + dv.getFriendlyVersionNumber() + ".zip")); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java index 80eea321fa5..6cfcb590681 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/OAI_OREExporter.java @@ -25,7 +25,7 @@ public class OAI_OREExporter implements Exporter { public void exportDataset(DatasetVersion version, JsonObject json, OutputStream outputStream) throws ExportException { try { - new OREMap(version, ExportService.settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false), OREMap.ARCHIVE).writeOREMap(outputStream); + new OREMap(version, ExportService.settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false)).writeOREMap(outputStream); } catch (Exception e) { logger.severe(e.getMessage()); e.printStackTrace(); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java index 72e5fd57593..ff82456d990 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java @@ -20,18 +20,13 @@ public ArchiverUtil() { } public static AbstractSubmitToArchiveCommand createSubmitToArchiveCommand(String className, DataverseRequest dvr, DatasetVersion version) { - return createSubmitToArchiveCommand(className, null, dvr, version); - } - - public static AbstractSubmitToArchiveCommand createSubmitToArchiveCommand(String className, String type, - DataverseRequest dvr, DatasetVersion version) { if (className != null) { try { Class clazz = Class.forName(className); if (AbstractSubmitToArchiveCommand.class.isAssignableFrom(clazz)) { Constructor ctor; - ctor = clazz.getConstructor(DataverseRequest.class, DatasetVersion.class, String.class); - return (AbstractSubmitToArchiveCommand) ctor.newInstance(new Object[] { dvr, version, type }); + ctor = clazz.getConstructor(DataverseRequest.class, DatasetVersion.class); + return (AbstractSubmitToArchiveCommand) ctor.newInstance(new Object[] { dvr, version }); } } catch (Exception e) { logger.warning("Unable to instantiate an Archiver of class: " + className); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java index 3eccafc634c..445883e846e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java @@ -136,12 +136,11 @@ public class BagGenerator { * and zipping are done in parallel, using a connection pool. The required space * on disk is ~ n+1/n of the final bag size, e.g. 125% of the bag size for a * 4-way parallel zip operation. - * @param type * @throws Exception * @throws JsonSyntaxException */ - public BagGenerator(OREMap oreMap, String dataciteXml, String type) throws JsonSyntaxException, Exception { + public BagGenerator(OREMap oreMap, String dataciteXml) throws JsonSyntaxException, Exception { this.oremap = oreMap; this.oremapObject = oreMap.getOREMap(); //(JsonObject) new JsonParser().parse(oreMap.getOREMap().toString()); From 6ce3a1c443d292e61b718115fdf6ce29cd34cb55 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 22 Dec 2020 15:01:43 -0500 Subject: [PATCH 096/354] fix terms retrieval --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index a8b12a85828..72e6a3b21ee 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -218,7 +218,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv } else if (datasetTerms.contains(key)) { if (!append || !isSet(terms, key)) { // Other Dataset-level TermsOfUseAndAccess - setSemTerm(terms, key, jsonld.getString(JsonLDTerm.termsOfUse.getUrl())); + setSemTerm(terms, key, jsonld.getString(key)); } else { throw new BadRequestException( "Can't append to a single-value field that already has a value: " + key); From 0bde4e8b938555edaddc5157be326589693a15a1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 22 Dec 2020 15:01:59 -0500 Subject: [PATCH 097/354] merge issue --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 9a890771179..56f08bad167 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -412,7 +412,7 @@ public Response recreateDataset(String jsonLDBody, @PathParam("identifier") Stri Dataset ds = new Dataset(); ds.setOwner(owner); - ds = JSONLDUtil.updateDatasetFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc, false); + ds = JSONLDUtil.updateDatasetMDFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc, false, true); //ToDo - verify PID is one Dataverse can manage (protocol/authority/shoulder match) if(! (ds.getAuthority().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Authority))&& From e93114940d5ed79cbfa138fcf5262b90d36b83ef Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 22 Dec 2020 15:01:43 -0500 Subject: [PATCH 098/354] fix terms retrieval --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index a8b12a85828..72e6a3b21ee 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -218,7 +218,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv } else if (datasetTerms.contains(key)) { if (!append || !isSet(terms, key)) { // Other Dataset-level TermsOfUseAndAccess - setSemTerm(terms, key, jsonld.getString(JsonLDTerm.termsOfUse.getUrl())); + setSemTerm(terms, key, jsonld.getString(key)); } else { throw new BadRequestException( "Can't append to a single-value field that already has a value: " + key); From e6366e4e29166365265ac947287d187c51f848cd Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 8 Jan 2021 09:55:52 -0500 Subject: [PATCH 099/354] add note --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 8b9118741b3..8e7b48bdec2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -652,6 +652,7 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); DatasetVersion dsv = ds.getEditVersion(); + //FIX ME - always true boolean updateDraft = ds.getLatestVersion().isDraft(); dsv = JSONLDUtil.updateDatasetVersionMDFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, !replaceTerms, false); From e8f737c7ce334643b75139770248780611de54bf Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 13 Jan 2021 17:29:12 -0500 Subject: [PATCH 100/354] date test fixes for locale --- .../edu/harvard/iq/dataverse/FileMetadataTest.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/FileMetadataTest.java b/src/test/java/edu/harvard/iq/dataverse/FileMetadataTest.java index e9e9ea4a051..bfbb25b3612 100644 --- a/src/test/java/edu/harvard/iq/dataverse/FileMetadataTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/FileMetadataTest.java @@ -1,10 +1,17 @@ package edu.harvard.iq.dataverse; import java.sql.Timestamp; +import java.text.DateFormat; +import java.time.Instant; +import java.time.LocalDate; +import java.time.Month; +import java.util.Date; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import edu.harvard.iq.dataverse.util.BundleUtil; + import static org.junit.jupiter.api.Assertions.assertEquals; public class FileMetadataTest { @@ -35,7 +42,7 @@ void testGetFileDateToDisplay_fileWithoutPublicationDate() { datafile.setPublicationDate(null); datafile.setCreateDate(Timestamp.valueOf("2019-01-01 00:00:00")); metadata.setDataFile(datafile); - assertEquals("Jan 1, 2019", metadata.getFileDateToDisplay()); + assertEquals(DateFormat.getDateInstance(DateFormat.DEFAULT, BundleUtil.getCurrentLocale()).format(Date.from(Instant.parse("2019-01-01T00:00:00.00Z"))), metadata.getFileDateToDisplay()); } @Test @@ -44,6 +51,6 @@ void testGetFileDateToDisplay_fileWithPublicationDate() { datafile.setPublicationDate(Timestamp.valueOf("2019-02-02 00:00:00")); datafile.setCreateDate(Timestamp.valueOf("2019-01-02 00:00:00")); metadata.setDataFile(datafile); - assertEquals("Feb 2, 2019", metadata.getFileDateToDisplay()); + assertEquals(DateFormat.getDateInstance(DateFormat.DEFAULT, BundleUtil.getCurrentLocale()).format(Date.from(Instant.parse("2019-02-02T00:00:00.00Z"))), metadata.getFileDateToDisplay()); } } From 1c93260df74dee477162dc24318412974dad8329 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 13 Jan 2021 17:29:49 -0500 Subject: [PATCH 101/354] Java 11 update and test fixes inc. for different exception mesg --- pom.xml | 27 +++---------------- .../dataverse/dataaccess/StorageIOTest.java | 16 ++++++----- 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/pom.xml b/pom.xml index 2bba4fec3b0..400354bf405 100644 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,7 @@ Jacoco 0.8.2 seems to break Netbeans code coverage integration so we'll use 0.8.1 instead. See https://github.com/jacoco/jacoco/issues/772 for discussion of how the XML changed. --> - 0.8.1 + 0.8.6 @@ -815,28 +815,7 @@ all-unit-tests - - - Java8 - - 1.8 - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.0 - - 1.8 - 1.8 - - ${compilerArgument} - - - - - + Java9Plus @@ -849,7 +828,7 @@ maven-compiler-plugin 3.8.0 - 8 + 11 ${compilerArgument} diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java index 513d4d90988..83cb0c72786 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java @@ -62,24 +62,28 @@ public void testGetReadChannel() throws Exception { @Test public void testGetDvObject() { assertEquals(null, instance.getDvObject()); - instance.setDvObject(new Dataset()); - assertEquals(new Dataset(), instance.getDataset()); + Dataset d= new Dataset(); + instance.setDvObject(d); + //assertSame uses == rather than the .equals() method which would (currently) be true for any two Datasets + assertSame(d, instance.getDataset()); try { instance.getDataFile(); fail("This should have thrown"); } catch (ClassCastException ex) { - assertEquals(ex.getMessage(), "edu.harvard.iq.dataverse.Dataset cannot be cast to edu.harvard.iq.dataverse.DataFile"); + //Test succeeds } try { instance.getDataverse(); fail("This should have thrown"); } catch (ClassCastException ex) { - assertEquals(ex.getMessage(), "edu.harvard.iq.dataverse.Dataset cannot be cast to edu.harvard.iq.dataverse.Dataverse"); + //Test succeeds } // null driver defaults to 'file' - assertEquals(new DataFile(), new FileAccessIO<>(new DataFile(), null, null).getDataFile()); - assertEquals(new Dataverse(), new FileAccessIO<>(new Dataverse(), null, null).getDataverse()); + DataFile f= new DataFile(); + Dataverse dv = new Dataverse(); + assertSame(f, new FileAccessIO<>(f, null, null).getDataFile()); + assertSame(dv, new FileAccessIO<>(dv, null, null).getDataverse()); } @Test From a85c1d68b3873ef00a96ecc5681d301d77190760 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 14 Jan 2021 13:36:03 -0500 Subject: [PATCH 102/354] update pom for v11 and running tests under 11 --- pom.xml | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/pom.xml b/pom.xml index 49029626431..7f05c3523e6 100644 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,7 @@ Jacoco 0.8.2 seems to break Netbeans code coverage integration so we'll use 0.8.1 instead. See https://github.com/jacoco/jacoco/issues/772 for discussion of how the XML changed. --> - 0.8.1 + 0.8.6 @@ -821,27 +821,6 @@ all-unit-tests - - Java8 - - 1.8 - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.0 - - 1.8 - 1.8 - - ${compilerArgument} - - - - - Java9Plus @@ -854,7 +833,7 @@ maven-compiler-plugin 3.8.0 - 8 + 11 ${compilerArgument} From a52353b8461d4d191f7e92561b3771411e895ff3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 14 Jan 2021 15:36:48 -0500 Subject: [PATCH 103/354] fix for edu.harvard.iq.dataverse.api.AdminIT test fail in Java 11 The DV code tested in testLoadMetadataBlock_ErrorHandling assumed it could parse the message of an ArrayOutOfBounds exception as an it to determine the column that fails. This message is now a String. Rather than parse it (and fail if it changes), I modified the code so that the length of the values array is visible in the catch and can be sent directly (the first out of bounds index is if/when the index is values.length). --- .../iq/dataverse/api/DatasetFieldServiceApi.java | 10 +++++----- .../iq/dataverse/api/DatasetFieldServiceApiTest.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java index c945ba4df56..8b300e6f308 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java @@ -252,11 +252,12 @@ public Response loadDatasetFields(File file) { int lineNumber = 0; HeaderType header = null; JsonArrayBuilder responseArr = Json.createArrayBuilder(); + String[] values = null; try { br = new BufferedReader(new FileReader("/" + file)); while ((line = br.readLine()) != null) { lineNumber++; - String[] values = line.split(splitBy); + values = line.split(splitBy); if (values[0].startsWith("#")) { // Header row switch (values[0]) { case "#metadataBlock": @@ -303,7 +304,7 @@ public Response loadDatasetFields(File file) { return error(Status.EXPECTATION_FAILED, "File not found"); } catch (ArrayIndexOutOfBoundsException e) { - String message = getArrayIndexOutOfBoundMessage(header, lineNumber, e); + String message = getArrayIndexOutOfBoundMessage(header, lineNumber, values.length); logger.log(Level.WARNING, message, e); alr.setActionResult(ActionLogRecord.Result.InternalError); alr.setInfo(alr.getInfo() + "// " + message); @@ -354,11 +355,10 @@ public String getGeneralErrorMessage(HeaderType header, int lineNumber, String m */ public String getArrayIndexOutOfBoundMessage(HeaderType header, int lineNumber, - ArrayIndexOutOfBoundsException e) { + int wrongIndex) { List columns = getColumnsByHeader(header); - int wrongIndex = Integer.parseInt(e.getMessage()); - + String column = columns.get(wrongIndex - 1); List arguments = new ArrayList<>(); arguments.add(header.name()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApiTest.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApiTest.java index d6257063bd2..559e5a7dfba 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApiTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApiTest.java @@ -43,7 +43,7 @@ public void testGeneralErrorMessageBundle() { @Test public void testGetArrayIndexOutOfBoundMessage() { DatasetFieldServiceApi api = new DatasetFieldServiceApi(); - String message = api.getArrayIndexOutOfBoundMessage(DatasetFieldServiceApi.HeaderType.DATASETFIELD, 5, new ArrayIndexOutOfBoundsException("4")); + String message = api.getArrayIndexOutOfBoundMessage(DatasetFieldServiceApi.HeaderType.DATASETFIELD, 5, 4); assertEquals( "Error parsing metadata block in DATASETFIELD part, line #5: missing 'watermark' column (#5)", message From b91ad4c277fe7727308d0954f5c5656bda56c95a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 29 Jan 2021 11:28:18 -0500 Subject: [PATCH 104/354] missed merge issue --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index d48ac22b12f..d039cd09dcb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -61,6 +61,8 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.StringUtil; import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty; + +import edu.harvard.iq.dataverse.util.json.JSONLDUtil; import edu.harvard.iq.dataverse.util.json.JsonParseException; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.brief; import java.io.StringReader; @@ -84,6 +86,7 @@ import javax.json.stream.JsonParsingException; import javax.validation.ConstraintViolation; import javax.validation.ConstraintViolationException; +import javax.ws.rs.BadRequestException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -393,7 +396,6 @@ public Response importDatasetDdi(String xml, @PathParam("identifier") String par } } -<<<<<<< HEAD @POST @Path("{identifier}/datasets/:startmigration") @Consumes("application/json-ld") @@ -447,8 +449,6 @@ public Response recreateDataset(String jsonLDBody, @PathParam("identifier") Stri } } -======= ->>>>>>> refs/heads/IQSS/6497-semantic_api private Dataset parseDataset(String datasetJson) throws WrappedResponse { try (StringReader rdr = new StringReader(datasetJson)) { return jsonParser().parseDataset(Json.createReader(rdr).readObject()); From 509746ca6091858a1ae9a8786f8a0e634d10c9d0 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 2 Feb 2021 14:48:36 -0500 Subject: [PATCH 105/354] adding a create from json-ld to mirror the other api calls --- .../harvard/iq/dataverse/api/Dataverses.java | 40 +++++++++++++++++++ .../iq/dataverse/util/json/JSONLDUtil.java | 28 +++++++------ 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index d039cd09dcb..e80c944f908 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -217,6 +217,7 @@ public Response addDataverse(String body, @PathParam("identifier") String parent @POST @Path("{identifier}/datasets") + @Consumes("application/json") public Response createDataset(String jsonBody, @PathParam("identifier") String parentIdtf) { try { User u = findUserOrDie(); @@ -254,6 +255,45 @@ public Response createDataset(String jsonBody, @PathParam("identifier") String p return ex.getResponse(); } } + + @POST + @Path("{identifier}/datasets") + @Consumes("application/json-ld") + public Response createDatasetFromJsonLd(String jsonLDBody, @PathParam("identifier") String parentIdtf) { + try { + User u = findUserOrDie(); + Dataverse owner = findDataverseOrDie(parentIdtf); + Dataset ds = new Dataset(); + + ds.setOwner(owner); + ds = JSONLDUtil.updateDatasetMDFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc, false, false); + + ds.setOwner(owner); + + + + // clean possible dataset/version metadata + DatasetVersion version = ds.getVersions().get(0); + version.setMinorVersionNumber(null); + version.setVersionNumber(null); + version.setVersionState(DatasetVersion.VersionState.DRAFT); + + ds.setAuthority(null); + ds.setIdentifier(null); + ds.setProtocol(null); + ds.setGlobalIdCreateTime(null); + + Dataset managedDs = execCommand(new CreateNewDatasetCommand(ds, createDataverseRequest(u))); + return created("/datasets/" + managedDs.getId(), + Json.createObjectBuilder() + .add("id", managedDs.getId()) + .add("persistentId", managedDs.getGlobalIdString()) + ); + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } @POST @Path("{identifier}/datasets/:import") diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 72e6a3b21ee..b4fa1b63ab9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -81,13 +81,15 @@ public static Dataset updateDatasetMDFromJsonLD(Dataset ds, String jsonLDBody, DatasetVersion dsv = new DatasetVersion(); JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); + if(migrating) { Optional maybePid = GlobalId.parse(jsonld.getString("@id")); - if (maybePid.isPresent()) { + if (maybePid.isPresent()) { ds.setGlobalId(maybePid.get()); - } else { + } else { // unparsable PID passed. Terminate. throw new BadRequestException("Cannot parse the @id '" + jsonld.getString("@id") + "'. Make sure it is in valid form - see Dataverse Native API documentation."); + } } dsv = updateDatasetVersionMDFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); @@ -97,16 +99,18 @@ public static Dataset updateDatasetMDFromJsonLD(Dataset ds, String jsonLDBody, versions.add(dsv); ds.setVersions(versions); - if (jsonld.containsKey(JsonLDTerm.schemaOrg("dateModified").getUrl())) { - String dateString = jsonld.getString(JsonLDTerm.schemaOrg("dateModified").getUrl()); - LocalDateTime dateTime = getDateTimeFrom(dateString); - ds.setModificationTime(Timestamp.valueOf(dateTime)); - } - try { - logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); + if (migrating) { + if (jsonld.containsKey(JsonLDTerm.schemaOrg("dateModified").getUrl())) { + String dateString = jsonld.getString(JsonLDTerm.schemaOrg("dateModified").getUrl()); + LocalDateTime dateTime = getDateTimeFrom(dateString); + ds.setModificationTime(Timestamp.valueOf(dateTime)); + } + try { + logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } return ds; } From d4c15cc81466963713f09c164037f44e78866ae9 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 2 Feb 2021 17:02:37 -0500 Subject: [PATCH 106/354] handle null terms on new dataset --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index b4fa1b63ab9..3a3c5c3404c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -148,7 +148,8 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv } fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); } - TermsOfUseAndAccess terms = dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess(); + + TermsOfUseAndAccess terms = (dsv.getTermsOfUseAndAccess()!=null) ? dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess() : new TermsOfUseAndAccess(); for (String key : jsonld.keySet()) { if (!key.equals("@context")) { From 05e1441396ae58e5926e4e395146f2053d8ba96f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 9 Feb 2021 15:04:41 -0500 Subject: [PATCH 107/354] initial documentation commit --- .../source/_static/api/dataset-migrate.jsonld | 42 +++++++++ .../developers/dataset-migration-api.rst | 50 +++++++++++ .../dataset-semantic-metadata-api.rst | 90 +++++++++++++++++++ doc/sphinx-guides/source/developers/index.rst | 2 + 4 files changed, 184 insertions(+) create mode 100644 doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld create mode 100644 doc/sphinx-guides/source/developers/dataset-migration-api.rst create mode 100644 doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst diff --git a/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld b/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld new file mode 100644 index 00000000000..520a2830cee --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld @@ -0,0 +1,42 @@ +{ +"citation:Depositor": "Admin, Dataverse", +"Title": "Test Dataset", +"Subject": "Computer and Information Science", +"Deposit Date": "2020-10-08", +"citation:Distributor": { +"distributor:Name": "Demo Dataverse Repository", +"distributor:Affiliation": "Dataverse Community", +"distributor:Abbreviation": "GDCC", +"distributor:URL": "https://dataverse.org/global-dataverse-community-consortium" +}, +"citation:Contact": { +"datasetContact:Name": "Admin, Dataverse", +"datasetContact:Affiliation": "GDCC", +"datasetContact:E-mail": "admin@demo.dataverse.org" +}, +"citation:Description": { +"dsDescription:Text": "A short description" +}, +"@id": "doi:10.33564/FK27U7YBV", +"schema:version": "1.0", +"schema:license": "https://creativecommons.org/publicdomain/zero/1.0/", +"dvcore:fileTermsOfAccess": { +"dvcore:fileRequestAccess": false +}, +"@context": { +"Creator": "http://purl.org/dc/terms/creator", +"Deposit Date": "http://purl.org/dc/terms/dateSubmitted", +"Identifier Scheme": "http://purl.org/spar/datacite/AgentIdentifierScheme", +"Subject": "http://purl.org/dc/terms/subject", +"Title": "http://purl.org/dc/terms/title", +"author": "https://dataverse.org/schema/citation/author#", +"citation": "https://dataverse.org/schema/citation/", +"datasetContact": "https://dataverse.org/schema/citation/datasetContact#", +"dcterms": "http://purl.org/dc/terms/", +"distributor": "https://dataverse.org/schema/citation/distributor#", +"dsDescription": "https://dataverse.org/schema/citation/dsDescription#", +"dvcore": "https://dataverse.org/schema/core#", +"keyword": "https://dataverse.org/schema/citation/keyword#", +"ore": "http://www.openarchives.org/ore/terms/", +"schema": "http://schema.org/" +}} diff --git a/doc/sphinx-guides/source/developers/dataset-migration-api.rst b/doc/sphinx-guides/source/developers/dataset-migration-api.rst new file mode 100644 index 00000000000..bfa6ba901c1 --- /dev/null +++ b/doc/sphinx-guides/source/developers/dataset-migration-api.rst @@ -0,0 +1,50 @@ +Dataset Migration API +===================== + +The Dataverse software includes several ways to add Datasets originally created elsewhere (not to mention Harvesting capabilities). These include the Sword API and the /dataverses/{id}/datasets/:import methods (json and ddi). + +This experimental migration API offers an additional option with some potential advantages: +- metadata can be specified using the json-ld format used in the OAI-ORE metadata export +- existing PIDs can be maintained (currently limited to the case where the PID can be managed by the Dataverse software, e.g. where the authority and shoulder match those the sfotware is configured for) +- adding files can be done via the standard APIs, including using direct-upload to S3 +- the dataset can be published keeping the original publication date + +This API consists of 2 calls: one to create an initial Dataset version, and one to publish the version with a specified publication date. +These calls can be used in concert with other API calls to add files, update metadata for additional versions, etc. + + +Start migrating a Dataset into a Dataverse Collection +----------------------------------------------------- + +.. note:: This action requires a Dataverse installation account with super-user permissions. + +To import a dataset with an existing persistent identifier (PID), the provided json-ld metadata should include it. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export DATAVERSE_ID=root + export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV + + curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$DATAVERSE_ID/datasets/:startmigration --upload-file dataset-migrate.jsonld + +An example jsonld file is available at :download:`dataset-migrate.jsonld <../_static/api/dataset-migrate.jsonld>` + + +Publish a Migrated Dataset +-------------------------- + +The call above creates a Dataset. Once it is created, other APIs can be used to add files, add additional metadata, etc. When a version is complete, the following call can be used to publish it with its original publication date + +.. note:: This action requires a Dataverse installation account with super-user permissions. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV + export SERVER_URL=https://demo.dataverse.org + + curl -H 'Content-Type: application/jsonld' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated" + +datePublished is the only metadata supported in this call. \ No newline at end of file diff --git a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst new file mode 100644 index 00000000000..519599d68f9 --- /dev/null +++ b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst @@ -0,0 +1,90 @@ +Dataset Semantic Metadata API +============================= + +The OAI_ORE metadata export format represents Dataset metadata using json-ld. As part of an RDA-supported effort to allow import of Datasets exported as Bags with an included OAI_ORE metadata file, +an experimental API has been created that provides a json-ld alternative the the v1.0 API calls to get/set/delete Dataset metadata. + +You may prefer to work with this API if you are building a tool to import from a Bag/OAI-ORE source or already work with json-ld representations of metadata, or if you prefer the flatter json-ld representation to Dataverse software's json representation (which includes structure related to the metadata blocks involved and the type/multiplicity of the metadata fields.) +You may not want to use this API if you need stability and backward compatibility (the 'experimental' designation for this API implies that community feedback is desired and that, in future Dataverse software versions, the API may be modified based on that feedback). + +Get Dataset Metadata +-------------------- + +To get the json-ld formatted metadata for a Dataset, specify the Dataset ID (DATASET_ID) or Persistent identifier (DATASET_PID), and, for specific versions, the version number. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export DATASET_ID='12345' + export DATASET_PID='doi:10.5072/FK2A1B2C3' + export VERSION='1.0' + export SERVER_URL=https://demo.dataverse.org + + Example 1: Get metadata for version '1.0' + + curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' "$SERVER_URL/api/datasets/$DATASET_ID/versions/$VERSION/metadata" + +Example 2: Get metadata for the latest version using the DATASET PID + + curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID" + +You should expect a 200 ("OK") response and JSON-LD mirroring the OAI-ORE representation in the returned 'data' object. + + +Add Dataset Metadata +-------------------- + +To add json-ld formatted metadata for a Dataset, specify the Dataset ID (DATASET_ID) or Persistent identifier (DATASET_PID). Adding '?replace=true' will overwrite an existing metadata value. The default (replace=false) will only add new metadata or add a new value to a multi-valued field. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export DATASET_ID='12345' + export DATASET_PID='doi:10.5072/FK2A1B2C3' + export VERSION='1.0' + export SERVER_URL=https://demo.dataverse.org + + Example: Change the Dataset title + + curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"Title": "Submit menu test", "@context":{"Title": "http://purl.org/dc/terms/title"}}' "$SERVER_URL/api/datasets/$DATASET_ID/metadata?replace=true" + +Example 2: Add a description using the DATASET PID + + curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"citation:Description": {"dsDescription:Text": "New description"}, "@context":{"citation": "https://dataverse.org/schema/citation/","dsDescription": "https://dataverse.org/schema/citation/dsDescription#"}}' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID" + +You should expect a 200 ("OK") response indicating whether a draft Dataset version was created or an existing draft was updated. + + +Delete Dataset Metadata +----------------------- + +To delete metadata for a Dataset, send a json-ld representation of the fields to delete and specify the Dataset ID (DATASET_ID) or Persistent identifier (DATASET_PID). + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export DATASET_ID='12345' + export DATASET_PID='doi:10.5072/FK2A1B2C3' + export VERSION='1.0' + export SERVER_URL=https://demo.dataverse.org + +Example: Delete the TermsOfUseAndAcess 'restrictions' value 'No restrictions' a for the latest version using the DATASET PID + + curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"https://dataverse.org/schema/core#restrictions":"No restrictions"}' "$SERVER_URL/api/datasets/:persistentId/metadata/delete?persistentId=$DATASET_PID" + +Note, this example uses the term URI directly rather than adding an '@context' element. You can use either form in any of these API calls. + +You should expect a 200 ("OK") response indicating whether a draft Dataset version was created or an existing draft was updated. + + +Create a Dataset +---------------- + +Specifying the Content-Type as application/json-ld with the existing /api/dataverses/{id}/datasets API call supports using the same metadata format when creating a Dataset. + +With curl, this is done using + +.. code-block:: bash + + -H 'Content-Type: application/json-ld' + diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst index 184d8aff85a..ef143a5b17c 100755 --- a/doc/sphinx-guides/source/developers/index.rst +++ b/doc/sphinx-guides/source/developers/index.rst @@ -34,4 +34,6 @@ Developer Guide selinux big-data-support aux-file-support + dataset-semantic-metadata-api + dataset-migration-api workflows From d885d633a96a649b296ada1b5fd3a0a18cf932a6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 9 Feb 2021 15:16:36 -0500 Subject: [PATCH 108/354] cleanup --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 3 +-- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4e8f08cddb2..bf3969f74cb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -640,7 +640,7 @@ public Response getVersionJsonLDMetadata(@PathParam("id") String id, @PathParam( @Path("{id}/metadata") @Produces("application/json-ld") public Response getVersionJsonLDMetadata(@PathParam("id") String id, @Context UriInfo uriInfo, @Context HttpHeaders headers) { - return getVersionJsonLDMetadata(id, ":draft", uriInfo, headers); + return getVersionJsonLDMetadata(id, ":latest", uriInfo, headers); } @PUT @@ -678,7 +678,6 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String @Path("{id}/metadata/delete") @Consumes("application/json-ld") public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id) { - logger.info("In delteMetadata"); try { Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 3a3c5c3404c..1173583f376 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -346,7 +346,7 @@ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv // Todo - normalize object vs. array JsonArray valArray = getValues(jsonld.get(key), dsft.isAllowMultiples(), dsft.getName()); -logger.info("Deleting: " + key + " : " + valArray.toString()); + logger.fine("Deleting: " + key + " : " + valArray.toString()); DatasetField dsf2 = getReplacementField(dsf, valArray); if(dsf2 == null) { //Exact match - remove the field From d0e1301c93a642798d91c22aa431c060c021b791 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 9 Feb 2021 15:59:22 -0500 Subject: [PATCH 109/354] links and cleanup --- .../developers/dataset-migration-api.rst | 11 ++++---- .../dataset-semantic-metadata-api.rst | 28 +++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dataset-migration-api.rst b/doc/sphinx-guides/source/developers/dataset-migration-api.rst index bfa6ba901c1..61644957008 100644 --- a/doc/sphinx-guides/source/developers/dataset-migration-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-migration-api.rst @@ -1,13 +1,14 @@ Dataset Migration API ===================== -The Dataverse software includes several ways to add Datasets originally created elsewhere (not to mention Harvesting capabilities). These include the Sword API and the /dataverses/{id}/datasets/:import methods (json and ddi). +The Dataverse software includes several ways to add Datasets originally created elsewhere (not to mention Harvesting capabilities). These include the Sword API (see the :doc:`/api/sword` guide) and the /dataverses/{id}/datasets/:import methods (json and ddi) (see the :doc:`/api/native-api` guide). This experimental migration API offers an additional option with some potential advantages: -- metadata can be specified using the json-ld format used in the OAI-ORE metadata export -- existing PIDs can be maintained (currently limited to the case where the PID can be managed by the Dataverse software, e.g. where the authority and shoulder match those the sfotware is configured for) -- adding files can be done via the standard APIs, including using direct-upload to S3 -- the dataset can be published keeping the original publication date + +* metadata can be specified using the json-ld format used in the OAI-ORE metadata export +* existing PIDs can be maintained (currently limited to the case where the PID can be managed by the Dataverse software, e.g. where the authority and shoulder match those the sfotware is configured for) +* adding files can be done via the standard APIs, including using direct-upload to S3 +* the dataset can be published keeping the original publication date This API consists of 2 calls: one to create an initial Dataset version, and one to publish the version with a specified publication date. These calls can be used in concert with other API calls to add files, update metadata for additional versions, etc. diff --git a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst index 519599d68f9..d55370f476f 100644 --- a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst @@ -1,8 +1,8 @@ Dataset Semantic Metadata API ============================= -The OAI_ORE metadata export format represents Dataset metadata using json-ld. As part of an RDA-supported effort to allow import of Datasets exported as Bags with an included OAI_ORE metadata file, -an experimental API has been created that provides a json-ld alternative the the v1.0 API calls to get/set/delete Dataset metadata. +The OAI_ORE metadata export format represents Dataset metadata using json-ld (see the :doc:`metadataexport` section). As part of an RDA-supported effort to allow import of Datasets exported as Bags with an included OAI_ORE metadata file, +an experimental API has been created that provides a json-ld alternative the the v1.0 API calls to get/set/delete Dataset metadata in the :doc:`/api/native-api`. You may prefer to work with this API if you are building a tool to import from a Bag/OAI-ORE source or already work with json-ld representations of metadata, or if you prefer the flatter json-ld representation to Dataverse software's json representation (which includes structure related to the metadata blocks involved and the type/multiplicity of the metadata fields.) You may not want to use this API if you need stability and backward compatibility (the 'experimental' designation for this API implies that community feedback is desired and that, in future Dataverse software versions, the API may be modified based on that feedback). @@ -20,13 +20,13 @@ To get the json-ld formatted metadata for a Dataset, specify the Dataset ID (DAT export VERSION='1.0' export SERVER_URL=https://demo.dataverse.org - Example 1: Get metadata for version '1.0' + Example 1: Get metadata for version '1.0' - curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' "$SERVER_URL/api/datasets/$DATASET_ID/versions/$VERSION/metadata" + curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' "$SERVER_URL/api/datasets/$DATASET_ID/versions/$VERSION/metadata" -Example 2: Get metadata for the latest version using the DATASET PID + Example 2: Get metadata for the latest version using the DATASET PID - curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID" + curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID" You should expect a 200 ("OK") response and JSON-LD mirroring the OAI-ORE representation in the returned 'data' object. @@ -44,13 +44,13 @@ To add json-ld formatted metadata for a Dataset, specify the Dataset ID (DATASET export VERSION='1.0' export SERVER_URL=https://demo.dataverse.org - Example: Change the Dataset title + Example: Change the Dataset title - curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"Title": "Submit menu test", "@context":{"Title": "http://purl.org/dc/terms/title"}}' "$SERVER_URL/api/datasets/$DATASET_ID/metadata?replace=true" + curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"Title": "Submit menu test", "@context":{"Title": "http://purl.org/dc/terms/title"}}' "$SERVER_URL/api/datasets/$DATASET_ID/metadata?replace=true" -Example 2: Add a description using the DATASET PID + Example 2: Add a description using the DATASET PID - curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"citation:Description": {"dsDescription:Text": "New description"}, "@context":{"citation": "https://dataverse.org/schema/citation/","dsDescription": "https://dataverse.org/schema/citation/dsDescription#"}}' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID" + curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"citation:Description": {"dsDescription:Text": "New description"}, "@context":{"citation": "https://dataverse.org/schema/citation/","dsDescription": "https://dataverse.org/schema/citation/dsDescription#"}}' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID" You should expect a 200 ("OK") response indicating whether a draft Dataset version was created or an existing draft was updated. @@ -68,9 +68,9 @@ To delete metadata for a Dataset, send a json-ld representation of the fields to export VERSION='1.0' export SERVER_URL=https://demo.dataverse.org -Example: Delete the TermsOfUseAndAcess 'restrictions' value 'No restrictions' a for the latest version using the DATASET PID + Example: Delete the TermsOfUseAndAcess 'restrictions' value 'No restrictions' a for the latest version using the DATASET PID - curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"https://dataverse.org/schema/core#restrictions":"No restrictions"}' "$SERVER_URL/api/datasets/:persistentId/metadata/delete?persistentId=$DATASET_PID" + curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"https://dataverse.org/schema/core#restrictions":"No restrictions"}' "$SERVER_URL/api/datasets/:persistentId/metadata/delete?persistentId=$DATASET_PID" Note, this example uses the term URI directly rather than adding an '@context' element. You can use either form in any of these API calls. @@ -80,9 +80,9 @@ You should expect a 200 ("OK") response indicating whether a draft Dataset versi Create a Dataset ---------------- -Specifying the Content-Type as application/json-ld with the existing /api/dataverses/{id}/datasets API call supports using the same metadata format when creating a Dataset. +Specifying the Content-Type as application/json-ld with the existing /api/dataverses/{id}/datasets API call (see :ref:`create-dataset-command`) supports using the same metadata format when creating a Dataset. -With curl, this is done using +With curl, this is done by adding the following header: .. code-block:: bash From 8851fe322cbc6db5b0ad3bade0c3bdaf21acbcee Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 9 Feb 2021 16:31:35 -0500 Subject: [PATCH 110/354] comments/cleanup --- scripts/api/data/metadatablocks/citation.tsv | 2 +- .../harvard/iq/dataverse/api/Datasets.java | 8 ++--- .../iq/dataverse/util/json/JSONLDUtil.java | 34 +++++-------------- 3 files changed, 13 insertions(+), 31 deletions(-) diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index 710f0ff4267..26e2b7c184a 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -79,7 +79,7 @@ originOfSources Origin of Sources For historical materials, information about the origin of the sources and the rules followed in establishing the sources should be specified. textbox 75 FALSE FALSE FALSE FALSE FALSE FALSE citation characteristicOfSources Characteristic of Sources Noted Assessment of characteristics and source material. textbox 76 FALSE FALSE FALSE FALSE FALSE FALSE citation accessToSources Documentation and Access to Sources Level of documentation of the original sources. textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation - metadataOnOrig Metadata on the original source of migrated datasets. textbox 78 FALSE FALSE FALSE FALSE FALSE FALSE citation + metadataOnOrig External Metadata Metadata that isn't supported in Dataverse, as provided by external applications textbox 78 FALSE FALSE FALSE FALSE FALSE FALSE citation #controlledVocabulary DatasetField Value identifier displayOrder subject Agricultural Sciences D01 0 subject Arts and Humanities D0 1 diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index bf3969f74cb..a41d9ef94c4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -684,7 +684,7 @@ public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id) { DatasetVersion dsv = ds.getEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); dsv = JSONLDUtil.deleteDatasetVersionMDFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc); - logger.info("Updating ver"); + logger.fine("Updating ver"); DatasetVersion managedVersion; if (updateDraft) { Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); @@ -1848,7 +1848,7 @@ public Response completeMPUpload(String partETagBody, @QueryParam("globalid") St "You are not permitted to complete file uploads with the supplied parameters."); } List eTagList = new ArrayList(); - logger.info("Etags: " + partETagBody); + logger.fine("Etags: " + partETagBody); try { JsonReader jsonReader = Json.createReader(new StringReader(partETagBody)); JsonObject object = jsonReader.readObject(); @@ -1857,10 +1857,10 @@ public Response completeMPUpload(String partETagBody, @QueryParam("globalid") St eTagList.add(new PartETag(Integer.parseInt(partNo), object.getString(partNo))); } for(PartETag et: eTagList) { - logger.info("Part: " + et.getPartNumber() + " : " + et.getETag()); + logger.fine("Part: " + et.getPartNumber() + " : " + et.getETag()); } } catch (JsonException je) { - logger.info("Unable to parse eTags from: " + partETagBody); + logger.fine("Unable to parse eTags from: " + partETagBody); throw new WrappedResponse(je, error( Response.Status.INTERNAL_SERVER_ERROR, "Could not complete multipart upload")); } try { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 1173583f376..9a57e97ffdf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -177,25 +177,8 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv addField(dsf, valArray, dsft, datasetFieldSvc, append); - // assemble new terms, add to existing - // multivalue? - // compound? - // merge with existing dv metadata - // dsfl.add(dsf); } else { - // Internal/non-metadatablock terms - // Add metadata related to the Dataset/DatasetVersion - - // ("@id", id) - check is equal to existing globalID? - // Add to 'md on original' ? - // (JsonLDTerm.schemaOrg("version").getLabel(), - // version.getFriendlyVersionNumber()) - // Citation metadata? - // (JsonLDTerm.schemaOrg("datePublished").getLabel(), - // dataset.getPublicationDateFormattedYYYYMMDD()) - // (JsonLDTerm.schemaOrg("name").getLabel()) - // (JsonLDTerm.schemaOrg("dateModified").getLabel()) - + //When migrating, the publication date and version number can be set if (key.equals(JsonLDTerm.schemaOrg("datePublished").getUrl())&& migrating && !append) { dsv.setVersionState(VersionState.RELEASED); } else if (key.equals(JsonLDTerm.schemaOrg("version").getUrl())&& migrating && !append) { @@ -206,6 +189,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index + 1))); } } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())) { + //Special handling for license if (!append || !isSet(terms, key)) { // Mirror rules from SwordServiceBean if (jsonld.containsKey(JsonLDTerm.termsOfUse.getUrl())) { @@ -221,19 +205,19 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv } } else if (datasetTerms.contains(key)) { + // Other Dataset-level TermsOfUseAndAccess if (!append || !isSet(terms, key)) { - // Other Dataset-level TermsOfUseAndAccess setSemTerm(terms, key, jsonld.getString(key)); } else { throw new BadRequestException( "Can't append to a single-value field that already has a value: " + key); } } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { + // Other DataFile-level TermsOfUseAndAccess JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); for (String fileKey : fAccessObject.keySet()) { if (datafileTerms.contains(fileKey)) { if (!append || !isSet(terms, fileKey)) { - // Other DataFile-level TermsOfUseAndAccess if (fileKey.equals(JsonLDTerm.fileRequestAccess.getUrl())) { setSemTerm(terms, fileKey, fAccessObject.getBoolean(fileKey)); } else { @@ -247,6 +231,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv } } } else { + //Metadata block metadata fields if (dsftMap.containsKey(JsonLDTerm.metadataOnOrig.getUrl())) { DatasetFieldType dsft = dsftMap.get(JsonLDTerm.metadataOnOrig.getUrl()); @@ -287,12 +272,9 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv } } dsv.setTermsOfUseAndAccess(terms); - // move to new dataverse? - // aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), - // dataset.getDataverseContext().getDisplayName()); - + // ToDo: support Dataverse location metadata? e.g. move to new dataverse? + // re: JsonLDTerm.schemaOrg("includedInDataCatalog") } - } } @@ -312,7 +294,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv */ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv, String jsonLDBody, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { -logger.info("deleteDatasetVersionMD"); + logger.fine("deleteDatasetVersionMD"); JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); //All terms are now URIs //Setup dsftMap - URI to datasetFieldType map From b161473935922307d5336bbb1afbd4c1b3339443 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 10 Feb 2021 14:45:28 -0500 Subject: [PATCH 111/354] adding first draft of release notes --- doc/release-notes/6497-migrate-api.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/release-notes/6497-migrate-api.md diff --git a/doc/release-notes/6497-migrate-api.md b/doc/release-notes/6497-migrate-api.md new file mode 100644 index 00000000000..64e74b7effa --- /dev/null +++ b/doc/release-notes/6497-migrate-api.md @@ -0,0 +1,7 @@ +# Release Highlights + +### Dataset Migration API (Experimental) + +Datasets can now imported/updated following the format of an OAI-ORE export (RDA-conformant Bags), allowing for not only easier migration from one Dataverse installation to another, but also for better support of import from other systems. This experimental endpoint also allows keeping the existing persistent identifier and publication dates. + +This development was supported by the [Research Data Alliance](https://re-alliance.org) and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). \ No newline at end of file From 94c926fba31fb919772ebdebac92f346441fa23d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 10 Feb 2021 16:52:50 -0500 Subject: [PATCH 112/354] Update doc/sphinx-guides/source/developers/dataset-migration-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/dataset-migration-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dataset-migration-api.rst b/doc/sphinx-guides/source/developers/dataset-migration-api.rst index 61644957008..5ea65df7868 100644 --- a/doc/sphinx-guides/source/developers/dataset-migration-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-migration-api.rst @@ -6,7 +6,7 @@ The Dataverse software includes several ways to add Datasets originally created This experimental migration API offers an additional option with some potential advantages: * metadata can be specified using the json-ld format used in the OAI-ORE metadata export -* existing PIDs can be maintained (currently limited to the case where the PID can be managed by the Dataverse software, e.g. where the authority and shoulder match those the sfotware is configured for) +* existing PIDs can be maintained (currently limited to the case where the PID can be managed by the Dataverse software, e.g. where the authority and shoulder match those the software is configured for) * adding files can be done via the standard APIs, including using direct-upload to S3 * the dataset can be published keeping the original publication date @@ -48,4 +48,4 @@ The call above creates a Dataset. Once it is created, other APIs can be used to curl -H 'Content-Type: application/jsonld' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated" -datePublished is the only metadata supported in this call. \ No newline at end of file +datePublished is the only metadata supported in this call. From 9e05baa188acb611ae5ed1a2c096bbfc41ce8f2b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 10 Feb 2021 16:54:24 -0500 Subject: [PATCH 113/354] Update doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst Co-authored-by: Philip Durbin --- .../source/developers/dataset-semantic-metadata-api.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst index d55370f476f..ddabedca1a5 100644 --- a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst @@ -68,7 +68,7 @@ To delete metadata for a Dataset, send a json-ld representation of the fields to export VERSION='1.0' export SERVER_URL=https://demo.dataverse.org - Example: Delete the TermsOfUseAndAcess 'restrictions' value 'No restrictions' a for the latest version using the DATASET PID + Example: Delete the TermsOfUseAndAccess 'restrictions' value 'No restrictions' for the latest version using the DATASET PID curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"https://dataverse.org/schema/core#restrictions":"No restrictions"}' "$SERVER_URL/api/datasets/:persistentId/metadata/delete?persistentId=$DATASET_PID" @@ -87,4 +87,3 @@ With curl, this is done by adding the following header: .. code-block:: bash -H 'Content-Type: application/json-ld' - From 45e585334d37be8a7204e7984fa7173412db3689 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 10 Feb 2021 16:58:53 -0500 Subject: [PATCH 114/354] doc updates from code review --- doc/release-notes/6497-migrate-api.md | 14 ++++++++++++-- .../developers/dataset-semantic-metadata-api.rst | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/6497-migrate-api.md b/doc/release-notes/6497-migrate-api.md index 64e74b7effa..22edd84ca2f 100644 --- a/doc/release-notes/6497-migrate-api.md +++ b/doc/release-notes/6497-migrate-api.md @@ -2,6 +2,16 @@ ### Dataset Migration API (Experimental) -Datasets can now imported/updated following the format of an OAI-ORE export (RDA-conformant Bags), allowing for not only easier migration from one Dataverse installation to another, but also for better support of import from other systems. This experimental endpoint also allows keeping the existing persistent identifier and publication dates. +Datasets can now imported/updated following the format of an OAI-ORE export (RDA-conformant Bags), allowing for not only easier migration from one Dataverse installation to another, but also for better support of import from other systems. This experimental endpoint also allows keeping the existing persistent identifier (where the authority and shoulder match those for which the software is configured) and publication dates. -This development was supported by the [Research Data Alliance](https://re-alliance.org) and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). \ No newline at end of file +This development was supported by the [Research Data Alliance](https://rd-alliance.org) and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). + +### Additional Upgrade Steps + +Update Solr Schema + +- copy schema_dv_mdb_fields.xml and schema_dv_mdb_copies.xml to solr server, for example into /usr/local/solr/solr-7.7.2/server/solr/collection1/conf/ directory + +- Restart Solr, or tell Solr to reload its configuration: + + `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"` diff --git a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst index d55370f476f..00ab268653c 100644 --- a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst @@ -68,7 +68,7 @@ To delete metadata for a Dataset, send a json-ld representation of the fields to export VERSION='1.0' export SERVER_URL=https://demo.dataverse.org - Example: Delete the TermsOfUseAndAcess 'restrictions' value 'No restrictions' a for the latest version using the DATASET PID + Example: Delete the TermsOfUseAndAccess 'restrictions' value 'No restrictions' a for the latest version using the DATASET PID curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"https://dataverse.org/schema/core#restrictions":"No restrictions"}' "$SERVER_URL/api/datasets/:persistentId/metadata/delete?persistentId=$DATASET_PID" From 1d995ff36ac834a54a749b8cd1a664d0f46a0911 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 10 Feb 2021 17:07:53 -0500 Subject: [PATCH 115/354] adding ref to #5899 support --- doc/release-notes/6497-migrate-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6497-migrate-api.md b/doc/release-notes/6497-migrate-api.md index 22edd84ca2f..e5699a0d155 100644 --- a/doc/release-notes/6497-migrate-api.md +++ b/doc/release-notes/6497-migrate-api.md @@ -2,7 +2,7 @@ ### Dataset Migration API (Experimental) -Datasets can now imported/updated following the format of an OAI-ORE export (RDA-conformant Bags), allowing for not only easier migration from one Dataverse installation to another, but also for better support of import from other systems. This experimental endpoint also allows keeping the existing persistent identifier (where the authority and shoulder match those for which the software is configured) and publication dates. +Datasets can now imported/updated following the format of an OAI-ORE export (RDA-conformant Bags), allowing for not only easier migration from one Dataverse installation to another, but also for better support of import from other systems. This experimental endpoint also allows keeping the existing persistent identifier (where the authority and shoulder match those for which the software is configured) and publication dates. This endpoint also allows for the update of terms metadata (#5899). This development was supported by the [Research Data Alliance](https://rd-alliance.org) and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). From d8472534f2df5aac2467f7679452aa252cafdc5b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 17 Mar 2021 16:01:36 +0100 Subject: [PATCH 116/354] feature(appserver): update to Payara 5.2021.1 #7700 --- doc/release-notes/7700-upgrade-payara.md | 6 ++++++ pom.xml | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 doc/release-notes/7700-upgrade-payara.md diff --git a/doc/release-notes/7700-upgrade-payara.md b/doc/release-notes/7700-upgrade-payara.md new file mode 100644 index 00000000000..370e4def9fc --- /dev/null +++ b/doc/release-notes/7700-upgrade-payara.md @@ -0,0 +1,6 @@ +### Payara 5.2021.1 (or Higher) Required + +Some changes in this release require an upgrade to Payara 5.2021.1 or higher. + +Instructions on how to update can be found in the +[Payara documentation](https://docs.payara.fish/community/docs/5.2021.1/documentation/user-guides/upgrade-payara.html) \ No newline at end of file diff --git a/pom.xml b/pom.xml index ed7e800f797..98b5e1967f0 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ false 8.0.0 - 5.2020.6 + 5.2021.1 42.2.19 1.11.762 1.2 @@ -58,6 +58,18 @@ + + payara-nexus-artifacts + Payara Nexus Artifacts + https://nexus.payara.fish/repository/payara-artifacts + + true + + + false + + + payara-patched-externals Payara Patched Externals From 0172cde4041648cc17c78302572e18522390668f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 18 Mar 2021 16:59:03 +0100 Subject: [PATCH 117/354] feature(appserver): update Docker AIO + Vagrant to Payara 5.2021.1 #7700 --- conf/docker-aio/0prep_deps.sh | 4 ++-- conf/docker-aio/c8.dockerfile | 2 +- downloads/download.sh | 2 +- scripts/vagrant/setup.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/docker-aio/0prep_deps.sh b/conf/docker-aio/0prep_deps.sh index 5bf44bfbe82..04706dad342 100755 --- a/conf/docker-aio/0prep_deps.sh +++ b/conf/docker-aio/0prep_deps.sh @@ -4,10 +4,10 @@ if [ ! -d dv/deps ]; then fi wdir=`pwd` -if [ ! -e dv/deps/payara-5.2020.6.zip ]; then +if [ ! -e dv/deps/payara-5.2021.1.zip ]; then echo "payara dependency prep" # no more fiddly patching :) - wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip -O dv/deps/payara-5.2020.6.zip + wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.1/payara-5.2021.1.zip -O dv/deps/payara-5.2021.1.zip fi if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then diff --git a/conf/docker-aio/c8.dockerfile b/conf/docker-aio/c8.dockerfile index 92c3dc24d56..d06918279f3 100644 --- a/conf/docker-aio/c8.dockerfile +++ b/conf/docker-aio/c8.dockerfile @@ -18,7 +18,7 @@ COPY disableipv6.conf /etc/sysctl.d/ RUN rm /etc/httpd/conf/* COPY httpd.conf /etc/httpd/conf RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz -RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2020.6.zip ; ln -s /opt/payara5 /opt/glassfish4 +RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.1.zip ; ln -s /opt/payara5 /opt/glassfish4 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm` # this dies under Java 11, do we keep it? diff --git a/downloads/download.sh b/downloads/download.sh index 8c2b51dd4c7..a4e132e4a40 100755 --- a/downloads/download.sh +++ b/downloads/download.sh @@ -1,5 +1,5 @@ #!/bin/sh -curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip +curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.1/payara-5.2021.1.zip curl -L -O https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh index 14f12cea692..f48895187eb 100644 --- a/scripts/vagrant/setup.sh +++ b/scripts/vagrant/setup.sh @@ -53,7 +53,7 @@ SOLR_USER=solr echo "Ensuring Unix user '$SOLR_USER' exists" useradd $SOLR_USER || : DOWNLOAD_DIR='/dataverse/downloads' -PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2020.6.zip" +PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2021.1.zip" SOLR_TGZ="$DOWNLOAD_DIR/solr-8.8.1.tgz" if [ ! -f $PAYARA_ZIP ] || [ ! -f $SOLR_TGZ ]; then echo "Couldn't find $PAYARA_ZIP or $SOLR_TGZ! Running download script...." From 079be6dad197b4623fcb3f9a9e656192c931cc1e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 18 Mar 2021 17:00:34 +0100 Subject: [PATCH 118/354] doc(appserver): change docs to refer to Payara 5.2021.1 #7700 --- doc/sphinx-guides/source/developers/dev-environment.rst | 4 ++-- doc/sphinx-guides/source/installation/prerequisites.rst | 6 +++--- .../edu/harvard/iq/dataverse/util/DataSourceProducer.java | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index ed1849e6059..45467ee23b1 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -85,9 +85,9 @@ To install Payara, run the following commands: ``cd /usr/local`` -``sudo curl -O -L https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip`` +``sudo curl -O -L https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.1/payara-5.2021.1.zip`` -``sudo unzip payara-5.2020.6.zip`` +``sudo unzip payara-5.2021.1.zip`` ``sudo chown -R $USER /usr/local/payara5`` diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 435c5c59b61..870f95a20b9 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -44,7 +44,7 @@ On RHEL/CentOS you can make Java 11 the default with the ``alternatives`` comman Payara ------ -Payara 5.2020.6 is recommended. Newer versions might work fine, regular updates are recommended. +Payara 5.2021.1 is recommended. Newer versions might work fine, regular updates are recommended. Installing Payara ================= @@ -55,8 +55,8 @@ Installing Payara - Download and install Payara (installed in ``/usr/local/payara5`` in the example commands below):: - # wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip - # unzip payara-5.2020.6.zip + # wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.1/payara-5.2021.1.zip + # unzip payara-5.2021.1.zip # mv payara5 /usr/local If you intend to install and run Payara under a service account (and we hope you do), chown -R the Payara hierarchy to root to protect it but give the service account access to the below directories: diff --git a/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java b/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java index 02ba331cdd5..aa6068ef9f2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java @@ -38,7 +38,7 @@ //}) // // ... but at this time we don't think we need any. The full list -// of properties can be found at https://docs.payara.fish/community/docs/5.2020.6/documentation/payara-server/jdbc/advanced-connection-pool-properties.html#full-list-of-properties +// of properties can be found at https://docs.payara.fish/community/docs/5.2021.1/documentation/payara-server/jdbc/advanced-connection-pool-properties.html#full-list-of-properties // // All these properties cannot be configured via MPCONFIG as Payara doesn't support this (yet). To be enhanced. // See also https://github.com/payara/Payara/issues/5024 From 3a3235451900dacd9dbf48ea33ed445f128214a1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 22 Mar 2021 08:58:01 -0400 Subject: [PATCH 119/354] method dropped in merge --- .../harvard/iq/dataverse/api/Datasets.java | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 9eb82996dea..777627437dd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -51,6 +51,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.DeleteDatasetLinkingDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.DeletePrivateUrlCommand; import edu.harvard.iq.dataverse.engine.command.impl.DestroyDatasetCommand; +import edu.harvard.iq.dataverse.engine.command.impl.FinalizeDatasetPublicationCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetDatasetCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetSpecificPublishedDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetDraftDatasetVersionCommand; @@ -103,6 +104,7 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.util.json.JSONLDUtil; +import edu.harvard.iq.dataverse.util.json.JsonLDTerm; import edu.harvard.iq.dataverse.util.json.JsonParseException; import edu.harvard.iq.dataverse.search.IndexServiceBean; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; @@ -113,6 +115,7 @@ import java.io.StringReader; import java.sql.Timestamp; import java.text.MessageFormat; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -137,6 +140,7 @@ import javax.json.stream.JsonParsingException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.BadRequestException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -1160,6 +1164,92 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S } } + @Path("{id}/actions/:releasemigrated") + @Consumes("application/json-ld") + public Response publishMigratedDataset(String jsonldBody, @PathParam("id") String id) { + try { + AuthenticatedUser user = findAuthenticatedUserOrDie(); + if (!user.isSuperuser()) { + return error(Response.Status.FORBIDDEN, "Only superusers can release migrated datasets"); + } + + Dataset ds = findDatasetOrDie(id); + try { + JsonObject metadata = JSONLDUtil.decontextualizeJsonLD(jsonldBody); + String pubDate = metadata.getString(JsonLDTerm.schemaOrg("datePublished").getUrl()); + logger.fine("Submitted date: " + pubDate); + LocalDateTime dateTime = JSONLDUtil.getDateTimeFrom(pubDate); + // dataset.getPublicationDateFormattedYYYYMMDD()) + ds.setPublicationDate(Timestamp.valueOf(dateTime)); + } catch (Exception e) { + logger.fine(e.getMessage()); + throw new BadRequestException("Unable to set publication date (" + + JsonLDTerm.schemaOrg("datePublished").getUrl() + "): " + e.getMessage()); + } + /* + * Note: The code here mirrors that in the + * edu.harvard.iq.dataverse.DatasetPage:updateCurrentVersion method. Any changes + * to the core logic (i.e. beyond updating the messaging about results) should + * be applied to the code there as well. + */ + String errorMsg = null; + String successMsg = null; + try { + FinalizeDatasetPublicationCommand cmd = new FinalizeDatasetPublicationCommand(ds, + createDataverseRequest(user), true); + ds = commandEngine.submit(cmd); + //Todo - update messages + successMsg = BundleUtil.getStringFromBundle("datasetversion.update.success"); + + // If configured, update archive copy as well + String className = settingsService.get(SettingsServiceBean.Key.ArchiverClassName.toString()); + DatasetVersion updateVersion = ds.getLatestVersion(); + AbstractSubmitToArchiveCommand archiveCommand = ArchiverUtil.createSubmitToArchiveCommand(className, + createDataverseRequest(user), updateVersion); + if (archiveCommand != null) { + // Delete the record of any existing copy since it is now out of date/incorrect + updateVersion.setArchivalCopyLocation(null); + /* + * Then try to generate and submit an archival copy. Note that running this + * command within the CuratePublishedDatasetVersionCommand was causing an error: + * "The attribute [id] of class + * [edu.harvard.iq.dataverse.DatasetFieldCompoundValue] is mapped to a primary + * key column in the database. Updates are not allowed." To avoid that, and to + * simplify reporting back to the GUI whether this optional step succeeded, I've + * pulled this out as a separate submit(). + */ + try { + updateVersion = commandEngine.submit(archiveCommand); + if (updateVersion.getArchivalCopyLocation() != null) { + successMsg = BundleUtil.getStringFromBundle("datasetversion.update.archive.success"); + } else { + successMsg = BundleUtil.getStringFromBundle("datasetversion.update.archive.failure"); + } + } catch (CommandException ex) { + successMsg = BundleUtil.getStringFromBundle("datasetversion.update.archive.failure") + " - " + + ex.toString(); + logger.severe(ex.getMessage()); + } + } + } catch (CommandException ex) { + errorMsg = BundleUtil.getStringFromBundle("datasetversion.update.failure") + " - " + ex.toString(); + logger.severe(ex.getMessage()); + } + if (errorMsg != null) { + return error(Response.Status.INTERNAL_SERVER_ERROR, errorMsg); + } else { + JsonObjectBuilder responseBld = Json.createObjectBuilder() + .add("id", ds.getId()) + .add("persistentId", ds.getGlobalId().toString()); + return Response.ok(Json.createObjectBuilder().add("status", STATUS_OK).add("status_details", successMsg) + .add("data", responseBld).build()).type(MediaType.APPLICATION_JSON).build(); + } + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + @POST @Path("{id}/move/{targetDataverseAlias}") public Response moveDataset(@PathParam("id") String id, @PathParam("targetDataverseAlias") String targetDataverseAlias, @QueryParam("forceMove") Boolean force) { From 65d80a3f86a3afe2ced181bcf94d7125b7759f76 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 22 Mar 2021 10:06:31 -0400 Subject: [PATCH 120/354] use dataset pub date for files in migrate case --- .../engine/command/impl/FinalizeDatasetPublicationCommand.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 7a32f515d36..3405f4e5a92 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -120,7 +120,8 @@ public Dataset execute(CommandContext ctxt) throws CommandException { theDataset.setModificationTime(getTimestamp()); theDataset.setFileAccessRequest(theDataset.getLatestVersion().getTermsOfUseAndAccess().isFileAccessRequest()); - updateFiles(getTimestamp(), ctxt); + //Use dataset pub date (which may not be the current date for migrated datasets) + updateFiles(theDataset.getPublicationDate(), ctxt); // // TODO: Not sure if this .merge() is necessary here - ? From 4e94a8015ae392d66868b77c35e6f1b5ec7124b5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 22 Mar 2021 14:06:07 -0400 Subject: [PATCH 121/354] include previous note about upgrading #7700 --- doc/release-notes/7700-upgrade-payara.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/7700-upgrade-payara.md b/doc/release-notes/7700-upgrade-payara.md index 370e4def9fc..72183772be7 100644 --- a/doc/release-notes/7700-upgrade-payara.md +++ b/doc/release-notes/7700-upgrade-payara.md @@ -3,4 +3,6 @@ Some changes in this release require an upgrade to Payara 5.2021.1 or higher. Instructions on how to update can be found in the -[Payara documentation](https://docs.payara.fish/community/docs/5.2021.1/documentation/user-guides/upgrade-payara.html) \ No newline at end of file +[Payara documentation](https://docs.payara.fish/community/docs/5.2021.1/documentation/user-guides/upgrade-payara.html) + +It would likely be safer to upgrade Payara first, while still running Dataverse 5.3, and then proceed with the steps below. Upgrading from an earlier version of Payara should be a straightforward process: Undeploy Dataverse; stop Payara; move the current Payara directory out of the way; unzip the new Payara version in its place; replace the brand new payara/glassfish/domains/domain1 with your old, preserved domain1; start Payara, deploy Dataverse 5.3. We still recommend that you read the detailed upgrade instructions above; and, if you run into any issues with this upgrade, it will help to be able to separate them from any problems with the upgrade of Dataverse proper. From 9ca076833a8b9cda803700e9af7812b1a88257e9 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Mar 2021 08:44:46 -0400 Subject: [PATCH 122/354] make sure release user is set --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 777627437dd..709194cb839 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1181,6 +1181,8 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin LocalDateTime dateTime = JSONLDUtil.getDateTimeFrom(pubDate); // dataset.getPublicationDateFormattedYYYYMMDD()) ds.setPublicationDate(Timestamp.valueOf(dateTime)); + //Release User is only set in FinalizeDatasetPublicationCommand if the pub date is null, so set it here. + ds.setReleaseUser((AuthenticatedUser) user); } catch (Exception e) { logger.fine(e.getMessage()); throw new BadRequestException("Unable to set publication date (" From 341734361c4ad2cfa2561f1b798f8da8b839b41a Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 29 Mar 2021 09:14:00 -0400 Subject: [PATCH 123/354] flyway script adding the new constraint (#7451) --- .../V5.4.0.1__7451-unique-constraint-storageidentifier.sql | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/main/resources/db/migration/V5.4.0.1__7451-unique-constraint-storageidentifier.sql diff --git a/src/main/resources/db/migration/V5.4.0.1__7451-unique-constraint-storageidentifier.sql b/src/main/resources/db/migration/V5.4.0.1__7451-unique-constraint-storageidentifier.sql new file mode 100644 index 00000000000..d890929569d --- /dev/null +++ b/src/main/resources/db/migration/V5.4.0.1__7451-unique-constraint-storageidentifier.sql @@ -0,0 +1 @@ +ALTER TABLE dvobject ADD CONSTRAINT unq_dvobject_storageidentifier UNIQUE(owner_id, storageidentifier); From 347f7e75c4972aee1b3d8413abe597ecbef4a660 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 30 Mar 2021 09:54:14 -0400 Subject: [PATCH 124/354] A diagnostics script, to check and fix any duplicated harvested storageidentifiers, and re-check the local ones for any new dupes, just in case. (#7451) --- scripts/issues/7451/check_datafiles_7451.sh | 133 ++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100755 scripts/issues/7451/check_datafiles_7451.sh diff --git a/scripts/issues/7451/check_datafiles_7451.sh b/scripts/issues/7451/check_datafiles_7451.sh new file mode 100755 index 00000000000..0eb5040b70a --- /dev/null +++ b/scripts/issues/7451/check_datafiles_7451.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +# begin config +# PostgresQL credentials: +# edit the following lines so that psql can talk to your database +pg_host=localhost +pg_port=5432 +pg_user=dvnapp +pg_db=dvndb +# you can leave the password blank, if Postgres is configured +# to accept connections without auth: +pg_pass= +# psql executable - add full path, if necessary: +PSQL_EXEC=psql + +# end config + +# check for duplicate storageidentifiers in harvested datafiles: + +PG_QUERY_0="SELECT COUNT(DISTINCT o.id) FROM datafile f, dataset s, dvobject p, dvobject o WHERE s.id = p.id AND o.id = f.id AND o.owner_id = s.id AND s.harvestingclient_id IS NOT null AND o.storageidentifier IS NOT null" + +PG_QUERY_1="SELECT s.id, o.storageidentifier FROM datafile f, dataset s, dvobject o WHERE o.id = f.id AND o.owner_id = s.id AND s.harvestingclient_id IS NOT null AND o.storageidentifier IS NOT null ORDER by s.id, o.storageidentifier" + +PG_QUERY_FIX_0="UPDATE dvobject SET storageidentifier=NULL WHERE dtype='DataFile' AND (storageidentifier='file://' OR storageidentifier='http://' OR storageidentifier='s3://')" + +PG_QUERY_FIX_1="UPDATE dvobject SET storageidentifier=CONCAT(storageidentifier, ' ', id) WHERE owner_id = %d AND storageidentifier='%s'" + +PGPASSWORD=$pg_pass; export PGPASSWORD + +echo "Checking the total number of storageidentifiers in harvested datafiles..." + +NUM_DATAFILES=`${PSQL_EXEC} -h ${pg_host} -U ${pg_user} -d ${pg_db} -tA -F ' ' -c "${PG_QUERY_0}"` +if [ $? != 0 ] +then + echo "FAILED to execute psql! Check the credentials and try again?" + echo "exiting..." + echo + echo "the command line that failed:" + echo "${PSQL_EXEC} -h ${pg_host} -U ${pg_user} -d ${pg_db} -tA -F ' ' -c \"${PG_QUERY_0}\"" + exit 1 +fi + +echo $NUM_DATAFILES total. +echo + +# Before we do anything else, reset the storageidentifiers of the datafiles (harvested and otherwise) that +# may have ended up set to invalid, prefix-only values like "file://" back to NULL: + +${PSQL_EXEC} -h ${pg_host} -U ${pg_user} -d ${pg_db} -q -c "${PG_QUERY_FIX_0}" + + +echo "Let's check if any harvested storage identifiers are referenced more than once within the same dataset:" + +${PSQL_EXEC} -h ${pg_host} -U ${pg_user} -d ${pg_db} -tA -F ' ' -c "${PG_QUERY_1}" | +uniq -c | +awk '{if ($1 > 1) print $0}' | sort -u > /tmp/harvestedidentifiers.tmp + +NUM_CONFIRMED=`cat /tmp/harvestedidentifiers.tmp | wc -l` + +if [ $NUM_CONFIRMED == 0 ] +then + echo + echo "Good news - it appears that there are NO duplicate storageidentifiers in your harvested datafiles;" + echo "nothing to fix." + echo +else + + echo "Found ${NUM_CONFIRMED} harvested files with identical storageidentifiers; fixing in place..." + + cat /tmp/harvestedidentifiers.tmp | sed 's:\\:\\\\:g' | while read howmany dataset storageidentifier + do + PG_QUERY_SI=`printf "${PG_QUERY_FIX_1}" $dataset "$storageidentifier"` + + echo ${PSQL_EXEC} -h ${pg_host} -U ${pg_user} -d ${pg_db} -tA -F ' ' -c \"${PG_QUERY_SI}\" + done + + echo "... done." + echo +fi + + +# now, check for duplicate storageidentifiers in local datafiles: + +PG_QUERY_3="SELECT COUNT(DISTINCT o.id) FROM datafile f, dataset s, dvobject p, dvobject o WHERE s.id = p.id AND o.id = f.id AND o.owner_id = s.id AND s.harvestingclient_id IS null AND o.storageidentifier IS NOT null" + +PG_QUERY_4="SELECT s.id, o.storageidentifier FROM datafile f, dataset s, dvobject o WHERE o.id = f.id AND o.owner_id = s.id AND s.harvestingclient_id IS null AND o.storageidentifier IS NOT null ORDER by s.id, o.storageidentifier" + +PG_QUERY_5="SELECT p.authority, p.identifier, o.storageidentifier, o.id, o.createdate, f.contenttype FROM datafile f, dvobject p, dvobject o WHERE o.id = f.id AND o.owner_id = p.id AND p.id = %d AND o.storageidentifier='%s' ORDER by o.id" + +echo "Checking the number of non-harvested datafiles in the database..." + +NUM_DATAFILES=`${PSQL_EXEC} -h ${pg_host} -U ${pg_user} -d ${pg_db} -tA -F ' ' -c "${PG_QUERY_3}"` +echo $NUM_DATAFILES total. +echo + +echo "Let's check if any storage identifiers are referenced more than once within the same dataset:" + +${PSQL_EXEC} -h ${pg_host} -U ${pg_user} -d ${pg_db} -tA -F ' ' -c "${PG_QUERY_4}" | +uniq -c | +awk '{if ($1 > 1) print $0}' > /tmp/storageidentifiers.tmp + +NUM_CONFIRMED=`cat /tmp/storageidentifiers.tmp | wc -l` + +if [ $NUM_CONFIRMED == 0 ] +then + echo + echo "Good news - it appears that there are NO duplicate DataFile objects in your database." + echo "Your installation is ready to be upgraded to Dataverse 5.5" + echo +else + + echo "The following storage identifiers appear to be referenced from multiple non-harvested DvObjects:" + cat /tmp/storageidentifiers.tmp + echo "(output saved in /tmp/storageidentifiers.tmp)" + + echo "Looking up details for the affected datafiles:" + + cat /tmp/storageidentifiers.tmp | while read howmany dataset storageidentifier + do + PG_QUERY_SI=`printf "${PG_QUERY_5}" $dataset "$storageidentifier"` + ${PSQL_EXEC} -h ${pg_host} -U ${pg_user} -d ${pg_db} -tA -F ' ' -c "${PG_QUERY_SI}" + done | tee /tmp/duplicates_info.tmp + + echo "(output saved in /tmp/duplicates_info.tmp)" + + echo + echo "Please send the output above to Dataverse support at support@dataverse.org." + echo "We will assist you in the process of cleaning up the affected files above." + echo "We apologize for any inconvenience." + echo +fi + + From de0085c65367fa8c15c248188171456f3a7e1527 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 30 Mar 2021 10:29:33 -0400 Subject: [PATCH 125/354] A pre-release text for the new diagnostics script (will discuss the approach in the PR/dv-tech) (#7451) --- scripts/issues/7451/PRE-RELEASE-INFO.txt | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 scripts/issues/7451/PRE-RELEASE-INFO.txt diff --git a/scripts/issues/7451/PRE-RELEASE-INFO.txt b/scripts/issues/7451/PRE-RELEASE-INFO.txt new file mode 100644 index 00000000000..27f3cb027e5 --- /dev/null +++ b/scripts/issues/7451/PRE-RELEASE-INFO.txt @@ -0,0 +1,31 @@ +In the next release another constraint is being added to existing +databases, to prevent any possibility of creating datafile objects +referencing the same file. This was originally planned for v4.20, but +in that release the constraint was only added to newly created +databases, and was not forced on the databases that had already +existed. If your current database was originally created by version +4.20 or newer, you don't need to do anything. + +If you have an older database, please run the following script: + +https://github.com/IQSS/dataverse/raw/develop/scripts/issues/7451/check_datafiles_7451.sh + +The script relies on the PostgreSQL utility psql to access the +database. You will need to edit the credentials at the top of the script +to match your database configuration. + +The script will check your database for any duplicated storage +identifiers that would violate the new constraint. It will resolve any +such conflicts found in harvested files; but for any duplicated local +files, it will not attempt to make any changes right away (this being +an issue of data integrity with a potential to affect your users). +The script would then instruct you to send the produced diagnostic +information to support@dataverse.org so that we could assist you in +resolving the issues in your database. If no inconsistencies are +found, the script will report that the database is ready to be +upgraded to the next release. + +Please note that there's a very good chance that your database does +not have any conflicts of this nature. But we want to do this to be +absolutely sure. We apologize for any inconvenience. + From bad80d077513783dcd761936d7fd1d01b14a4d8b Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 30 Mar 2021 10:37:56 -0400 Subject: [PATCH 126/354] Arming the script bomb... (#7451) --- scripts/issues/7451/check_datafiles_7451.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/issues/7451/check_datafiles_7451.sh b/scripts/issues/7451/check_datafiles_7451.sh index 0eb5040b70a..fa1b22e96ed 100755 --- a/scripts/issues/7451/check_datafiles_7451.sh +++ b/scripts/issues/7451/check_datafiles_7451.sh @@ -71,7 +71,7 @@ else do PG_QUERY_SI=`printf "${PG_QUERY_FIX_1}" $dataset "$storageidentifier"` - echo ${PSQL_EXEC} -h ${pg_host} -U ${pg_user} -d ${pg_db} -tA -F ' ' -c \"${PG_QUERY_SI}\" + ${PSQL_EXEC} -h ${pg_host} -U ${pg_user} -d ${pg_db} -tA -F ' ' -c "${PG_QUERY_SI}" done echo "... done." From 4b5ad664ac18165dc032ab7faeee240cf6002aa2 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 30 Mar 2021 11:04:13 -0400 Subject: [PATCH 127/354] Assign a version if not set --- .../harvard/iq/dataverse/api/Datasets.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index c2cb940901e..94573973457 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1213,8 +1213,27 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin LocalDateTime dateTime = JSONLDUtil.getDateTimeFrom(pubDate); // dataset.getPublicationDateFormattedYYYYMMDD()) ds.setPublicationDate(Timestamp.valueOf(dateTime)); - //Release User is only set in FinalizeDatasetPublicationCommand if the pub date is null, so set it here. + // Release User is only set in FinalizeDatasetPublicationCommand if the pub date + // is null, so set it here. ds.setReleaseUser((AuthenticatedUser) user); + // Assign a version number if not set + if (ds.getLatestVersion().getVersionNumber() == null) { + + if (ds.getVersions().size() == 1) { + // First Release + ds.getLatestVersion().setVersionNumber(new Long(1)); + ds.getLatestVersion().setMinorVersionNumber(new Long(0)); + + } else if (ds.getLatestVersion().isMinorUpdate()) { + ds.getLatestVersion().setVersionNumber(new Long(ds.getVersionNumber())); + ds.getLatestVersion().setMinorVersionNumber(new Long(ds.getMinorVersionNumber() + 1)); + + } else { + // major, non-first release + ds.getLatestVersion().setVersionNumber(new Long(ds.getVersionNumber() + 1)); + ds.getLatestVersion().setMinorVersionNumber(new Long(0)); + } + } } catch (Exception e) { logger.fine(e.getMessage()); throw new BadRequestException("Unable to set publication date (" From a17c4bb05aa66eb375279174355bcbb79a1ad5ed Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 30 Mar 2021 11:08:58 -0400 Subject: [PATCH 128/354] Don't call archiver directly The direct call here was originally from when the Curate command was used (in which workflows don't run but any current archival copy needs to be updated). Now that FinalizePub is used instead, it will run any post-publication workflow (e.g. for archiving) and/or the admin can run archiving separately. --- .../harvard/iq/dataverse/api/Datasets.java | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 94573973457..35648cc16cf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1253,37 +1253,6 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin ds = commandEngine.submit(cmd); //Todo - update messages successMsg = BundleUtil.getStringFromBundle("datasetversion.update.success"); - - // If configured, update archive copy as well - String className = settingsService.get(SettingsServiceBean.Key.ArchiverClassName.toString()); - DatasetVersion updateVersion = ds.getLatestVersion(); - AbstractSubmitToArchiveCommand archiveCommand = ArchiverUtil.createSubmitToArchiveCommand(className, - createDataverseRequest(user), updateVersion); - if (archiveCommand != null) { - // Delete the record of any existing copy since it is now out of date/incorrect - updateVersion.setArchivalCopyLocation(null); - /* - * Then try to generate and submit an archival copy. Note that running this - * command within the CuratePublishedDatasetVersionCommand was causing an error: - * "The attribute [id] of class - * [edu.harvard.iq.dataverse.DatasetFieldCompoundValue] is mapped to a primary - * key column in the database. Updates are not allowed." To avoid that, and to - * simplify reporting back to the GUI whether this optional step succeeded, I've - * pulled this out as a separate submit(). - */ - try { - updateVersion = commandEngine.submit(archiveCommand); - if (updateVersion.getArchivalCopyLocation() != null) { - successMsg = BundleUtil.getStringFromBundle("datasetversion.update.archive.success"); - } else { - successMsg = BundleUtil.getStringFromBundle("datasetversion.update.archive.failure"); - } - } catch (CommandException ex) { - successMsg = BundleUtil.getStringFromBundle("datasetversion.update.archive.failure") + " - " - + ex.toString(); - logger.severe(ex.getMessage()); - } - } } catch (CommandException ex) { errorMsg = BundleUtil.getStringFromBundle("datasetversion.update.failure") + " - " + ex.toString(); logger.severe(ex.getMessage()); From c3b12195db2c5132f11672295e777f411eda4759 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 31 Mar 2021 09:19:03 -0400 Subject: [PATCH 129/354] add @POST per Chris Muller --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 35648cc16cf..459de504c12 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1196,6 +1196,7 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S } } + @POST @Path("{id}/actions/:releasemigrated") @Consumes("application/json-ld") public Response publishMigratedDataset(String jsonldBody, @PathParam("id") String id) { From a069812e3b97e96b3ffd7ede05905460539a702f Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 31 Mar 2021 17:22:05 -0400 Subject: [PATCH 130/354] switched to a conditional constraint. (#7451) --- ....0.1__7451-unique-constraint-storageidentifier.sql | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.4.0.1__7451-unique-constraint-storageidentifier.sql b/src/main/resources/db/migration/V5.4.0.1__7451-unique-constraint-storageidentifier.sql index d890929569d..93104da27a6 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7451-unique-constraint-storageidentifier.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7451-unique-constraint-storageidentifier.sql @@ -1 +1,10 @@ -ALTER TABLE dvobject ADD CONSTRAINT unq_dvobject_storageidentifier UNIQUE(owner_id, storageidentifier); +-- ALTER TABLE dvobject ADD CONSTRAINT unq_dvobject_storageidentifier UNIQUE(owner_id, storageidentifier); +-- Instead of a uniform constraint on all dvobjects (as above), force a +-- conditional unique constraint on datafiles only: +CREATE UNIQUE INDEX IF NOT EXISTS unq_dvobject_storageidentifier ON dvobject (owner_id, storageidentifier) WHERE dtype='DataFile'; +-- This is not going to have any effect on new databases (since v4.20+), +-- where the table was created with the full constraint; but for legacy +-- installations it would spare them having to clean up any dataset-level +-- storageidentifiers. We know that some old installations have datasets +-- with junk values in that column (like "file" - ??) that are meaningless, +-- but otherwise harmless. From 58a2317d9fb696999fc7439bbcea18d05bc2728e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 1 Apr 2021 17:27:14 -0400 Subject: [PATCH 131/354] handle mutliple versions, setting pub/release dates correctly previously, dataset pub date was getting updated with new versions, datasetversion release date was not being set File pub dates were getting set to the dataset pub date and not the date for the version --- .../harvard/iq/dataverse/api/Datasets.java | 23 +++++++++++++++---- .../FinalizeDatasetPublicationCommand.java | 7 ++++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 459de504c12..a6ae292dc52 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -115,6 +115,7 @@ import java.io.StringReader; import java.sql.Timestamp; import java.text.MessageFormat; +import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -160,6 +161,8 @@ import javax.ws.rs.core.Response.Status; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import javax.ws.rs.core.UriInfo; + +import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.SolrServerException; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -1211,12 +1214,14 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin JsonObject metadata = JSONLDUtil.decontextualizeJsonLD(jsonldBody); String pubDate = metadata.getString(JsonLDTerm.schemaOrg("datePublished").getUrl()); logger.fine("Submitted date: " + pubDate); - LocalDateTime dateTime = JSONLDUtil.getDateTimeFrom(pubDate); + LocalDateTime dateTime = null; + if(!StringUtils.isEmpty(pubDate)) { + dateTime = JSONLDUtil.getDateTimeFrom(pubDate); + final Timestamp time = Timestamp.valueOf(dateTime); + //Set version release date + ds.getLatestVersion().setReleaseTime(new Date(time.getTime())); + } // dataset.getPublicationDateFormattedYYYYMMDD()) - ds.setPublicationDate(Timestamp.valueOf(dateTime)); - // Release User is only set in FinalizeDatasetPublicationCommand if the pub date - // is null, so set it here. - ds.setReleaseUser((AuthenticatedUser) user); // Assign a version number if not set if (ds.getLatestVersion().getVersionNumber() == null) { @@ -1224,6 +1229,13 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin // First Release ds.getLatestVersion().setVersionNumber(new Long(1)); ds.getLatestVersion().setMinorVersionNumber(new Long(0)); + //Also set publication date if this is the first + if(dateTime != null) { + ds.setPublicationDate(Timestamp.valueOf(dateTime)); + } + // Release User is only set in FinalizeDatasetPublicationCommand if the pub date + // is null, so set it here. + ds.setReleaseUser((AuthenticatedUser) user); } else if (ds.getLatestVersion().isMinorUpdate()) { ds.getLatestVersion().setVersionNumber(new Long(ds.getVersionNumber())); @@ -1234,6 +1246,7 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin ds.getLatestVersion().setVersionNumber(new Long(ds.getVersionNumber() + 1)); ds.getLatestVersion().setMinorVersionNumber(new Long(0)); } + } } catch (Exception e) { logger.fine(e.getMessage()); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index ddb658bb44a..3b445c2683f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -115,13 +115,16 @@ public Dataset execute(CommandContext ctxt) throws CommandException { } // update metadata - theDataset.getLatestVersion().setReleaseTime(getTimestamp()); + if (theDataset.getLatestVersion().getReleaseTime() == null) { + // Allow migrated versions to keep original release dates + theDataset.getLatestVersion().setReleaseTime(getTimestamp()); + } theDataset.getLatestVersion().setLastUpdateTime(getTimestamp()); theDataset.setModificationTime(getTimestamp()); theDataset.setFileAccessRequest(theDataset.getLatestVersion().getTermsOfUseAndAccess().isFileAccessRequest()); //Use dataset pub date (which may not be the current date for migrated datasets) - updateFiles(theDataset.getPublicationDate(), ctxt); + updateFiles(new Timestamp(theDataset.getLatestVersion().getReleaseTime().getTime()), ctxt); // // TODO: Not sure if this .merge() is necessary here - ? From 59bbaaf56f1047077e31151d1156436cc51d94cf Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Wed, 7 Apr 2021 15:09:32 -0400 Subject: [PATCH 132/354] Update PRE-RELEASE-INFO.txt --- scripts/issues/7451/PRE-RELEASE-INFO.txt | 27 +++++++++++++----------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/scripts/issues/7451/PRE-RELEASE-INFO.txt b/scripts/issues/7451/PRE-RELEASE-INFO.txt index 27f3cb027e5..b0aca07bfae 100644 --- a/scripts/issues/7451/PRE-RELEASE-INFO.txt +++ b/scripts/issues/7451/PRE-RELEASE-INFO.txt @@ -15,17 +15,20 @@ database. You will need to edit the credentials at the top of the script to match your database configuration. The script will check your database for any duplicated storage -identifiers that would violate the new constraint. It will resolve any -such conflicts found in harvested files; but for any duplicated local -files, it will not attempt to make any changes right away (this being -an issue of data integrity with a potential to affect your users). -The script would then instruct you to send the produced diagnostic -information to support@dataverse.org so that we could assist you in -resolving the issues in your database. If no inconsistencies are -found, the script will report that the database is ready to be -upgraded to the next release. - -Please note that there's a very good chance that your database does +identifiers that would violate the new constraint. + +For harvested files, it will directly resolve any conflicts. + +For local files, rather than attempt to make any changes right away +(this being an issue of data integrity with a potential to affect your +users) it will instruct you to send the produced diagnostic +information to support@dataverse.org so that we can assist you +in resolving the issues in your database. + +If no inconsistencies are found, the script will report that the database +is ready to be upgraded to the next release. + +(Please note that there's a very good chance that your database does not have any conflicts of this nature. But we want to do this to be -absolutely sure. We apologize for any inconvenience. +absolutely sure. We apologize for any inconvenience.) From 804086ebdfedb59ce19b7ead24c4e43d4e44660c Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 Apr 2021 17:01:31 +0200 Subject: [PATCH 133/354] chore(appserver): upgrade to Payara 5.2021.2 #7700 --- conf/docker-aio/0prep_deps.sh | 4 ++-- conf/docker-aio/c8.dockerfile | 2 +- doc/release-notes/7700-upgrade-payara.md | 19 +++++++++++++++---- .../source/developers/dev-environment.rst | 4 ++-- .../source/installation/prerequisites.rst | 6 +++--- downloads/download.sh | 2 +- pom.xml | 2 +- scripts/vagrant/setup.sh | 2 +- .../iq/dataverse/util/DataSourceProducer.java | 2 +- 9 files changed, 27 insertions(+), 16 deletions(-) diff --git a/conf/docker-aio/0prep_deps.sh b/conf/docker-aio/0prep_deps.sh index 04706dad342..f75b762623a 100755 --- a/conf/docker-aio/0prep_deps.sh +++ b/conf/docker-aio/0prep_deps.sh @@ -4,10 +4,10 @@ if [ ! -d dv/deps ]; then fi wdir=`pwd` -if [ ! -e dv/deps/payara-5.2021.1.zip ]; then +if [ ! -e dv/deps/payara-5.2021.2.zip ]; then echo "payara dependency prep" # no more fiddly patching :) - wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.1/payara-5.2021.1.zip -O dv/deps/payara-5.2021.1.zip + wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.2/payara-5.2021.2.zip -O dv/deps/payara-5.2021.2.zip fi if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then diff --git a/conf/docker-aio/c8.dockerfile b/conf/docker-aio/c8.dockerfile index d06918279f3..ff2c2950568 100644 --- a/conf/docker-aio/c8.dockerfile +++ b/conf/docker-aio/c8.dockerfile @@ -18,7 +18,7 @@ COPY disableipv6.conf /etc/sysctl.d/ RUN rm /etc/httpd/conf/* COPY httpd.conf /etc/httpd/conf RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz -RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.1.zip ; ln -s /opt/payara5 /opt/glassfish4 +RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.2.zip ; ln -s /opt/payara5 /opt/glassfish4 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm` # this dies under Java 11, do we keep it? diff --git a/doc/release-notes/7700-upgrade-payara.md b/doc/release-notes/7700-upgrade-payara.md index 72183772be7..36f40d7e665 100644 --- a/doc/release-notes/7700-upgrade-payara.md +++ b/doc/release-notes/7700-upgrade-payara.md @@ -1,8 +1,19 @@ -### Payara 5.2021.1 (or Higher) Required +### Payara 5.2021.2 (or Higher) Required -Some changes in this release require an upgrade to Payara 5.2021.1 or higher. +Some changes in this release require an upgrade to Payara 5.2021.2 or higher. Instructions on how to update can be found in the -[Payara documentation](https://docs.payara.fish/community/docs/5.2021.1/documentation/user-guides/upgrade-payara.html) +[Payara documentation](https://docs.payara.fish/community/docs/5.2021.2/documentation/user-guides/upgrade-payara.html) -It would likely be safer to upgrade Payara first, while still running Dataverse 5.3, and then proceed with the steps below. Upgrading from an earlier version of Payara should be a straightforward process: Undeploy Dataverse; stop Payara; move the current Payara directory out of the way; unzip the new Payara version in its place; replace the brand new payara/glassfish/domains/domain1 with your old, preserved domain1; start Payara, deploy Dataverse 5.3. We still recommend that you read the detailed upgrade instructions above; and, if you run into any issues with this upgrade, it will help to be able to separate them from any problems with the upgrade of Dataverse proper. +It would likely be safer to upgrade Payara first, while still running Dataverse 5.4, and then proceed with the steps +below. Upgrading from an earlier version of Payara should be a straightforward process: + +1. Undeploy Dataverse +2. Stop Payara +3. Move the current Payara directory out of the way +4. Unzip the new Payara version in its place +5. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved `domain1` +6. Start Payara, deploy Dataverse 5.4. + +We still recommend that you read the detailed upgrade instructions above and if you run into any issues with this +upgrade, it will help to be able to separate them from any problems with the upgrade of Dataverse proper. diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 45467ee23b1..818f4682091 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -85,9 +85,9 @@ To install Payara, run the following commands: ``cd /usr/local`` -``sudo curl -O -L https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.1/payara-5.2021.1.zip`` +``sudo curl -O -L https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.2/payara-5.2021.2.zip`` -``sudo unzip payara-5.2021.1.zip`` +``sudo unzip payara-5.2021.2.zip`` ``sudo chown -R $USER /usr/local/payara5`` diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 870f95a20b9..f919c078c70 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -44,7 +44,7 @@ On RHEL/CentOS you can make Java 11 the default with the ``alternatives`` comman Payara ------ -Payara 5.2021.1 is recommended. Newer versions might work fine, regular updates are recommended. +Payara 5.2021.2 is recommended. Newer versions might work fine, regular updates are recommended. Installing Payara ================= @@ -55,8 +55,8 @@ Installing Payara - Download and install Payara (installed in ``/usr/local/payara5`` in the example commands below):: - # wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.1/payara-5.2021.1.zip - # unzip payara-5.2021.1.zip + # wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.2/payara-5.2021.2.zip + # unzip payara-5.2021.2.zip # mv payara5 /usr/local If you intend to install and run Payara under a service account (and we hope you do), chown -R the Payara hierarchy to root to protect it but give the service account access to the below directories: diff --git a/downloads/download.sh b/downloads/download.sh index a4e132e4a40..55db21b92b0 100755 --- a/downloads/download.sh +++ b/downloads/download.sh @@ -1,5 +1,5 @@ #!/bin/sh -curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.1/payara-5.2021.1.zip +curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.2/payara-5.2021.2.zip curl -L -O https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar diff --git a/pom.xml b/pom.xml index c0aa9fd25a0..de5ba582873 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ false 8.0.0 - 5.2021.1 + 5.2021.2 42.2.19 1.11.762 1.2 diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh index f48895187eb..081b6cd0b90 100644 --- a/scripts/vagrant/setup.sh +++ b/scripts/vagrant/setup.sh @@ -53,7 +53,7 @@ SOLR_USER=solr echo "Ensuring Unix user '$SOLR_USER' exists" useradd $SOLR_USER || : DOWNLOAD_DIR='/dataverse/downloads' -PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2021.1.zip" +PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2021.2.zip" SOLR_TGZ="$DOWNLOAD_DIR/solr-8.8.1.tgz" if [ ! -f $PAYARA_ZIP ] || [ ! -f $SOLR_TGZ ]; then echo "Couldn't find $PAYARA_ZIP or $SOLR_TGZ! Running download script...." diff --git a/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java b/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java index aa6068ef9f2..5828861f0e2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java @@ -38,7 +38,7 @@ //}) // // ... but at this time we don't think we need any. The full list -// of properties can be found at https://docs.payara.fish/community/docs/5.2021.1/documentation/payara-server/jdbc/advanced-connection-pool-properties.html#full-list-of-properties +// of properties can be found at https://docs.payara.fish/community/docs/5.2021.2/documentation/payara-server/jdbc/advanced-connection-pool-properties.html#full-list-of-properties // // All these properties cannot be configured via MPCONFIG as Payara doesn't support this (yet). To be enhanced. // See also https://github.com/payara/Payara/issues/5024 From da37d7cbe304e2ce686111f8c194245fef0d547f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 14 Apr 2021 15:10:31 +0200 Subject: [PATCH 134/354] refactor(deps): switch from Apache Commons Lang (v2) to Lang3 (v3). #6070 --- pom.xml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index f33ba575009..8e4e1743a1a 100644 --- a/pom.xml +++ b/pom.xml @@ -30,6 +30,7 @@ 42.2.19 1.11.762 1.2 + 3.12.0 4.5.5 4.13.1 5.7.0 @@ -120,6 +121,11 @@ commons-logging ${commons.logging.version} + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + org.apache.httpcomponents httpclient @@ -318,9 +324,8 @@ provided - commons-lang - commons-lang - 2.6 + org.apache.commons + commons-lang3 org.apache.solr From eb59a1100a7879d2dcb364441e7b313a863e149c Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 14 Apr 2021 15:29:43 +0200 Subject: [PATCH 135/354] refactor(deps): make main Java classes use Apache Commons Lang3 #6070 Also switch from bulk imports of commons.lang3.* to single class style. Fixing for other parts like java.util, too. --- .../dataverse/DOIDataCiteRegisterService.java | 2 +- .../harvard/iq/dataverse/DataCitation.java | 4 +- .../iq/dataverse/DataFileServiceBean.java | 2 +- .../edu/harvard/iq/dataverse/DataFileTag.java | 2 +- .../harvard/iq/dataverse/DatasetField.java | 2 +- .../dataverse/DatasetFieldCompoundValue.java | 2 +- .../iq/dataverse/DatasetFieldValidator.java | 2 +- .../iq/dataverse/DatasetFieldValue.java | 2 +- .../dataverse/DatasetFieldValueValidator.java | 2 +- .../edu/harvard/iq/dataverse/DatasetPage.java | 2 +- .../iq/dataverse/DatasetServiceBean.java | 2 +- .../harvard/iq/dataverse/DatasetVersion.java | 2 +- .../dataverse/DatasetVersionDifference.java | 2 +- .../dataverse/DatasetVersionServiceBean.java | 2 +- .../edu/harvard/iq/dataverse/Dataverse.java | 2 +- .../iq/dataverse/DataverseHeaderFragment.java | 2 +- .../harvard/iq/dataverse/DataversePage.java | 4 +- .../iq/dataverse/DvObjectContainer.java | 2 +- .../iq/dataverse/DvObjectServiceBean.java | 2 +- .../iq/dataverse/EditDatafilesPage.java | 2 +- .../edu/harvard/iq/dataverse/Guestbook.java | 2 +- .../harvard/iq/dataverse/GuestbookPage.java | 2 +- .../iq/dataverse/HandlenetServiceBean.java | 2 +- .../iq/dataverse/HarvestingClientsPage.java | 2 +- .../iq/dataverse/HarvestingSetsPage.java | 2 +- .../harvard/iq/dataverse/MailServiceBean.java | 2 +- .../dataverse/ManageFilePermissionsPage.java | 2 +- .../iq/dataverse/ManageGroupsPage.java | 2 +- .../iq/dataverse/ManagePermissionsPage.java | 2 +- .../iq/dataverse/NavigationWrapper.java | 2 +- .../iq/dataverse/RoleAssigneeServiceBean.java | 2 +- .../iq/dataverse/RolePermissionFragment.java | 4 +- .../java/edu/harvard/iq/dataverse/Shib.java | 2 +- .../iq/dataverse/ThemeWidgetFragment.java | 2 +- .../harvard/iq/dataverse/UserServiceBean.java | 2 +- .../iq/dataverse/api/AbstractApiBean.java | 2 +- .../dataverse/api/DatasetFieldServiceApi.java | 2 +- .../edu/harvard/iq/dataverse/api/Groups.java | 2 +- .../iq/dataverse/api/HarvestingServer.java | 2 +- .../edu/harvard/iq/dataverse/api/Search.java | 2 +- .../api/datadeposit/SwordServiceBean.java | 2 +- .../dataverse/api/datadeposit/UrlManager.java | 2 +- .../api/imports/ImportDDIServiceBean.java | 2 +- .../api/imports/ImportServiceBean.java | 2 +- .../AuthTestDataServiceBean.java | 2 +- .../DataverseRolePermissionHelper.java | 2 +- .../providers/builtin/DataverseUserPage.java | 2 +- .../providers/builtin/PasswordEncryption.java | 2 +- .../iq/dataverse/dataaccess/DataAccess.java | 2 +- .../dataaccess/TabularSubsetGenerator.java | 21 ++++++-- .../datasetutility/AddReplaceFileHelper.java | 2 +- .../command/impl/ImportDatasetCommand.java | 2 +- .../harvest/client/HarvesterServiceBean.java | 4 +- .../harvest/client/oai/OaiHandler.java | 2 +- .../server/web/servlet/OAIServlet.java | 2 +- .../ingest/IngestableDataChecker.java | 2 +- .../ingest/tabulardata/InvalidData.java | 4 +- .../impl/plugins/csv/CSVFileReader.java | 2 +- .../impl/plugins/dta/DTAFileReader.java | 48 ++++++++++++------- .../impl/plugins/dta/NewDTAFileReader.java | 33 ++++++++++--- .../impl/plugins/por/PORFileReader.java | 43 +++++++++++++---- .../impl/plugins/rdata/RDATAFileReader.java | 4 +- .../impl/plugins/rdata/RTabFileParser.java | 2 +- .../impl/plugins/sav/SAVFileReader.java | 44 +++++++++++++---- .../impl/plugins/xlsx/XLSXFileReader.java | 2 +- .../iq/dataverse/mydata/DataRetrieverAPI.java | 2 +- .../dataverse/mydata/MyDataFilterParams.java | 2 +- .../iq/dataverse/mydata/MyDataFinder.java | 2 +- .../iq/dataverse/mydata/RoleTagRetriever.java | 2 +- .../dataverse/mydata/SolrQueryFormatter.java | 2 +- .../passwordreset/PasswordResetPage.java | 2 +- .../iq/dataverse/rserve/RJobRequest.java | 12 +++-- .../rserve/RemoteDataFrameService.java | 25 ++++++++-- .../rserve/VariableNameCheckerForR.java | 10 ++-- .../dataverse/search/AdvancedSearchPage.java | 2 +- .../iq/dataverse/search/IndexServiceBean.java | 2 +- .../search/SearchIncludeFragment.java | 2 +- .../iq/dataverse/search/SearchUtil.java | 2 +- .../dataverse/util/FileSortFieldAndOrder.java | 2 +- .../harvard/iq/dataverse/util/FileUtil.java | 2 +- .../iq/dataverse/util/MarkupChecker.java | 2 +- .../iq/dataverse/util/SumStatCalculator.java | 10 ++-- .../validation/PasswordValidatorUtil.java | 2 +- 83 files changed, 261 insertions(+), 145 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java index 815733f1b7b..1fc8564e389 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java @@ -25,7 +25,7 @@ import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import javax.persistence.TypedQuery; -import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang3.StringEscapeUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java index 2e8df48f4f1..867f37b65c0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java @@ -33,8 +33,8 @@ import javax.xml.stream.XMLStreamWriter; import edu.harvard.iq.dataverse.util.BundleUtil; -import org.apache.commons.lang.StringEscapeUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 29b8eb85ead..bc51fa76094 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -38,7 +38,7 @@ import javax.persistence.Query; import javax.persistence.StoredProcedureQuery; import javax.persistence.TypedQuery; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java b/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java index 02d3da128f1..275d47cf1de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java @@ -20,7 +20,7 @@ import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; import javax.persistence.Table; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index 79f8916deb9..ef49c68834f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -33,7 +33,7 @@ import javax.persistence.OrderBy; import javax.persistence.Table; import javax.persistence.Transient; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; @Entity @ValidateDatasetFieldType diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java index 221922ea004..407a1d57bd3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java @@ -25,7 +25,7 @@ import javax.persistence.OneToMany; import javax.persistence.OrderBy; import javax.persistence.Table; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java index 00b7be97b83..c685fcb3e54 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java @@ -11,7 +11,7 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Collections; import java.util.List; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java index 27929dd3a39..2447a6478fd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java @@ -21,7 +21,7 @@ import javax.persistence.ManyToOne; import javax.persistence.Table; import javax.persistence.Transient; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java index e571fd89627..0b1619e6851 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java @@ -18,7 +18,7 @@ import java.util.regex.Pattern; import javax.validation.ConstraintValidator; import javax.validation.ConstraintValidatorContext; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index b9cc26ab89b..6e89ed2814a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -114,7 +114,7 @@ import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletResponse; -import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.io.IOUtils; diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 224ccfd22f3..685017200b5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -49,7 +49,7 @@ import javax.persistence.Query; import javax.persistence.StoredProcedureQuery; import javax.persistence.TypedQuery; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.ocpsoft.common.util.Strings; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index d28ce5175d4..1f0467577a2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -49,7 +49,7 @@ import javax.validation.Validator; import javax.validation.ValidatorFactory; import javax.validation.constraints.Size; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java index 8cc25f5148e..fee8a66a290 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java @@ -12,7 +12,7 @@ import java.util.List; import java.util.Set; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; import java.util.Date; diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java index e4eb6aac88e..d1fadfbae6f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java @@ -34,7 +34,7 @@ import javax.persistence.PersistenceContext; import javax.persistence.Query; import javax.persistence.TypedQuery; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.SolrServerException; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index b46333a4287..342aaec187a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -33,7 +33,7 @@ import javax.validation.constraints.Pattern; import javax.validation.constraints.Size; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.hibernate.validator.constraints.NotBlank; import org.hibernate.validator.constraints.NotEmpty; diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseHeaderFragment.java b/src/main/java/edu/harvard/iq/dataverse/DataverseHeaderFragment.java index 7e9655b3970..b806ef8e22d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseHeaderFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseHeaderFragment.java @@ -25,7 +25,7 @@ import javax.faces.view.ViewScoped; import javax.inject.Inject; import javax.inject.Named; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index c7f816ce219..84f3d793303 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -52,8 +52,8 @@ import javax.ejb.EJBException; import javax.faces.event.ValueChangeEvent; import javax.faces.model.SelectItem; -import org.apache.commons.lang.StringEscapeUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.lang3.StringUtils; import org.primefaces.PrimeFaces; import org.primefaces.event.TransferEvent; diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java index f6b396f4c00..ad72f3819fb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java @@ -2,7 +2,7 @@ import edu.harvard.iq.dataverse.dataaccess.DataAccess; import javax.persistence.MappedSuperclass; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * A {@link DvObject} that can contain other {@link DvObject}s. diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java index 4830c422d05..01b0890d588 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java @@ -19,7 +19,7 @@ import javax.persistence.NonUniqueResultException; import javax.persistence.PersistenceContext; import javax.persistence.Query; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.ocpsoft.common.util.Strings; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index aef3f7d3446..7d190618330 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -76,7 +76,7 @@ import javax.faces.event.FacesEvent; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletResponse; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.mutable.MutableBoolean; import org.primefaces.PrimeFaces; diff --git a/src/main/java/edu/harvard/iq/dataverse/Guestbook.java b/src/main/java/edu/harvard/iq/dataverse/Guestbook.java index 742e73403c1..1f25930a4b5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Guestbook.java +++ b/src/main/java/edu/harvard/iq/dataverse/Guestbook.java @@ -23,7 +23,7 @@ import javax.persistence.Transient; import edu.harvard.iq.dataverse.util.DateUtil; -import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang3.StringEscapeUtils; import org.hibernate.validator.constraints.NotBlank; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java index a7fb2b5a3fd..37d7169b959 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java @@ -24,7 +24,7 @@ import javax.faces.view.ViewScoped; import javax.inject.Inject; import javax.inject.Named; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java index 0549baaf8ae..defb884fd3e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java @@ -48,7 +48,7 @@ import net.handle.hdllib.PublicKeyAuthenticationInfo; import net.handle.hdllib.ResolutionRequest; import net.handle.hdllib.Util; -import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.NotImplementedException; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java b/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java index a88c4833f54..bc83c15dcd7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java @@ -40,7 +40,7 @@ import javax.inject.Inject; import javax.inject.Named; import javax.servlet.http.HttpServletRequest; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/HarvestingSetsPage.java b/src/main/java/edu/harvard/iq/dataverse/HarvestingSetsPage.java index 496050ca7f4..9781ff90732 100644 --- a/src/main/java/edu/harvard/iq/dataverse/HarvestingSetsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/HarvestingSetsPage.java @@ -35,7 +35,7 @@ import javax.faces.view.ViewScoped; import javax.inject.Inject; import javax.inject.Named; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index a0a91e22c32..912e278ffeb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -39,7 +39,7 @@ import javax.mail.internet.AddressException; import javax.mail.internet.InternetAddress; import javax.mail.internet.MimeMessage; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index 82e1681b37d..6d0e2f77c50 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -34,7 +34,7 @@ import javax.inject.Named; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.primefaces.event.SelectEvent; import org.primefaces.event.ToggleSelectEvent; import org.primefaces.event.UnselectEvent; diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageGroupsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageGroupsPage.java index e86ce1e22ad..d08337ec832 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageGroupsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageGroupsPage.java @@ -33,7 +33,7 @@ import javax.inject.Named; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * @author michaelsuo diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index 79a3ca800e2..90f441cadfe 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -41,7 +41,7 @@ import javax.inject.Named; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang3.StringEscapeUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/NavigationWrapper.java b/src/main/java/edu/harvard/iq/dataverse/NavigationWrapper.java index c9bc8722391..37a11396f37 100644 --- a/src/main/java/edu/harvard/iq/dataverse/NavigationWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/NavigationWrapper.java @@ -20,7 +20,7 @@ import javax.inject.Named; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java index 6b207ed0e75..c6f2b7f28a5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java @@ -27,7 +27,7 @@ import javax.inject.Named; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * The place to obtain {@link RoleAssignee}s, based on their identifiers. diff --git a/src/main/java/edu/harvard/iq/dataverse/RolePermissionFragment.java b/src/main/java/edu/harvard/iq/dataverse/RolePermissionFragment.java index 204806e7894..b5683b625cd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RolePermissionFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/RolePermissionFragment.java @@ -36,8 +36,8 @@ import javax.persistence.PersistenceContext; import edu.harvard.iq.dataverse.util.BundleUtil; -import org.apache.commons.lang.StringEscapeUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/Shib.java b/src/main/java/edu/harvard/iq/dataverse/Shib.java index 4ad50320f23..b71fe3cd566 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Shib.java +++ b/src/main/java/edu/harvard/iq/dataverse/Shib.java @@ -15,7 +15,7 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.JsfHelper; import edu.harvard.iq.dataverse.util.SystemConfig; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.io.IOException; import java.sql.Timestamp; diff --git a/src/main/java/edu/harvard/iq/dataverse/ThemeWidgetFragment.java b/src/main/java/edu/harvard/iq/dataverse/ThemeWidgetFragment.java index 9f282b436db..e270d3842f6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ThemeWidgetFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/ThemeWidgetFragment.java @@ -29,7 +29,7 @@ import javax.inject.Inject; import javax.inject.Named; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.primefaces.PrimeFaces; //import org.primefaces.context.RequestContext; diff --git a/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java index 5707f477a87..04ddcdfc570 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java @@ -19,7 +19,7 @@ import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import javax.persistence.Query; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.ocpsoft.common.util.Strings; @Stateless diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 6b84a883287..49e5fa9a2d3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -77,7 +77,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; -import static org.apache.commons.lang.StringUtils.isNumeric; +import static org.apache.commons.lang3.StringUtils.isNumeric; /** * Base class for API beans diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java index 646abf51b6c..315c3de4400 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java @@ -34,7 +34,7 @@ import javax.ws.rs.core.Response; import edu.harvard.iq.dataverse.util.BundleUtil; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.asJsonArray; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Groups.java b/src/main/java/edu/harvard/iq/dataverse/api/Groups.java index f2a262bb7fb..5a587efadf3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Groups.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Groups.java @@ -29,7 +29,7 @@ import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.PathParam; -import static org.apache.commons.lang.StringUtils.isNumeric; +import static org.apache.commons.lang3.StringUtils.isNumeric; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingServer.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingServer.java index cb28d1fae49..b8950edc6a0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingServer.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingServer.java @@ -46,7 +46,7 @@ import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Response; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java index 3a49385c0f6..93baa878e9d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Search.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Search.java @@ -35,7 +35,7 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * User-facing documentation: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordServiceBean.java index 4daaad76978..e4171c19017 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordServiceBean.java @@ -16,7 +16,7 @@ import javax.ejb.EJB; import javax.ejb.Stateless; import javax.inject.Named; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.swordapp.server.SwordEntry; import org.swordapp.server.SwordError; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/UrlManager.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/UrlManager.java index ce1e7fb9051..e8621ef7a35 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/UrlManager.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/UrlManager.java @@ -6,7 +6,7 @@ import java.util.Arrays; import java.util.List; import java.util.logging.Logger; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.swordapp.server.SwordError; import org.swordapp.server.UriRegistry; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java index 52459dc3c31..06be5104c6a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java @@ -31,7 +31,7 @@ import javax.xml.stream.XMLInputFactory; import edu.harvard.iq.dataverse.util.json.ControlledVocabularyException; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java index 9f7444f028c..6fc385af3ff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java @@ -67,7 +67,7 @@ import javax.validation.Validator; import javax.validation.ValidatorFactory; import javax.xml.stream.XMLStreamException; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthTestDataServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthTestDataServiceBean.java index ec64800c92d..3715900733c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthTestDataServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthTestDataServiceBean.java @@ -16,7 +16,7 @@ import java.util.Map; import java.util.logging.Logger; import javax.ejb.Stateless; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; @Stateless public class AuthTestDataServiceBean { diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/DataverseRolePermissionHelper.java b/src/main/java/edu/harvard/iq/dataverse/authorization/DataverseRolePermissionHelper.java index 6d0df17e520..4e6b54a8d49 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/DataverseRolePermissionHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/DataverseRolePermissionHelper.java @@ -19,7 +19,7 @@ import javax.ejb.EJB; import javax.ejb.Stateless; import javax.inject.Named; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /* * To change this license header, choose License Headers in Project Properties. diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index 673839450d6..d050dbc0dbd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -62,7 +62,7 @@ import javax.faces.view.ViewScoped; import javax.inject.Inject; import javax.inject.Named; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.hibernate.validator.constraints.NotBlank; import org.primefaces.event.TabChangeEvent; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/PasswordEncryption.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/PasswordEncryption.java index 38cabb6ef25..4446f68228d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/PasswordEncryption.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/PasswordEncryption.java @@ -3,7 +3,7 @@ import java.io.UnsupportedEncodingException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.mindrot.jbcrypt.BCrypt; //import org.primefaces.util.Base64; import java.util.Base64; diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index 0e2320401dd..a422a825259 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -27,7 +27,7 @@ import java.util.Properties; import java.util.logging.Logger; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * * @author Leonid Andreev diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetGenerator.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetGenerator.java index c97dc747505..0b6b37af9f0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetGenerator.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetGenerator.java @@ -22,11 +22,21 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.datavariable.DataVariable; -import java.util.*; -import java.util.Scanner; -import java.util.logging.*; -import java.io.*; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; +import java.util.Set; import java.math.BigDecimal; import java.math.MathContext; import java.math.RoundingMode; @@ -34,10 +44,11 @@ import java.nio.channels.FileChannel; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; +import java.util.logging.Logger; import java.util.regex.Matcher; -import org.apache.commons.lang.*; +import org.apache.commons.lang3.StringUtils; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index e82e6d7ae37..10ade521563 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -45,7 +45,7 @@ import javax.json.JsonObjectBuilder; import javax.validation.ConstraintViolation; import javax.ws.rs.core.Response; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.io.IOUtils; import org.ocpsoft.common.util.Strings; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ImportDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ImportDatasetCommand.java index daab9771b4e..a6875369493 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ImportDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ImportDatasetCommand.java @@ -13,7 +13,7 @@ import java.util.logging.Logger; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.methods.GetMethod; -import static org.apache.commons.lang.StringUtils.isEmpty; +import static org.apache.commons.lang3.StringUtils.isEmpty; /** * Imports a dataset from a different system. This command validates that the PID diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java index eb5cda8d72b..71cc23e242b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java @@ -31,8 +31,8 @@ //import javax.xml.bind.Unmarshaller; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; -import org.apache.commons.lang.mutable.MutableBoolean; -import org.apache.commons.lang.mutable.MutableLong; +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.apache.commons.lang3.mutable.MutableLong; import org.xml.sax.SAXException; import com.lyncode.xoai.model.oaipmh.Header; diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java index 38ebd83843b..d1aaea50793 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java @@ -24,7 +24,7 @@ import java.io.UnsupportedEncodingException; import javax.xml.parsers.ParserConfigurationException; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.xml.sax.SAXException; import javax.xml.transform.TransformerException; import java.net.URLEncoder; diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java index 5d4c02a87e2..8e436eae8c1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java @@ -40,7 +40,7 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.MailUtil; import edu.harvard.iq.dataverse.util.SystemConfig; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.io.ByteArrayOutputStream; diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java index 512c744c07a..5f771d2756b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java @@ -29,7 +29,7 @@ import java.util.regex.*; import java.util.zip.*; import java.util.logging.Logger; -import org.apache.commons.lang.builder.*; +import org.apache.commons.lang3.builder.*; import org.apache.commons.io.IOUtils; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/InvalidData.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/InvalidData.java index 4f584abc4cb..de8e2f7aa82 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/InvalidData.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/InvalidData.java @@ -20,8 +20,8 @@ package edu.harvard.iq.dataverse.ingest.tabulardata; import java.util.*; -import org.apache.commons.lang.builder.ToStringBuilder; -import org.apache.commons.lang.builder.ToStringStyle; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; /** * A class that stores information about a variables' invalid data. * Akio Sone's original DVN v.2.* implementation, virtually unchanged. diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/csv/CSVFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/csv/CSVFileReader.java index d1e85c208ae..26dd515cc16 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/csv/CSVFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/csv/CSVFileReader.java @@ -50,7 +50,7 @@ import java.util.Set; import java.util.logging.Logger; import org.apache.commons.csv.CSVFormat; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.CSVRecord; diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/DTAFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/DTAFileReader.java index 48a6212ffd7..2dec701592e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/DTAFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/DTAFileReader.java @@ -20,34 +20,46 @@ package edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.dta; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InvalidObjectException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; + +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; - -import java.io.*; -import java.nio.*; -import java.util.logging.*; - -import java.util.*; -import java.util.regex.*; -import java.text.*; - - -import org.apache.commons.lang.*; import org.apache.commons.codec.binary.Hex; -import javax.inject.Inject; -import javax.naming.Context; -import javax.naming.InitialContext; -import javax.naming.NamingException; import edu.harvard.iq.dataverse.DataTable; import edu.harvard.iq.dataverse.datavariable.DataVariable; import edu.harvard.iq.dataverse.datavariable.VariableCategory; -//import edu.harvard.iq.dataverse.datavariable.VariableFormatType; -//import edu.harvard.iq.dataverse.datavariable.VariableServiceBean; -import edu.harvard.iq.dataverse.ingest.plugin.spi.*; import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataFileReader; import edu.harvard.iq.dataverse.ingest.tabulardata.spi.TabularDataFileReaderSpi; import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataIngest; +import org.apache.commons.lang3.StringUtils; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/NewDTAFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/NewDTAFileReader.java index d523b3d4587..994b4901bee 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/NewDTAFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/NewDTAFileReader.java @@ -1,12 +1,30 @@ package edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.dta; -import java.io.*; -import java.util.logging.*; - -import java.util.*; -import java.text.*; - -import org.apache.commons.lang.*; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; +import java.util.Set; +import java.util.TimeZone; +import java.util.logging.Logger; import edu.harvard.iq.dataverse.DataTable; import edu.harvard.iq.dataverse.datavariable.DataVariable; @@ -15,6 +33,7 @@ import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataFileReader; import edu.harvard.iq.dataverse.ingest.tabulardata.spi.TabularDataFileReaderSpi; import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataIngest; +import org.apache.commons.lang3.StringUtils; /** * ingest plugin for Stata 13-15 (117-119) DTA file format. A copy and paste from diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/por/PORFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/por/PORFileReader.java index a39b044e1b3..c90b0ea6950 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/por/PORFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/por/PORFileReader.java @@ -19,19 +19,45 @@ */ package edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por; -import java.io.*; -import java.nio.*; -import java.util.logging.*; - -import java.util.*; -import java.util.regex.*; -import java.text.*; +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.Writer; +import java.nio.ByteBuffer; + +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; +import java.util.Set; +import java.util.TimeZone; +import java.util.logging.Logger; import java.math.BigDecimal; import java.math.MathContext; import java.math.RoundingMode; +import java.util.regex.Matcher; +import java.util.regex.Pattern; -import org.apache.commons.lang.*; import org.apache.commons.codec.binary.Hex; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; import edu.harvard.iq.dataverse.DataTable; import edu.harvard.iq.dataverse.datavariable.DataVariable; @@ -44,7 +70,6 @@ import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SPSSConstants; - /** * ingest plugin for SPSS/POR ("portable") file format. * diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RDATAFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RDATAFileReader.java index b2da2f12ff6..c2899b29d1f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RDATAFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RDATAFileReader.java @@ -52,8 +52,8 @@ import javax.naming.NamingException; -import org.apache.commons.lang.RandomStringUtils; -import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.ArrayUtils; /** * Dataverse 4.0 implementation of TabularDataFileReader for the * RData Binary Format. diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RTabFileParser.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RTabFileParser.java index d5ca06a96f3..f60b7733463 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RTabFileParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RTabFileParser.java @@ -24,7 +24,7 @@ import java.util.logging.*; import edu.harvard.iq.dataverse.util.BundleUtil; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import edu.harvard.iq.dataverse.DataTable; import edu.harvard.iq.dataverse.datavariable.DataVariable; diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/sav/SAVFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/sav/SAVFileReader.java index 53441c0de65..682b8f1166c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/sav/SAVFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/sav/SAVFileReader.java @@ -19,17 +19,42 @@ */ package edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav; -import java.io.*; -import java.nio.*; -import java.util.logging.*; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; -import java.util.*; -import java.util.regex.*; -import java.text.*; - -import org.apache.commons.lang.*; import org.apache.commons.codec.binary.Hex; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; import edu.harvard.iq.dataverse.DataTable; import edu.harvard.iq.dataverse.datavariable.DataVariable; @@ -40,8 +65,7 @@ import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataFileReader; import edu.harvard.iq.dataverse.ingest.tabulardata.spi.TabularDataFileReaderSpi; import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataIngest; -import edu.harvard.iq.dataverse.ingest.tabulardata.InvalidData; - +import edu.harvard.iq.dataverse.ingest.tabulardata.InvalidData; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/xlsx/XLSXFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/xlsx/XLSXFileReader.java index 5a7642126fe..914e8d56432 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/xlsx/XLSXFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/xlsx/XLSXFileReader.java @@ -33,7 +33,7 @@ import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataIngest; import edu.harvard.iq.dataverse.util.BundleUtil; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.poi.xssf.eventusermodel.XSSFReader; import org.apache.poi.xssf.usermodel.XSSFRichTextString; diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java index 523d06bb8a1..ec3a1f357f8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java @@ -39,7 +39,7 @@ import javax.ws.rs.QueryParam; import edu.harvard.iq.dataverse.util.BundleUtil; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java index a1e94e01815..0e99220005c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java @@ -20,7 +20,7 @@ import javax.json.Json; import javax.json.JsonArrayBuilder; import javax.json.JsonObjectBuilder; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java index c9faf797d57..eaea5ab2296 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java @@ -21,7 +21,7 @@ import javax.json.Json; import javax.json.JsonArrayBuilder; import javax.json.JsonObjectBuilder; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * Given a user and a set of filters (dvobject type, roles, publication status): diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/RoleTagRetriever.java b/src/main/java/edu/harvard/iq/dataverse/mydata/RoleTagRetriever.java index 9f03f8addb1..4556c92ff19 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/RoleTagRetriever.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/RoleTagRetriever.java @@ -25,7 +25,7 @@ import java.util.logging.Logger; import javax.json.Json; import javax.json.JsonArrayBuilder; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * Input: dvObject id, parent Id, and dvObject type (from Solr) diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/SolrQueryFormatter.java b/src/main/java/edu/harvard/iq/dataverse/mydata/SolrQueryFormatter.java index 9aefbee240c..1c3fca1537b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/SolrQueryFormatter.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/SolrQueryFormatter.java @@ -12,7 +12,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * Convenience methods for formatting long arrays of ids into solrQuery strings diff --git a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java index aea910c496e..e0ab786b68b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java @@ -29,7 +29,7 @@ import java.util.List; import javax.faces.component.UIComponent; import javax.faces.component.UIInput; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.hibernate.validator.constraints.NotBlank; @ViewScoped diff --git a/src/main/java/edu/harvard/iq/dataverse/rserve/RJobRequest.java b/src/main/java/edu/harvard/iq/dataverse/rserve/RJobRequest.java index d59e8f1050f..9171b8313ff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/rserve/RJobRequest.java +++ b/src/main/java/edu/harvard/iq/dataverse/rserve/RJobRequest.java @@ -25,9 +25,15 @@ * @author Leonid Andreev */ import edu.harvard.iq.dataverse.datavariable.DataVariable; -import java.util.*; -import java.util.logging.*; -import org.apache.commons.lang.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +import org.apache.commons.lang3.StringUtils; public class RJobRequest { diff --git a/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java b/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java index 503125eb45a..f13b6f11434 100644 --- a/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java +++ b/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java @@ -23,16 +23,31 @@ import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.dataaccess.DataAccessRequest; -import java.io.*; -import java.util.*; -import java.util.logging.*; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; + import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; import org.rosuda.REngine.*; import org.rosuda.REngine.Rserve.*; -import org.apache.commons.lang.*; - /** * * @author Leonid Andreev diff --git a/src/main/java/edu/harvard/iq/dataverse/rserve/VariableNameCheckerForR.java b/src/main/java/edu/harvard/iq/dataverse/rserve/VariableNameCheckerForR.java index dea0c886cd1..65a64241364 100644 --- a/src/main/java/edu/harvard/iq/dataverse/rserve/VariableNameCheckerForR.java +++ b/src/main/java/edu/harvard/iq/dataverse/rserve/VariableNameCheckerForR.java @@ -20,10 +20,14 @@ package edu.harvard.iq.dataverse.rserve; -import org.apache.commons.lang.*; -import java.util.*; -import java.util.regex.*; +import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/search/AdvancedSearchPage.java b/src/main/java/edu/harvard/iq/dataverse/search/AdvancedSearchPage.java index f3af7bf90c7..55516d68880 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/AdvancedSearchPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/AdvancedSearchPage.java @@ -23,7 +23,7 @@ import javax.faces.view.ViewScoped; import javax.inject.Inject; import javax.inject.Named; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; @ViewScoped @Named("AdvancedSearchPage") diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index f61f879eee7..bfc2af2fcc4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -61,7 +61,7 @@ import javax.inject.Named; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.SortClause; diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index f7a3b874c24..09ab372c687 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -39,7 +39,7 @@ import javax.inject.Inject; import javax.inject.Named; import javax.servlet.http.HttpServletRequest; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; //@ViewScoped @RequestScoped diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java index a8e013f3ca3..c226d77f885 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java @@ -5,7 +5,7 @@ import java.util.ArrayList; import java.util.List; import java.util.StringTokenizer; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.solr.common.SolrInputDocument; public class SearchUtil { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileSortFieldAndOrder.java b/src/main/java/edu/harvard/iq/dataverse/util/FileSortFieldAndOrder.java index ba66c404349..ef921ff185c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileSortFieldAndOrder.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileSortFieldAndOrder.java @@ -1,7 +1,7 @@ package edu.harvard.iq.dataverse.util; import edu.harvard.iq.dataverse.search.SortBy; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; public class FileSortFieldAndOrder { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index da7eece7826..3ec698cfc64 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -98,7 +98,7 @@ import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker; import java.util.Arrays; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; /** * a 4.0 implementation of the DVN FileUtil; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java b/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java index 7b096786351..e08f045d6cb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java @@ -5,7 +5,7 @@ */ package edu.harvard.iq.dataverse.util; -import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang3.StringEscapeUtils; import org.jsoup.Jsoup; import org.jsoup.safety.Whitelist; import org.jsoup.parser.Parser; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SumStatCalculator.java b/src/main/java/edu/harvard/iq/dataverse/util/SumStatCalculator.java index ee37bd51206..545f2937f70 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SumStatCalculator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SumStatCalculator.java @@ -19,13 +19,13 @@ */ package edu.harvard.iq.dataverse.util; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.logging.Logger; -import org.apache.commons.lang.*; -import org.apache.commons.math.stat.*; -//import cern.colt.list.*; -//import cern.jet.stat.Descriptive; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.math.stat.StatUtils; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/validation/PasswordValidatorUtil.java b/src/main/java/edu/harvard/iq/dataverse/validation/PasswordValidatorUtil.java index 6d78ab44a5f..9a4f1610e62 100644 --- a/src/main/java/edu/harvard/iq/dataverse/validation/PasswordValidatorUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/validation/PasswordValidatorUtil.java @@ -5,7 +5,7 @@ import java.util.Arrays; import java.util.List; import java.util.logging.Logger; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.passay.CharacterRule; import org.passay.EnglishCharacterData; From 9b33f880c51b55412c82cf4d23c8e3628329fe94 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 14 Apr 2021 15:48:57 +0200 Subject: [PATCH 136/354] refactor(deps): move test Java classed from Apache Commons Lang (v2) to Lang3 (v3). #6070 --- src/test/java/edu/harvard/iq/dataverse/DataCitationTest.java | 2 +- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 +- .../groups/impl/maildomain/MailDomainGroupTest.java | 2 +- .../edu/harvard/iq/dataverse/mydata/SolrQueryFormatterTest.java | 2 +- .../harvard/iq/dataverse/validation/PasswordValidatorTest.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/DataCitationTest.java b/src/test/java/edu/harvard/iq/dataverse/DataCitationTest.java index b32243d446d..e6e2fecf5af 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DataCitationTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DataCitationTest.java @@ -1,6 +1,6 @@ package edu.harvard.iq.dataverse; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.Test; import java.io.ByteArrayOutputStream; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 04d7c77d4fb..a7dc41d9c74 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -26,7 +26,7 @@ import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import java.util.UUID; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import com.jayway.restassured.parsing.Parser; import static com.jayway.restassured.path.json.JsonPath.with; import com.jayway.restassured.path.xml.XmlPath; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index d00045679f9..629756f1e36 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -16,7 +16,7 @@ import java.util.logging.Level; import edu.harvard.iq.dataverse.api.datadeposit.SwordConfigurationImpl; import com.jayway.restassured.path.xml.XmlPath; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.Test; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import com.jayway.restassured.specification.RequestSpecification; diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/groups/impl/maildomain/MailDomainGroupTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/groups/impl/maildomain/MailDomainGroupTest.java index 2014318232a..40c4ec8a904 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/groups/impl/maildomain/MailDomainGroupTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/groups/impl/maildomain/MailDomainGroupTest.java @@ -1,7 +1,7 @@ package edu.harvard.iq.dataverse.authorization.groups.impl.maildomain; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.Test; import org.mockito.Mockito; diff --git a/src/test/java/edu/harvard/iq/dataverse/mydata/SolrQueryFormatterTest.java b/src/test/java/edu/harvard/iq/dataverse/mydata/SolrQueryFormatterTest.java index 22faa57831d..c15bc280316 100644 --- a/src/test/java/edu/harvard/iq/dataverse/mydata/SolrQueryFormatterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/mydata/SolrQueryFormatterTest.java @@ -17,7 +17,7 @@ import java.util.Random; import java.util.Set; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.Test; import org.junit.experimental.runners.Enclosed; import org.junit.runner.RunWith; diff --git a/src/test/java/edu/harvard/iq/dataverse/validation/PasswordValidatorTest.java b/src/test/java/edu/harvard/iq/dataverse/validation/PasswordValidatorTest.java index 27a88260870..bbca418fe7d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/validation/PasswordValidatorTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/validation/PasswordValidatorTest.java @@ -1,6 +1,6 @@ package edu.harvard.iq.dataverse.validation; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; From eca8b8fd403002327439f81f54c6ed9edde391c7 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 14 Apr 2021 16:22:22 +0200 Subject: [PATCH 137/354] fix(deps): move to Apache Commons Text for escapeHtml() and escapeXml() #6070 Apache Commons Lang3 3.12.0 moved `StringEscapeUtils` to Apache Commons Text. Also, escapeHtml() and escapeXml() from Apache Commons Lang (v2) have been renamed to escapeHtml4() and escapeXml10() with v3.0 --- pom.xml | 6 ++++++ .../harvard/iq/dataverse/DOIDataCiteRegisterService.java | 6 +++--- src/main/java/edu/harvard/iq/dataverse/DataCitation.java | 6 +++--- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 6 +++--- src/main/java/edu/harvard/iq/dataverse/DataversePage.java | 8 ++++---- src/main/java/edu/harvard/iq/dataverse/Guestbook.java | 2 +- .../edu/harvard/iq/dataverse/ManagePermissionsPage.java | 6 +++--- .../edu/harvard/iq/dataverse/RolePermissionFragment.java | 4 ++-- .../java/edu/harvard/iq/dataverse/util/MarkupChecker.java | 4 ++-- .../edu/harvard/iq/dataverse/workflows/WorkflowUtil.java | 2 +- src/main/webapp/dataset-license-terms.xhtml | 2 +- src/main/webapp/dataverse_header.xhtml | 6 +++--- src/main/webapp/manage-guestbooks.xhtml | 2 +- 13 files changed, 33 insertions(+), 27 deletions(-) diff --git a/pom.xml b/pom.xml index 8e4e1743a1a..5b910c3e33d 100644 --- a/pom.xml +++ b/pom.xml @@ -327,6 +327,12 @@ org.apache.commons commons-lang3 + + + org.apache.commons + commons-text + 1.9 + org.apache.solr solr-solrj diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java index 1fc8564e389..18b98833bef 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java @@ -25,7 +25,7 @@ import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import javax.persistence.TypedQuery; -import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.text.StringEscapeUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -186,7 +186,7 @@ public static String getMetadataFromDvObject(String identifier, Map" + StringEscapeUtils.escapeHtml(text) + ""; + return "" + StringEscapeUtils.escapeHtml4(text) + ""; } else { return text; } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 6e89ed2814a..9b2994c3d6d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -114,7 +114,7 @@ import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletResponse; -import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.text.StringEscapeUtils; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.io.IOUtils; @@ -3078,8 +3078,8 @@ public void updateFileCounts(){ private List getSuccessMessageArguments() { List arguments = new ArrayList<>(); String dataverseString = ""; - arguments.add(StringEscapeUtils.escapeHtml(dataset.getDisplayName())); - dataverseString += " " + StringEscapeUtils.escapeHtml(selectedDataverseForLinking.getDisplayName()) + ""; + arguments.add(StringEscapeUtils.escapeHtml4(dataset.getDisplayName())); + dataverseString += " " + StringEscapeUtils.escapeHtml4(selectedDataverseForLinking.getDisplayName()) + ""; arguments.add(dataverseString); return arguments; } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index 84f3d793303..1e2d3f507a1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -52,7 +52,7 @@ import javax.ejb.EJBException; import javax.faces.event.ValueChangeEvent; import javax.faces.model.SelectItem; -import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.text.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; import org.primefaces.PrimeFaces; import org.primefaces.event.TransferEvent; @@ -805,8 +805,8 @@ public String saveLinkedDataverse() { private List getSuccessMessageArguments() { List arguments = new ArrayList<>(); - arguments.add(StringEscapeUtils.escapeHtml(dataverse.getDisplayName())); - String linkString = "" + StringEscapeUtils.escapeHtml(linkingDataverse.getDisplayName()) + ""; + arguments.add(StringEscapeUtils.escapeHtml4(dataverse.getDisplayName())); + String linkString = "" + StringEscapeUtils.escapeHtml4(linkingDataverse.getDisplayName()) + ""; arguments.add(linkString); return arguments; } @@ -867,7 +867,7 @@ public String saveSavedSearch() { commandEngine.submit(cmd); List arguments = new ArrayList<>(); - String linkString = "" + StringEscapeUtils.escapeHtml(linkingDataverse.getDisplayName()) + ""; + String linkString = "" + StringEscapeUtils.escapeHtml4(linkingDataverse.getDisplayName()) + ""; arguments.add(linkString); String successMessageString = BundleUtil.getStringFromBundle("dataverse.saved.search.success", arguments); JsfHelper.addSuccessMessage(successMessageString); diff --git a/src/main/java/edu/harvard/iq/dataverse/Guestbook.java b/src/main/java/edu/harvard/iq/dataverse/Guestbook.java index 1f25930a4b5..18913bfd5bf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Guestbook.java +++ b/src/main/java/edu/harvard/iq/dataverse/Guestbook.java @@ -23,7 +23,7 @@ import javax.persistence.Transient; import edu.harvard.iq.dataverse.util.DateUtil; -import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.text.StringEscapeUtils; import org.hibernate.validator.constraints.NotBlank; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index 90f441cadfe..4a734c11941 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -41,7 +41,7 @@ import javax.inject.Named; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.text.StringEscapeUtils; /** * @@ -516,7 +516,7 @@ private void assignRole(RoleAssignee ra, DataverseRole r) { List args = Arrays.asList( r.getName(), ra.getDisplayInfo().getTitle(), - StringEscapeUtils.escapeHtml(dvObject.getDisplayName()) + StringEscapeUtils.escapeHtml4(dvObject.getDisplayName()) ); JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("permission.roleAssignedToFor", args)); // don't notify if role = file downloader and object is not released @@ -530,7 +530,7 @@ private void assignRole(RoleAssignee ra, DataverseRole r) { List args = Arrays.asList( r.getName(), ra.getDisplayInfo().getTitle(), - StringEscapeUtils.escapeHtml(dvObject.getDisplayName()) + StringEscapeUtils.escapeHtml4(dvObject.getDisplayName()) ); String message = BundleUtil.getStringFromBundle("permission.roleNotAssignedFor", args); JsfHelper.addErrorMessage(message); diff --git a/src/main/java/edu/harvard/iq/dataverse/RolePermissionFragment.java b/src/main/java/edu/harvard/iq/dataverse/RolePermissionFragment.java index b5683b625cd..dd3044d3749 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RolePermissionFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/RolePermissionFragment.java @@ -36,7 +36,7 @@ import javax.persistence.PersistenceContext; import edu.harvard.iq.dataverse.util.BundleUtil; -import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.text.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; /** @@ -193,7 +193,7 @@ private void assignRole(RoleAssignee ra, DataverseRole r) { commandEngine.submit(new AssignRoleCommand(ra, r, dvObject, dvRequestService.getDataverseRequest(), privateUrlToken)); JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("permission.roleAssignedToOn" , - Arrays.asList( r.getName() , ra.getDisplayInfo().getTitle() , StringEscapeUtils.escapeHtml(dvObject.getDisplayName()) )) ); + Arrays.asList( r.getName() , ra.getDisplayInfo().getTitle() , StringEscapeUtils.escapeHtml4(dvObject.getDisplayName()) )) ); } catch (CommandException ex) { JH.addMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("permission.cannotAssignRole" , Arrays.asList( ex.getMessage()))); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java b/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java index e08f045d6cb..3131afbf010 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java @@ -5,7 +5,7 @@ */ package edu.harvard.iq.dataverse.util; -import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.text.StringEscapeUtils; import org.jsoup.Jsoup; import org.jsoup.safety.Whitelist; import org.jsoup.parser.Parser; @@ -60,7 +60,7 @@ public static String stripAllTags(String unsafe) { } public static String escapeHtml(String unsafe) { - return StringEscapeUtils.escapeHtml(unsafe); + return StringEscapeUtils.escapeHtml4(unsafe); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java index aeb8bcf6c87..e6e6bfd23c8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java @@ -19,7 +19,7 @@ import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; -import static org.apache.commons.lang3.StringEscapeUtils.escapeHtml4; +import static org.apache.commons.text.StringEscapeUtils.escapeHtml4; public class WorkflowUtil { diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index 924f07c14a0..c5bdc8638cf 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -24,7 +24,7 @@
- +
- + + diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index 0e4931e7086..c1a649cff87 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -87,14 +87,14 @@
- +
-
+
diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 74e1f531234..380e64e6402 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -52,7 +52,7 @@ - + + + + diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java index 348f3e7b2dd..46c372ad313 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public class CreatePrivateUrlCommandTest { @@ -97,7 +98,7 @@ public void testDatasetNull() { String actual = null; PrivateUrl privateUrl = null; try { - privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset)); + privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset, false)); } catch (CommandException ex) { actual = ex.getMessage(); } @@ -112,7 +113,7 @@ public void testAlreadyExists() { String actual = null; PrivateUrl privateUrl = null; try { - privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset)); + privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset, false)); } catch (CommandException ex) { actual = ex.getMessage(); } @@ -133,7 +134,7 @@ public void testAttemptCreatePrivateUrlOnNonDraft() { String actual = null; PrivateUrl privateUrl = null; try { - privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset)); + privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset, false)); } catch (CommandException ex) { actual = ex.getMessage(); } @@ -145,7 +146,7 @@ public void testAttemptCreatePrivateUrlOnNonDraft() { public void testCreatePrivateUrlSuccessfully() throws CommandException { dataset = new Dataset(); dataset.setId(createDatasetLong); - PrivateUrl privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset)); + PrivateUrl privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset, false)); assertNotNull(privateUrl); assertNotNull(privateUrl.getDataset()); assertNotNull(privateUrl.getRoleAssignment()); @@ -157,5 +158,23 @@ public void testCreatePrivateUrlSuccessfully() throws CommandException { assertNotNull(privateUrl.getToken()); assertEquals("https://dataverse.example.edu/privateurl.xhtml?token=" + privateUrl.getToken(), privateUrl.getLink()); } + + @Test + public void testCreateAnonymizedAccessPrivateUrlSuccessfully() throws CommandException { + dataset = new Dataset(); + dataset.setId(createDatasetLong); + PrivateUrl privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset, true)); + assertNotNull(privateUrl); + assertNotNull(privateUrl.getDataset()); + assertNotNull(privateUrl.getRoleAssignment()); + PrivateUrlUser expectedUser = new PrivateUrlUser(dataset.getId()); + assertEquals(expectedUser.getIdentifier(), privateUrl.getRoleAssignment().getAssigneeIdentifier()); + assertEquals(expectedUser.isSuperuser(), false); + assertEquals(expectedUser.isAuthenticated(), false); + assertEquals(expectedUser.getDisplayInfo().getTitle(), "Private URL Enabled"); + assertNotNull(privateUrl.getToken()); + assertTrue(privateUrl.isAnonymizedAccess()); + assertEquals("https://dataverse.example.edu/privateurl.xhtml?token=" + privateUrl.getToken(), privateUrl.getLink()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java index 3cb719b5c95..3984f972308 100644 --- a/src/test/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java @@ -312,7 +312,7 @@ public void testGetPrivateUrlUserFromRoleAssignmentAndAssigneeSuccess() { @Test public void testGetRequiredPermissions() { - CreatePrivateUrlCommand createPrivateUrlCommand = new CreatePrivateUrlCommand(null, null); + CreatePrivateUrlCommand createPrivateUrlCommand = new CreatePrivateUrlCommand(null, null, false); CommandException ex = new CommandException(null, createPrivateUrlCommand); List strings = PrivateUrlUtil.getRequiredPermissions(ex); assertEquals(Arrays.asList("ManageDatasetPermissions"), strings); From ec52ab10e87ddeecb39463fbd7450a4568d01f56 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 27 May 2021 13:13:44 -0400 Subject: [PATCH 159/354] also disable metadata downloads. --- src/main/webapp/dataset.xhtml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 72c4350ce82..43b117762d3 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -798,7 +798,8 @@
+ or (DatasetPage.workingVersion.deaccessioned and DatasetPage.canUpdateDataset())) + and !DatasetPage.anonymizedAccess}"> From 9a6e5d9b8001e0292b581e8e51f05d0c9053450b Mon Sep 17 00:00:00 2001 From: pkiraly Date: Thu, 27 May 2021 19:44:56 +0200 Subject: [PATCH 160/354] #7906 add integration tests. --- src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index fc2ecd2955e..e0d66aa62b3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -71,10 +71,15 @@ public void testListAuthenticatedUsers() throws Exception { Response deleteNonSuperuser = UtilIT.deleteUser(nonSuperuserUsername); assertEquals(200, deleteNonSuperuser.getStatusCode()); + assertEquals( + "{\"status\":\"OK\",\"data\":{\"message\":\"AuthenticatedUser @" + nonSuperuserUsername + " deleted.\"}}", + deleteNonSuperuser.getBody().asString()); Response deleteSuperuser = UtilIT.deleteUser(superuserUsername); assertEquals(200, deleteSuperuser.getStatusCode()); - + assertEquals( + "{\"status\":\"OK\",\"data\":{\"message\":\"AuthenticatedUser @" + superuserUsername + " deleted.\"}}", + deleteSuperuser.getBody().asString()); } From f1e726eacd2c4114ee78b47ffa505d5de4754327 Mon Sep 17 00:00:00 2001 From: pkiraly Date: Thu, 27 May 2021 20:45:57 +0200 Subject: [PATCH 161/354] #7906 remove integration tests. --- src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index e0d66aa62b3..c34ee2dd4bf 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -71,17 +71,10 @@ public void testListAuthenticatedUsers() throws Exception { Response deleteNonSuperuser = UtilIT.deleteUser(nonSuperuserUsername); assertEquals(200, deleteNonSuperuser.getStatusCode()); - assertEquals( - "{\"status\":\"OK\",\"data\":{\"message\":\"AuthenticatedUser @" + nonSuperuserUsername + " deleted.\"}}", - deleteNonSuperuser.getBody().asString()); Response deleteSuperuser = UtilIT.deleteUser(superuserUsername); assertEquals(200, deleteSuperuser.getStatusCode()); - assertEquals( - "{\"status\":\"OK\",\"data\":{\"message\":\"AuthenticatedUser @" + superuserUsername + " deleted.\"}}", - deleteSuperuser.getBody().asString()); } - @Test public void testFilterAuthenticatedUsersForbidden() throws Exception { From 8905d8dec04e392dbb05ace8a2e1eebd3429e5a7 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 27 May 2021 15:23:41 -0400 Subject: [PATCH 162/354] #7767 download guestbook responses api --- .../GuestbookResponseServiceBean.java | 49 ++++++++----------- .../harvard/iq/dataverse/api/Dataverses.java | 48 ++++++++++++++++++ 2 files changed, 69 insertions(+), 28 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java index 809417e3f9c..701ee6aa7f2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java @@ -128,6 +128,18 @@ public void streamResponsesByDataverseIdAndGuestbookId(OutputStream out, Long da // of queries now) -- L.A. Map customQandAs = mapCustomQuestionAnswersAsStrings(dataverseId, guestbookId); + + List guestbookResults = getGuestbookResults( dataverseId, guestbookId ); + // the CSV header: + out.write("Guestbook, Dataset, Dataset PID, Date, Type, File Name, File Id, File PID, User Name, Email, Institution, Position, Custom Questions\n".getBytes()); + for (Object[] result : guestbookResults) { + StringBuilder sb = convertGuestbookResponsesToCSV(customQandAs, result); + out.write(sb.toString().getBytes()); + out.flush(); + } + } + + public List getGuestbookResults(Long dataverseId, Long guestbookId ){ String queryString = BASE_QUERY_STRING_FOR_DOWNLOAD_AS_CSV + " and o.owner_id = " @@ -140,12 +152,12 @@ public void streamResponsesByDataverseIdAndGuestbookId(OutputStream out, Long da queryString += ";"; logger.fine("stream responses query: " + queryString); - List guestbookResults = em.createNativeQuery(queryString).getResultList(); - - // the CSV header: - out.write("Guestbook, Dataset, Dataset PID, Date, Type, File Name, File Id, File PID, User Name, Email, Institution, Position, Custom Questions\n".getBytes()); + return em.createNativeQuery(queryString).getResultList(); - for (Object[] result : guestbookResults) { + } + + public StringBuilder convertGuestbookResponsesToCSV ( Map customQandAs, Object[] result) throws IOException { + Integer guestbookResponseId = (Integer)result[0]; StringBuilder sb = new StringBuilder(); @@ -208,36 +220,17 @@ public void streamResponsesByDataverseIdAndGuestbookId(OutputStream out, Long da // Finally, custom questions and answers, if present: - // (the old implementation, below, would run one extra query FOR EVERY SINGLE - // guestbookresponse entry! -- instead, we are now pre-caching all the - // available custom question responses, with a single native query at - // the top of this method. -- L.A.) - - /*String cqString = "select q.questionstring, r.response from customquestionresponse r, customquestion q where q.id = r.customquestion_id and r.guestbookResponse_id = " + result[0]; - List customResponses = em.createNativeQuery(cqString).getResultList(); - if (customResponses != null) { - for (Object[] response : customResponses) { - sb.append(SEPARATOR); - sb.append(response[0]); - sb.append(SEPARATOR); - sb.append(response[1] == null ? "" : response[1]); - } - }*/ if (customQandAs.containsKey(guestbookResponseId)) { sb.append(customQandAs.get(guestbookResponseId)); } sb.append(NEWLINE); - - // Finally, write the line out: - // (i.e., we are writing one guestbook response at a time, thus allowing the - // whole thing to stream in real time -- L.A.) - out.write(sb.toString().getBytes()); - out.flush(); - } + return sb; + } + private String formatPersistentIdentifier(String protocol, String authority, String identifier) { // Note that the persistent id may be unavailable for this dvObject: if (StringUtil.nonEmpty(protocol) && StringUtil.nonEmpty(authority) && StringUtil.nonEmpty(identifier)) { @@ -349,7 +342,7 @@ private Map mapCustomQuestionAnswersAsLists(Long dataverseId, L return selectCustomQuestionAnswers(dataverseId, guestbookId, false, firstResponse, lastResponse); } - private Map mapCustomQuestionAnswersAsStrings(Long dataverseId, Long guestbookId) { + public Map mapCustomQuestionAnswersAsStrings(Long dataverseId, Long guestbookId) { return selectCustomQuestionAnswers(dataverseId, guestbookId, true, null, null); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 2c73c5ad36e..2498641045d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -11,6 +11,8 @@ import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; +import edu.harvard.iq.dataverse.GuestbookServiceBean; import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.RoleAssignment; import static edu.harvard.iq.dataverse.api.AbstractApiBean.error; @@ -98,10 +100,15 @@ import javax.ws.rs.core.Response.Status; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.toJsonArray; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; import java.text.MessageFormat; import java.util.Arrays; import java.util.Date; +import java.util.Map; import java.util.Optional; +import javax.ws.rs.core.Response.ResponseBuilder; import javax.xml.stream.XMLStreamException; /** @@ -123,6 +130,12 @@ public class Dataverses extends AbstractApiBean { @EJB SettingsServiceBean settingsService; + + @EJB + GuestbookResponseServiceBean guestbookResponseService; + + @EJB + GuestbookServiceBean guestbookService; @POST public Response addRoot(String body) { @@ -832,6 +845,41 @@ public Response getGroupByOwnerAndAliasInOwner(@PathParam("identifier") String d req, grpAliasInOwner)))); } + + @GET + @Path("{identifier}/guestbookResponses/") + @Produces({"application/download"}) + public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String dvIdtf, + @QueryParam("guestbookId") Long gbId) { + Dataverse dv = findDataverse(dvIdtf); + + try { + + String filename = dv.getAlias() + "GBResponses.csv"; + String contentDispositionString = "attachment;filename=" + filename; + File file = new File(filename); + System.out.print("filename: " + filename); + ResponseBuilder response = Response.ok((Object) file); + response.header("Content-Disposition", contentDispositionString); + FileOutputStream outputStream = new FileOutputStream(filename); + Map customQandAs = guestbookResponseService.mapCustomQuestionAnswersAsStrings(dv.getId(), gbId); + + List guestbookResults = guestbookResponseService.getGuestbookResults(dv.getId(), gbId); + outputStream.write("Guestbook, Dataset, Dataset PID, Date, Type, File Name, File Id, File PID, User Name, Email, Institution, Position, Custom Questions\n".getBytes()); + for (Object[] result : guestbookResults) { + StringBuilder sb = guestbookResponseService.convertGuestbookResponsesToCSV(customQandAs, result); + outputStream.write(sb.toString().getBytes()); + outputStream.flush(); + } + + return response.build(); + } catch (IOException io) { + return error(Status.BAD_REQUEST, "exception: " + io.getMessage()); + + } + + // return error(Status.BAD_REQUEST, "filler"); + } @PUT @Path("{identifier}/groups/{aliasInOwner}") From 8b51789e76c4d072e0d4abde88ccf34a0b947ea0 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 28 May 2021 15:22:56 +0200 Subject: [PATCH 163/354] chore: update release note on Payara upgrade to Dataverse 5.6 --- doc/release-notes/7700-upgrade-payara.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/7700-upgrade-payara.md b/doc/release-notes/7700-upgrade-payara.md index 45c75d6c0f1..7f76d993b18 100644 --- a/doc/release-notes/7700-upgrade-payara.md +++ b/doc/release-notes/7700-upgrade-payara.md @@ -5,7 +5,7 @@ Some changes in this release require an upgrade to Payara 5.2021.3 or higher. Instructions on how to update can be found in the [Payara documentation](https://docs.payara.fish/community/docs/5.2021.3/documentation/user-guides/upgrade-payara.html) -It would likely be safer to upgrade Payara first, while still running Dataverse 5.4, and then proceed with the steps +It would likely be safer to upgrade Payara first, while still running Dataverse 5.6, and then proceed with the steps below. Upgrading from an earlier version of Payara should be a straightforward process: 1. Undeploy Dataverse @@ -13,7 +13,7 @@ below. Upgrading from an earlier version of Payara should be a straightforward p 3. Move the current Payara directory out of the way 4. Unzip the new Payara version in its place 5. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved `domain1` -6. Start Payara, deploy Dataverse 5.4. +6. Start Payara, deploy Dataverse 5.6. We still recommend that you read the detailed upgrade instructions above and if you run into any issues with this upgrade, it will help to be able to separate them from any problems with the upgrade of Dataverse proper. From 749a33ed2a1bdc9ee2fbef80d64548c11f25a2e4 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 18 May 2021 16:06:34 +0200 Subject: [PATCH 164/354] feat(testing): add reviewdog GH action linting PRs with Maven Checkstyle plugin. #6070 --- .github/workflows/reviewdog_checkstyle.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/reviewdog_checkstyle.yml diff --git a/.github/workflows/reviewdog_checkstyle.yml b/.github/workflows/reviewdog_checkstyle.yml new file mode 100644 index 00000000000..90a0dd7d06b --- /dev/null +++ b/.github/workflows/reviewdog_checkstyle.yml @@ -0,0 +1,21 @@ +name: Maven CheckStyle Task +on: + pull_request: + paths: + - "**.java" + +jobs: + checkstyle_job: + runs-on: ubuntu-latest + name: Checkstyle job + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Run check style + uses: nikitasavinov/checkstyle-action@master + with: + fail_on_error: true + reporter: github-pr-review + checkstyle_config: checkstyle.xml + github_token: ${{ secrets.GITHUB_TOKEN }} + From 5de75201457a52e4eaaa8c0b488112fd54d62c2c Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Fri, 28 May 2021 10:59:25 -0400 Subject: [PATCH 165/354] 7910 declare instream inside try-with-resources --- .../dataverse/api/DownloadInstanceWriter.java | 141 +++++++++--------- 1 file changed, 71 insertions(+), 70 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index 7f3b652af33..a0377faede2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -385,91 +385,92 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] } - InputStream instream = storageIO.getInputStream(); - if (instream != null) { - // headers: - - String fileName = storageIO.getFileName(); - String mimeType = storageIO.getMimeType(); - - // Provide both the "Content-disposition" and "Content-Type" headers, - // to satisfy the widest selection of browsers out there. - // Encode the filename as UTF-8, then deal with spaces. "encode" changes - // a space to + so we change it back to a space (%20). - String finalFileName = URLEncoder.encode(fileName, "UTF-8").replaceAll("\\+", "%20"); - httpHeaders.add("Content-disposition", "attachment; filename=\"" + finalFileName + "\""); - httpHeaders.add("Content-Type", mimeType + "; name=\"" + finalFileName + "\""); - - long contentSize; - boolean useChunkedTransfer = false; - //if ((contentSize = getFileSize(di, storageIO.getVarHeader())) > 0) { - if ((contentSize = getContentSize(storageIO)) > 0) { - logger.fine("Content size (retrieved from the AccessObject): " + contentSize); - httpHeaders.add("Content-Length", contentSize); - } else { - //httpHeaders.add("Transfer-encoding", "chunked"); - //useChunkedTransfer = true; - } + try (InputStream instream = storageIO.getInputStream()) { + if (instream != null) { + // headers: + + String fileName = storageIO.getFileName(); + String mimeType = storageIO.getMimeType(); + + // Provide both the "Content-disposition" and "Content-Type" headers, + // to satisfy the widest selection of browsers out there. + // Encode the filename as UTF-8, then deal with spaces. "encode" changes + // a space to + so we change it back to a space (%20). + String finalFileName = URLEncoder.encode(fileName, "UTF-8").replaceAll("\\+", "%20"); + httpHeaders.add("Content-disposition", "attachment; filename=\"" + finalFileName + "\""); + httpHeaders.add("Content-Type", mimeType + "; name=\"" + finalFileName + "\""); + + long contentSize; + boolean useChunkedTransfer = false; + //if ((contentSize = getFileSize(di, storageIO.getVarHeader())) > 0) { + if ((contentSize = getContentSize(storageIO)) > 0) { + logger.fine("Content size (retrieved from the AccessObject): " + contentSize); + httpHeaders.add("Content-Length", contentSize); + } else { + //httpHeaders.add("Transfer-encoding", "chunked"); + //useChunkedTransfer = true; + } + + // (the httpHeaders map must be modified *before* writing any + // data in the output stream!) + int bufsize; + byte[] bffr = new byte[4 * 8192]; + byte[] chunkClose = "\r\n".getBytes(); + + // before writing out any bytes from the input stream, flush + // any extra content, such as the variable header for the + // subsettable files: + if (storageIO.getVarHeader() != null) { + if (storageIO.getVarHeader().getBytes().length > 0) { + if (useChunkedTransfer) { + String chunkSizeLine = String.format("%x\r\n", storageIO.getVarHeader().getBytes().length); + outstream.write(chunkSizeLine.getBytes()); + } + outstream.write(storageIO.getVarHeader().getBytes()); + if (useChunkedTransfer) { + outstream.write(chunkClose); + } + } + } - // (the httpHeaders map must be modified *before* writing any - // data in the output stream!) - int bufsize; - byte[] bffr = new byte[4 * 8192]; - byte[] chunkClose = "\r\n".getBytes(); - - // before writing out any bytes from the input stream, flush - // any extra content, such as the variable header for the - // subsettable files: - if (storageIO.getVarHeader() != null) { - if (storageIO.getVarHeader().getBytes().length > 0) { + while ((bufsize = instream.read(bffr)) != -1) { if (useChunkedTransfer) { - String chunkSizeLine = String.format("%x\r\n", storageIO.getVarHeader().getBytes().length); + String chunkSizeLine = String.format("%x\r\n", bufsize); outstream.write(chunkSizeLine.getBytes()); } - outstream.write(storageIO.getVarHeader().getBytes()); + outstream.write(bffr, 0, bufsize); if (useChunkedTransfer) { outstream.write(chunkClose); } } - } - while ((bufsize = instream.read(bffr)) != -1) { - if (useChunkedTransfer) { - String chunkSizeLine = String.format("%x\r\n", bufsize); - outstream.write(chunkSizeLine.getBytes()); - } - outstream.write(bffr, 0, bufsize); if (useChunkedTransfer) { - outstream.write(chunkClose); + String chunkClosing = "0\r\n\r\n"; + outstream.write(chunkClosing.getBytes()); } - } - if (useChunkedTransfer) { - String chunkClosing = "0\r\n\r\n"; - outstream.write(chunkClosing.getBytes()); - } + logger.fine("di conversion param: " + di.getConversionParam() + ", value: " + di.getConversionParamValue()); - logger.fine("di conversion param: " + di.getConversionParam() + ", value: " + di.getConversionParamValue()); - - // Downloads of thumbnail images (scaled down, low-res versions of graphic image files) and - // "preprocessed metadata" records for tabular data files are NOT considered "real" downloads, - // so these should not produce guestbook entries: - if (di.getGbr() != null && !(isThumbnailDownload(di) || isPreprocessedMetadataDownload(di))) { - try { - logger.fine("writing guestbook response."); - Command cmd = new CreateGuestbookResponseCommand(di.getDataverseRequestService().getDataverseRequest(), di.getGbr(), di.getGbr().getDataFile().getOwner()); - di.getCommand().submit(cmd); - MakeDataCountEntry entry = new MakeDataCountEntry(di.getRequestUriInfo(), di.getRequestHttpHeaders(), di.getDataverseRequestService(), di.getGbr().getDataFile()); - mdcLogService.logEntry(entry); - } catch (CommandException e) { + // Downloads of thumbnail images (scaled down, low-res versions of graphic image files) and + // "preprocessed metadata" records for tabular data files are NOT considered "real" downloads, + // so these should not produce guestbook entries: + if (di.getGbr() != null && !(isThumbnailDownload(di) || isPreprocessedMetadataDownload(di))) { + try { + logger.fine("writing guestbook response."); + Command cmd = new CreateGuestbookResponseCommand(di.getDataverseRequestService().getDataverseRequest(), di.getGbr(), di.getGbr().getDataFile().getOwner()); + di.getCommand().submit(cmd); + MakeDataCountEntry entry = new MakeDataCountEntry(di.getRequestUriInfo(), di.getRequestHttpHeaders(), di.getDataverseRequestService(), di.getGbr().getDataFile()); + mdcLogService.logEntry(entry); + } catch (CommandException e) { + } + } else { + logger.fine("not writing guestbook response"); } - } else { - logger.fine("not writing guestbook response"); - } - instream.close(); - outstream.close(); - return; + instream.close(); + outstream.close(); + return; + } } } } From 9204c45d61ab44f5ab60e1c945e6769a0d84a9be Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 28 May 2021 11:57:54 -0400 Subject: [PATCH 166/354] handle above the fold fields --- src/main/webapp/dataset.xhtml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 43b117762d3..4592c87fc55 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -549,10 +549,13 @@ - + #{DatasetPage.datasetVersionUI.relPublicationId} + + + @@ -560,7 +563,7 @@ - + + + + From c0eba6d375fbff567d81e4fcfa077eeec3f5a3ec Mon Sep 17 00:00:00 2001 From: chenganj Date: Fri, 28 May 2021 14:12:36 -0400 Subject: [PATCH 167/354] handling exception --- .../harvard/iq/dataverse/api/Datasets.java | 160 ++-------------- .../datasetutility/AddReplaceFileHelper.java | 180 +++++++++++++++++- .../iq/dataverse/ingest/IngestUtil.java | 2 +- 3 files changed, 182 insertions(+), 160 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 2333c18a973..f5f00d4d719 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2454,8 +2454,6 @@ public Response getTimestamps(@PathParam("identifier") String id) { public Response addFilesToDataset(@PathParam("id") String idSupplied, @FormDataParam("jsonData") String jsonData) { - JsonArrayBuilder jarr = Json.createArrayBuilder(); - if (!systemConfig.isHTTPUpload()) { return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.httpDisabled")); } @@ -2495,155 +2493,19 @@ public Response addFilesToDataset(@PathParam("id") String idSupplied, } } - msgt("(addFilesToDataset) jsonData: " + jsonData.toString()); - - JsonArray filesJson = null; - - int totalNumberofFiles = 0; - int successNumberofFiles = 0; - // ----------------------------------------------------------- - // Read jsonData and Parse files information from jsondata : - // ----------------------------------------------------------- - try (StringReader rdr = new StringReader(jsonData)) { - JsonReader dbJsonReader = Json.createReader(rdr); - filesJson = dbJsonReader.readArray(); - dbJsonReader.close(); - - DataverseRequest dvRequest = createDataverseRequest(authUser); - - AddReplaceFileHelper addFileHelper = new AddReplaceFileHelper( - dvRequest, - ingestService, - datasetService, - fileService, - permissionSvc, - commandEngine, - systemConfig - ); - - - if (filesJson != null) { - totalNumberofFiles = filesJson.getValuesAs(JsonObject.class).size(); - - for (JsonObject fileJson : filesJson.getValuesAs(JsonObject.class)) { + DataverseRequest dvRequest = createDataverseRequest(authUser); - OptionalFileParams optionalFileParams = null; - try { - optionalFileParams = new OptionalFileParams(fileJson.toString()); - - String newFilename = null; - String newFileContentType = null; - String newStorageIdentifier = null; - if (optionalFileParams.hasStorageIdentifier()) { - newStorageIdentifier = optionalFileParams.getStorageIdentifier(); - if (optionalFileParams.hasFileName()) { - newFilename = optionalFileParams.getFileName(); - if (optionalFileParams.hasMimetype()) { - newFileContentType = optionalFileParams.getMimeType(); - } - } - - msgt("ADD! = " + newFilename); - - addFileHelper.runAddFileByDataset(dataset, - newFilename, - newFileContentType, - newStorageIdentifier, - null, - optionalFileParams, true); - - if (addFileHelper.hasError()) { - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("storageIdentifier", newStorageIdentifier) - .add("errorMessage", addFileHelper.getHttpErrorCode().toString() +":"+addFileHelper.getErrorMessagesAsString("\n")) - .add("fileDetails", fileJson); - jarr.add(fileoutput); - } else { - JsonObject successresult = addFileHelper.getSuccessResultAsJsonObjectBuilder().build(); - String duplicateWarning = addFileHelper.getDuplicateFileWarning(); - - if (duplicateWarning != null && !duplicateWarning.isEmpty()) { - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("storageIdentifier", newStorageIdentifier) - .add("warningMessage", addFileHelper.getDuplicateFileWarning()) - .add("fileDetails", successresult.getJsonArray("files").getJsonObject(0)); - jarr.add(fileoutput); - } else { - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("storageIdentifier", newStorageIdentifier) - .add("successMessage", "Added successfully to the dataset") - .add("fileDetails", successresult.getJsonArray("files").getJsonObject(0)); - jarr.add(fileoutput); - } - } - successNumberofFiles = successNumberofFiles + 1; - } else { - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("errorMessage", "You must provide a storageidentifier, filename, and mimetype.") - .add("fileDetails", fileJson); - - jarr.add(fileoutput); - } - - } catch (DataFileTagException ex) { - Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("errorCode", Response.Status.BAD_REQUEST.getStatusCode()) - .add("message", ex.getMessage()) - .add("fileDetails", fileJson); - jarr.add(fileoutput); - - } - catch (NoFilesException ex) { - Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("errorCode", Response.Status.BAD_REQUEST.getStatusCode()) - .add("message", BundleUtil.getStringFromBundle("NoFileException! Serious Error! See administrator!")) - .add("fileDetails", fileJson); - jarr.add(fileoutput); - } - - }// End of adding files - - DatasetLock eipLock = dataset.getLockFor(DatasetLock.Reason.EditInProgress); - if (eipLock == null) { - logger.log(Level.WARNING, "Dataset not locked for EditInProgress "); - } else { - datasetService.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); - logger.log(Level.INFO, "Removed EditInProgress lock "); - } - - try { - Command cmd = new UpdateDatasetVersionCommand(dataset, dvRequest); - ((UpdateDatasetVersionCommand) cmd).setValidateLenient(true); - commandEngine.submit(cmd); - } catch (CommandException ex) { - return error(Response.Status.INTERNAL_SERVER_ERROR, "CommandException updating DatasetVersion from addFiles job: " + ex.getMessage()); - } - - dataset = datasetService.find(dataset.getId()); - - List s = dataset.getFiles(); - for (DataFile dataFile : s) { - } - //ingest job - ingestService.startIngestJobsForDataset(dataset, (AuthenticatedUser) authUser); - - } - } - catch ( javax.json.stream.JsonParsingException ex) { - ex.printStackTrace(); - return error(BAD_REQUEST, "Json Parsing Exception :" + ex.getMessage()); - } - catch (Exception e) { - e.printStackTrace(); - return error(BAD_REQUEST, e.getMessage()); - } + AddReplaceFileHelper addFileHelper = new AddReplaceFileHelper( + dvRequest, + this.ingestService, + this.datasetService, + this.fileService, + this.permissionSvc, + this.commandEngine, + this.systemConfig + ); - JsonObjectBuilder result = Json.createObjectBuilder() - .add("Total number of files", totalNumberofFiles) - .add("Number of files successfully added", successNumberofFiles); + return addFileHelper.addFiles(jsonData, dataset, authUser); - return ok(Json.createObjectBuilder().add("Files", jarr).add("Result", result)); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index cec4303c1fa..01c5f1b90f4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -5,16 +5,11 @@ */ package edu.harvard.iq.dataverse.datasetutility; -import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DataFile.ChecksumType; -import edu.harvard.iq.dataverse.DataFileServiceBean; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetServiceBean; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.EjbDataverseEngine; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.PermissionServiceBean; +import edu.harvard.iq.dataverse.api.Files; import edu.harvard.iq.dataverse.api.Util; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -31,6 +26,7 @@ import edu.harvard.iq.dataverse.util.json.JsonPrinter; import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -42,13 +38,20 @@ import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.EJBException; -import javax.json.JsonObjectBuilder; +import javax.json.*; import javax.validation.ConstraintViolation; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; + +import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import org.apache.commons.lang3.StringUtils; import org.apache.commons.io.IOUtils; import org.ocpsoft.common.util.Strings; +import static edu.harvard.iq.dataverse.api.AbstractApiBean.STATUS_ERROR; +import static edu.harvard.iq.dataverse.api.AbstractApiBean.STATUS_OK; +import static javax.ws.rs.core.Response.Status.BAD_REQUEST; + /** * Methods to add or replace a single file. * @@ -1986,7 +1989,164 @@ public String getDuplicateFileWarning() { public void setDuplicateFileWarning(String duplicateFileWarning) { this.duplicateFileWarning = duplicateFileWarning; } - + + + public Response addFiles(String jsonData, Dataset dataset, User authUser) + { + msgt("(addFilesToDataset) jsonData: " + jsonData.toString()); + + JsonArrayBuilder jarr = Json.createArrayBuilder(); + + JsonArray filesJson = null; + + int totalNumberofFiles = 0; + int successNumberofFiles = 0; + // ----------------------------------------------------------- + // Read jsonData and Parse files information from jsondata : + // ----------------------------------------------------------- + try (StringReader rdr = new StringReader(jsonData)) { + JsonReader dbJsonReader = Json.createReader(rdr); + filesJson = dbJsonReader.readArray(); + dbJsonReader.close(); + + + if (filesJson != null) { + totalNumberofFiles = filesJson.getValuesAs(JsonObject.class).size(); + + for (JsonObject fileJson : filesJson.getValuesAs(JsonObject.class)) { + + OptionalFileParams optionalFileParams = null; + try { + optionalFileParams = new OptionalFileParams(fileJson.toString()); + + String newFilename = null; + String newFileContentType = null; + String newStorageIdentifier = null; + if (optionalFileParams.hasStorageIdentifier()) { + newStorageIdentifier = optionalFileParams.getStorageIdentifier(); + if (optionalFileParams.hasFileName()) { + newFilename = optionalFileParams.getFileName(); + if (optionalFileParams.hasMimetype()) { + newFileContentType = optionalFileParams.getMimeType(); + } + } + + msgt("ADD! = " + newFilename); + + runAddFileByDataset(dataset, + newFilename, + newFileContentType, + newStorageIdentifier, + null, + optionalFileParams, true); + + if (hasError()) { + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("storageIdentifier", newStorageIdentifier) + .add("errorMessage", getHttpErrorCode().toString() +":"+ getErrorMessagesAsString("\n")) + .add("fileDetails", fileJson); + jarr.add(fileoutput); + } else { + JsonObject successresult = getSuccessResultAsJsonObjectBuilder().build(); + String duplicateWarning = getDuplicateFileWarning(); + + if (duplicateWarning != null && !duplicateWarning.isEmpty()) { + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("storageIdentifier", newStorageIdentifier) + .add("warningMessage", getDuplicateFileWarning()) + .add("fileDetails", successresult.getJsonArray("files").getJsonObject(0)); + jarr.add(fileoutput); + } else { + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("storageIdentifier", newStorageIdentifier) + .add("successMessage", "Added successfully to the dataset") + .add("fileDetails", successresult.getJsonArray("files").getJsonObject(0)); + jarr.add(fileoutput); + } + } + successNumberofFiles = successNumberofFiles + 1; + } else { + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("errorMessage", "You must provide a storageidentifier, filename, and mimetype.") + .add("fileDetails", fileJson); + + jarr.add(fileoutput); + } + + } catch (DataFileTagException ex) { + Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("errorCode", Response.Status.BAD_REQUEST.getStatusCode()) + .add("message", ex.getMessage()) + .add("fileDetails", fileJson); + jarr.add(fileoutput); + + } + catch (NoFilesException ex) { + Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("errorCode", Response.Status.BAD_REQUEST.getStatusCode()) + .add("message", BundleUtil.getStringFromBundle("NoFileException! Serious Error! See administrator!")) + .add("fileDetails", fileJson); + jarr.add(fileoutput); + } + + }// End of adding files + + DatasetLock eipLock = dataset.getLockFor(DatasetLock.Reason.EditInProgress); + if (eipLock == null) { + logger.log(Level.WARNING, "Dataset not locked for EditInProgress "); + } else { + datasetService.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); + logger.log(Level.INFO, "Removed EditInProgress lock "); + } + + try { + Command cmd = new UpdateDatasetVersionCommand(dataset, dvRequest); + ((UpdateDatasetVersionCommand) cmd).setValidateLenient(true); + commandEngine.submit(cmd); + } catch (CommandException ex) { + return error(Response.Status.INTERNAL_SERVER_ERROR, "CommandException updating DatasetVersion from addFiles job: " + ex.getMessage()); + } + + dataset = datasetService.find(dataset.getId()); + + List s = dataset.getFiles(); + for (DataFile dataFile : s) { + } + //ingest job + ingestService.startIngestJobsForDataset(dataset, (AuthenticatedUser) authUser); + + } + } + catch ( javax.json.stream.JsonParsingException ex) { + ex.printStackTrace(); + return error(BAD_REQUEST, "Json Parsing Exception :" + ex.getMessage()); + } + catch (Exception e) { + e.printStackTrace(); + return error(BAD_REQUEST, e.getMessage()); + } + + JsonObjectBuilder result = Json.createObjectBuilder() + .add("Total number of files", totalNumberofFiles) + .add("Number of files successfully added", successNumberofFiles); + + + return Response.ok().entity(Json.createObjectBuilder() + .add("status", STATUS_OK) + .add("data", Json.createObjectBuilder().add("Files", jarr).add("Result", result)).build() ).build(); + } + + protected static Response error(Response.Status sts, String msg ) { + return Response.status(sts) + .entity( NullSafeJsonBuilder.jsonObjectBuilder() + .add("status", STATUS_ERROR) + .add( "message", msg ).build() + ).type(MediaType.APPLICATION_JSON_TYPE).build(); + } + + } // end class /* DatasetPage sequence: diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java index 13d4ed96815..046a0f4b56b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java @@ -257,7 +257,7 @@ public static Set existingPathNamesAsSet(DatasetVersion version, FileMet // #6942 added proxy for existing files to a boolean set when dataset version copy is done for (Iterator fmIt = version.getFileMetadatas().iterator(); fmIt.hasNext();) { FileMetadata fm = fmIt.next(); - if((fm.isInPriorVersion() || fm.getId() != null) && (replacedFmd==null) || (!fm.getDataFile().equals(replacedFmd.getDataFile()))) { + if((fm.isInPriorVersion() || fm.getId() != null) && (replacedFmd==null || !fm.getDataFile().equals(replacedFmd.getDataFile()))) { String existingName = fm.getLabel(); String existingDir = fm.getDirectoryLabel(); From a24392c349afb503d1c02ec6c57c82f29d75baaf Mon Sep 17 00:00:00 2001 From: chenganj Date: Fri, 28 May 2021 14:14:15 -0400 Subject: [PATCH 168/354] Revert "handling exception" This reverts commit c0eba6d3 --- .../harvard/iq/dataverse/api/Datasets.java | 160 ++++++++++++++-- .../datasetutility/AddReplaceFileHelper.java | 180 +----------------- .../iq/dataverse/ingest/IngestUtil.java | 2 +- 3 files changed, 160 insertions(+), 182 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index f5f00d4d719..2333c18a973 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2454,6 +2454,8 @@ public Response getTimestamps(@PathParam("identifier") String id) { public Response addFilesToDataset(@PathParam("id") String idSupplied, @FormDataParam("jsonData") String jsonData) { + JsonArrayBuilder jarr = Json.createArrayBuilder(); + if (!systemConfig.isHTTPUpload()) { return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.httpDisabled")); } @@ -2493,19 +2495,155 @@ public Response addFilesToDataset(@PathParam("id") String idSupplied, } } - DataverseRequest dvRequest = createDataverseRequest(authUser); + msgt("(addFilesToDataset) jsonData: " + jsonData.toString()); + + JsonArray filesJson = null; + + int totalNumberofFiles = 0; + int successNumberofFiles = 0; + // ----------------------------------------------------------- + // Read jsonData and Parse files information from jsondata : + // ----------------------------------------------------------- + try (StringReader rdr = new StringReader(jsonData)) { + JsonReader dbJsonReader = Json.createReader(rdr); + filesJson = dbJsonReader.readArray(); + dbJsonReader.close(); + + DataverseRequest dvRequest = createDataverseRequest(authUser); + + AddReplaceFileHelper addFileHelper = new AddReplaceFileHelper( + dvRequest, + ingestService, + datasetService, + fileService, + permissionSvc, + commandEngine, + systemConfig + ); + + + if (filesJson != null) { + totalNumberofFiles = filesJson.getValuesAs(JsonObject.class).size(); + + for (JsonObject fileJson : filesJson.getValuesAs(JsonObject.class)) { - AddReplaceFileHelper addFileHelper = new AddReplaceFileHelper( - dvRequest, - this.ingestService, - this.datasetService, - this.fileService, - this.permissionSvc, - this.commandEngine, - this.systemConfig - ); + OptionalFileParams optionalFileParams = null; + try { + optionalFileParams = new OptionalFileParams(fileJson.toString()); + + String newFilename = null; + String newFileContentType = null; + String newStorageIdentifier = null; + if (optionalFileParams.hasStorageIdentifier()) { + newStorageIdentifier = optionalFileParams.getStorageIdentifier(); + if (optionalFileParams.hasFileName()) { + newFilename = optionalFileParams.getFileName(); + if (optionalFileParams.hasMimetype()) { + newFileContentType = optionalFileParams.getMimeType(); + } + } + + msgt("ADD! = " + newFilename); + + addFileHelper.runAddFileByDataset(dataset, + newFilename, + newFileContentType, + newStorageIdentifier, + null, + optionalFileParams, true); + + if (addFileHelper.hasError()) { + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("storageIdentifier", newStorageIdentifier) + .add("errorMessage", addFileHelper.getHttpErrorCode().toString() +":"+addFileHelper.getErrorMessagesAsString("\n")) + .add("fileDetails", fileJson); + jarr.add(fileoutput); + } else { + JsonObject successresult = addFileHelper.getSuccessResultAsJsonObjectBuilder().build(); + String duplicateWarning = addFileHelper.getDuplicateFileWarning(); + + if (duplicateWarning != null && !duplicateWarning.isEmpty()) { + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("storageIdentifier", newStorageIdentifier) + .add("warningMessage", addFileHelper.getDuplicateFileWarning()) + .add("fileDetails", successresult.getJsonArray("files").getJsonObject(0)); + jarr.add(fileoutput); + } else { + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("storageIdentifier", newStorageIdentifier) + .add("successMessage", "Added successfully to the dataset") + .add("fileDetails", successresult.getJsonArray("files").getJsonObject(0)); + jarr.add(fileoutput); + } + } + successNumberofFiles = successNumberofFiles + 1; + } else { + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("errorMessage", "You must provide a storageidentifier, filename, and mimetype.") + .add("fileDetails", fileJson); + + jarr.add(fileoutput); + } + + } catch (DataFileTagException ex) { + Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("errorCode", Response.Status.BAD_REQUEST.getStatusCode()) + .add("message", ex.getMessage()) + .add("fileDetails", fileJson); + jarr.add(fileoutput); + + } + catch (NoFilesException ex) { + Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("errorCode", Response.Status.BAD_REQUEST.getStatusCode()) + .add("message", BundleUtil.getStringFromBundle("NoFileException! Serious Error! See administrator!")) + .add("fileDetails", fileJson); + jarr.add(fileoutput); + } + + }// End of adding files + + DatasetLock eipLock = dataset.getLockFor(DatasetLock.Reason.EditInProgress); + if (eipLock == null) { + logger.log(Level.WARNING, "Dataset not locked for EditInProgress "); + } else { + datasetService.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); + logger.log(Level.INFO, "Removed EditInProgress lock "); + } + + try { + Command cmd = new UpdateDatasetVersionCommand(dataset, dvRequest); + ((UpdateDatasetVersionCommand) cmd).setValidateLenient(true); + commandEngine.submit(cmd); + } catch (CommandException ex) { + return error(Response.Status.INTERNAL_SERVER_ERROR, "CommandException updating DatasetVersion from addFiles job: " + ex.getMessage()); + } + + dataset = datasetService.find(dataset.getId()); + + List s = dataset.getFiles(); + for (DataFile dataFile : s) { + } + //ingest job + ingestService.startIngestJobsForDataset(dataset, (AuthenticatedUser) authUser); + + } + } + catch ( javax.json.stream.JsonParsingException ex) { + ex.printStackTrace(); + return error(BAD_REQUEST, "Json Parsing Exception :" + ex.getMessage()); + } + catch (Exception e) { + e.printStackTrace(); + return error(BAD_REQUEST, e.getMessage()); + } - return addFileHelper.addFiles(jsonData, dataset, authUser); + JsonObjectBuilder result = Json.createObjectBuilder() + .add("Total number of files", totalNumberofFiles) + .add("Number of files successfully added", successNumberofFiles); + return ok(Json.createObjectBuilder().add("Files", jarr).add("Result", result)); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 01c5f1b90f4..cec4303c1fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -5,11 +5,16 @@ */ package edu.harvard.iq.dataverse.datasetutility; -import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataFile.ChecksumType; -import edu.harvard.iq.dataverse.api.Files; +import edu.harvard.iq.dataverse.DataFileServiceBean; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetServiceBean; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.EjbDataverseEngine; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.PermissionServiceBean; import edu.harvard.iq.dataverse.api.Util; -import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -26,7 +31,6 @@ import edu.harvard.iq.dataverse.util.json.JsonPrinter; import java.io.IOException; import java.io.InputStream; -import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -38,20 +42,13 @@ import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.EJBException; -import javax.json.*; +import javax.json.JsonObjectBuilder; import javax.validation.ConstraintViolation; -import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; - -import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import org.apache.commons.lang3.StringUtils; import org.apache.commons.io.IOUtils; import org.ocpsoft.common.util.Strings; -import static edu.harvard.iq.dataverse.api.AbstractApiBean.STATUS_ERROR; -import static edu.harvard.iq.dataverse.api.AbstractApiBean.STATUS_OK; -import static javax.ws.rs.core.Response.Status.BAD_REQUEST; - /** * Methods to add or replace a single file. * @@ -1989,164 +1986,7 @@ public String getDuplicateFileWarning() { public void setDuplicateFileWarning(String duplicateFileWarning) { this.duplicateFileWarning = duplicateFileWarning; } - - - public Response addFiles(String jsonData, Dataset dataset, User authUser) - { - msgt("(addFilesToDataset) jsonData: " + jsonData.toString()); - - JsonArrayBuilder jarr = Json.createArrayBuilder(); - - JsonArray filesJson = null; - - int totalNumberofFiles = 0; - int successNumberofFiles = 0; - // ----------------------------------------------------------- - // Read jsonData and Parse files information from jsondata : - // ----------------------------------------------------------- - try (StringReader rdr = new StringReader(jsonData)) { - JsonReader dbJsonReader = Json.createReader(rdr); - filesJson = dbJsonReader.readArray(); - dbJsonReader.close(); - - - if (filesJson != null) { - totalNumberofFiles = filesJson.getValuesAs(JsonObject.class).size(); - - for (JsonObject fileJson : filesJson.getValuesAs(JsonObject.class)) { - - OptionalFileParams optionalFileParams = null; - try { - optionalFileParams = new OptionalFileParams(fileJson.toString()); - - String newFilename = null; - String newFileContentType = null; - String newStorageIdentifier = null; - if (optionalFileParams.hasStorageIdentifier()) { - newStorageIdentifier = optionalFileParams.getStorageIdentifier(); - if (optionalFileParams.hasFileName()) { - newFilename = optionalFileParams.getFileName(); - if (optionalFileParams.hasMimetype()) { - newFileContentType = optionalFileParams.getMimeType(); - } - } - - msgt("ADD! = " + newFilename); - - runAddFileByDataset(dataset, - newFilename, - newFileContentType, - newStorageIdentifier, - null, - optionalFileParams, true); - - if (hasError()) { - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("storageIdentifier", newStorageIdentifier) - .add("errorMessage", getHttpErrorCode().toString() +":"+ getErrorMessagesAsString("\n")) - .add("fileDetails", fileJson); - jarr.add(fileoutput); - } else { - JsonObject successresult = getSuccessResultAsJsonObjectBuilder().build(); - String duplicateWarning = getDuplicateFileWarning(); - - if (duplicateWarning != null && !duplicateWarning.isEmpty()) { - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("storageIdentifier", newStorageIdentifier) - .add("warningMessage", getDuplicateFileWarning()) - .add("fileDetails", successresult.getJsonArray("files").getJsonObject(0)); - jarr.add(fileoutput); - } else { - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("storageIdentifier", newStorageIdentifier) - .add("successMessage", "Added successfully to the dataset") - .add("fileDetails", successresult.getJsonArray("files").getJsonObject(0)); - jarr.add(fileoutput); - } - } - successNumberofFiles = successNumberofFiles + 1; - } else { - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("errorMessage", "You must provide a storageidentifier, filename, and mimetype.") - .add("fileDetails", fileJson); - - jarr.add(fileoutput); - } - - } catch (DataFileTagException ex) { - Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("errorCode", Response.Status.BAD_REQUEST.getStatusCode()) - .add("message", ex.getMessage()) - .add("fileDetails", fileJson); - jarr.add(fileoutput); - - } - catch (NoFilesException ex) { - Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("errorCode", Response.Status.BAD_REQUEST.getStatusCode()) - .add("message", BundleUtil.getStringFromBundle("NoFileException! Serious Error! See administrator!")) - .add("fileDetails", fileJson); - jarr.add(fileoutput); - } - - }// End of adding files - - DatasetLock eipLock = dataset.getLockFor(DatasetLock.Reason.EditInProgress); - if (eipLock == null) { - logger.log(Level.WARNING, "Dataset not locked for EditInProgress "); - } else { - datasetService.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); - logger.log(Level.INFO, "Removed EditInProgress lock "); - } - - try { - Command cmd = new UpdateDatasetVersionCommand(dataset, dvRequest); - ((UpdateDatasetVersionCommand) cmd).setValidateLenient(true); - commandEngine.submit(cmd); - } catch (CommandException ex) { - return error(Response.Status.INTERNAL_SERVER_ERROR, "CommandException updating DatasetVersion from addFiles job: " + ex.getMessage()); - } - - dataset = datasetService.find(dataset.getId()); - - List s = dataset.getFiles(); - for (DataFile dataFile : s) { - } - //ingest job - ingestService.startIngestJobsForDataset(dataset, (AuthenticatedUser) authUser); - - } - } - catch ( javax.json.stream.JsonParsingException ex) { - ex.printStackTrace(); - return error(BAD_REQUEST, "Json Parsing Exception :" + ex.getMessage()); - } - catch (Exception e) { - e.printStackTrace(); - return error(BAD_REQUEST, e.getMessage()); - } - - JsonObjectBuilder result = Json.createObjectBuilder() - .add("Total number of files", totalNumberofFiles) - .add("Number of files successfully added", successNumberofFiles); - - - return Response.ok().entity(Json.createObjectBuilder() - .add("status", STATUS_OK) - .add("data", Json.createObjectBuilder().add("Files", jarr).add("Result", result)).build() ).build(); - } - - protected static Response error(Response.Status sts, String msg ) { - return Response.status(sts) - .entity( NullSafeJsonBuilder.jsonObjectBuilder() - .add("status", STATUS_ERROR) - .add( "message", msg ).build() - ).type(MediaType.APPLICATION_JSON_TYPE).build(); - } - - + } // end class /* DatasetPage sequence: diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java index 046a0f4b56b..13d4ed96815 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java @@ -257,7 +257,7 @@ public static Set existingPathNamesAsSet(DatasetVersion version, FileMet // #6942 added proxy for existing files to a boolean set when dataset version copy is done for (Iterator fmIt = version.getFileMetadatas().iterator(); fmIt.hasNext();) { FileMetadata fm = fmIt.next(); - if((fm.isInPriorVersion() || fm.getId() != null) && (replacedFmd==null || !fm.getDataFile().equals(replacedFmd.getDataFile()))) { + if((fm.isInPriorVersion() || fm.getId() != null) && (replacedFmd==null) || (!fm.getDataFile().equals(replacedFmd.getDataFile()))) { String existingName = fm.getLabel(); String existingDir = fm.getDirectoryLabel(); From a3385af9149908d731c820aceb21dbf7a1bb2c48 Mon Sep 17 00:00:00 2001 From: "don.sizemore" Date: Fri, 28 May 2021 14:48:12 -0400 Subject: [PATCH 169/354] #7910 remove newly-redundant instream.close() --- .../edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index a0377faede2..e06080b8d7a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -467,7 +467,6 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] logger.fine("not writing guestbook response"); } - instream.close(); outstream.close(); return; } From db6944774fdbdf0de4eebb9d6c0675586bc8e3c7 Mon Sep 17 00:00:00 2001 From: chenganj Date: Fri, 28 May 2021 14:49:14 -0400 Subject: [PATCH 170/354] Refactored /addFiles --- .../harvard/iq/dataverse/api/Datasets.java | 160 ++-------------- .../datasetutility/AddReplaceFileHelper.java | 176 +++++++++++++++++- .../iq/dataverse/ingest/IngestUtil.java | 4 +- 3 files changed, 180 insertions(+), 160 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 2333c18a973..f5f00d4d719 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2454,8 +2454,6 @@ public Response getTimestamps(@PathParam("identifier") String id) { public Response addFilesToDataset(@PathParam("id") String idSupplied, @FormDataParam("jsonData") String jsonData) { - JsonArrayBuilder jarr = Json.createArrayBuilder(); - if (!systemConfig.isHTTPUpload()) { return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.httpDisabled")); } @@ -2495,155 +2493,19 @@ public Response addFilesToDataset(@PathParam("id") String idSupplied, } } - msgt("(addFilesToDataset) jsonData: " + jsonData.toString()); - - JsonArray filesJson = null; - - int totalNumberofFiles = 0; - int successNumberofFiles = 0; - // ----------------------------------------------------------- - // Read jsonData and Parse files information from jsondata : - // ----------------------------------------------------------- - try (StringReader rdr = new StringReader(jsonData)) { - JsonReader dbJsonReader = Json.createReader(rdr); - filesJson = dbJsonReader.readArray(); - dbJsonReader.close(); - - DataverseRequest dvRequest = createDataverseRequest(authUser); - - AddReplaceFileHelper addFileHelper = new AddReplaceFileHelper( - dvRequest, - ingestService, - datasetService, - fileService, - permissionSvc, - commandEngine, - systemConfig - ); - - - if (filesJson != null) { - totalNumberofFiles = filesJson.getValuesAs(JsonObject.class).size(); - - for (JsonObject fileJson : filesJson.getValuesAs(JsonObject.class)) { + DataverseRequest dvRequest = createDataverseRequest(authUser); - OptionalFileParams optionalFileParams = null; - try { - optionalFileParams = new OptionalFileParams(fileJson.toString()); - - String newFilename = null; - String newFileContentType = null; - String newStorageIdentifier = null; - if (optionalFileParams.hasStorageIdentifier()) { - newStorageIdentifier = optionalFileParams.getStorageIdentifier(); - if (optionalFileParams.hasFileName()) { - newFilename = optionalFileParams.getFileName(); - if (optionalFileParams.hasMimetype()) { - newFileContentType = optionalFileParams.getMimeType(); - } - } - - msgt("ADD! = " + newFilename); - - addFileHelper.runAddFileByDataset(dataset, - newFilename, - newFileContentType, - newStorageIdentifier, - null, - optionalFileParams, true); - - if (addFileHelper.hasError()) { - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("storageIdentifier", newStorageIdentifier) - .add("errorMessage", addFileHelper.getHttpErrorCode().toString() +":"+addFileHelper.getErrorMessagesAsString("\n")) - .add("fileDetails", fileJson); - jarr.add(fileoutput); - } else { - JsonObject successresult = addFileHelper.getSuccessResultAsJsonObjectBuilder().build(); - String duplicateWarning = addFileHelper.getDuplicateFileWarning(); - - if (duplicateWarning != null && !duplicateWarning.isEmpty()) { - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("storageIdentifier", newStorageIdentifier) - .add("warningMessage", addFileHelper.getDuplicateFileWarning()) - .add("fileDetails", successresult.getJsonArray("files").getJsonObject(0)); - jarr.add(fileoutput); - } else { - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("storageIdentifier", newStorageIdentifier) - .add("successMessage", "Added successfully to the dataset") - .add("fileDetails", successresult.getJsonArray("files").getJsonObject(0)); - jarr.add(fileoutput); - } - } - successNumberofFiles = successNumberofFiles + 1; - } else { - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("errorMessage", "You must provide a storageidentifier, filename, and mimetype.") - .add("fileDetails", fileJson); - - jarr.add(fileoutput); - } - - } catch (DataFileTagException ex) { - Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("errorCode", Response.Status.BAD_REQUEST.getStatusCode()) - .add("message", ex.getMessage()) - .add("fileDetails", fileJson); - jarr.add(fileoutput); - - } - catch (NoFilesException ex) { - Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); - JsonObjectBuilder fileoutput = Json.createObjectBuilder() - .add("errorCode", Response.Status.BAD_REQUEST.getStatusCode()) - .add("message", BundleUtil.getStringFromBundle("NoFileException! Serious Error! See administrator!")) - .add("fileDetails", fileJson); - jarr.add(fileoutput); - } - - }// End of adding files - - DatasetLock eipLock = dataset.getLockFor(DatasetLock.Reason.EditInProgress); - if (eipLock == null) { - logger.log(Level.WARNING, "Dataset not locked for EditInProgress "); - } else { - datasetService.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); - logger.log(Level.INFO, "Removed EditInProgress lock "); - } - - try { - Command cmd = new UpdateDatasetVersionCommand(dataset, dvRequest); - ((UpdateDatasetVersionCommand) cmd).setValidateLenient(true); - commandEngine.submit(cmd); - } catch (CommandException ex) { - return error(Response.Status.INTERNAL_SERVER_ERROR, "CommandException updating DatasetVersion from addFiles job: " + ex.getMessage()); - } - - dataset = datasetService.find(dataset.getId()); - - List s = dataset.getFiles(); - for (DataFile dataFile : s) { - } - //ingest job - ingestService.startIngestJobsForDataset(dataset, (AuthenticatedUser) authUser); - - } - } - catch ( javax.json.stream.JsonParsingException ex) { - ex.printStackTrace(); - return error(BAD_REQUEST, "Json Parsing Exception :" + ex.getMessage()); - } - catch (Exception e) { - e.printStackTrace(); - return error(BAD_REQUEST, e.getMessage()); - } + AddReplaceFileHelper addFileHelper = new AddReplaceFileHelper( + dvRequest, + this.ingestService, + this.datasetService, + this.fileService, + this.permissionSvc, + this.commandEngine, + this.systemConfig + ); - JsonObjectBuilder result = Json.createObjectBuilder() - .add("Total number of files", totalNumberofFiles) - .add("Number of files successfully added", successNumberofFiles); + return addFileHelper.addFiles(jsonData, dataset, authUser); - return ok(Json.createObjectBuilder().add("Files", jarr).add("Result", result)); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index cec4303c1fa..6dbe33abc05 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -5,16 +5,11 @@ */ package edu.harvard.iq.dataverse.datasetutility; -import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DataFile.ChecksumType; -import edu.harvard.iq.dataverse.DataFileServiceBean; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetServiceBean; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.EjbDataverseEngine; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.PermissionServiceBean; +import edu.harvard.iq.dataverse.api.Files; import edu.harvard.iq.dataverse.api.Util; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -31,6 +26,7 @@ import edu.harvard.iq.dataverse.util.json.JsonPrinter; import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -42,13 +38,20 @@ import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.EJBException; -import javax.json.JsonObjectBuilder; +import javax.json.*; import javax.validation.ConstraintViolation; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; + +import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import org.apache.commons.lang3.StringUtils; import org.apache.commons.io.IOUtils; import org.ocpsoft.common.util.Strings; +import static edu.harvard.iq.dataverse.api.AbstractApiBean.STATUS_ERROR; +import static edu.harvard.iq.dataverse.api.AbstractApiBean.STATUS_OK; +import static javax.ws.rs.core.Response.Status.BAD_REQUEST; + /** * Methods to add or replace a single file. * @@ -1986,6 +1989,161 @@ public String getDuplicateFileWarning() { public void setDuplicateFileWarning(String duplicateFileWarning) { this.duplicateFileWarning = duplicateFileWarning; } + + public Response addFiles(String jsonData, Dataset dataset, User authUser) + { + msgt("(addFilesToDataset) jsonData: " + jsonData.toString()); + + JsonArrayBuilder jarr = Json.createArrayBuilder(); + + JsonArray filesJson = null; + + int totalNumberofFiles = 0; + int successNumberofFiles = 0; + // ----------------------------------------------------------- + // Read jsonData and Parse files information from jsondata : + // ----------------------------------------------------------- + try (StringReader rdr = new StringReader(jsonData)) { + JsonReader dbJsonReader = Json.createReader(rdr); + filesJson = dbJsonReader.readArray(); + dbJsonReader.close(); + + + if (filesJson != null) { + totalNumberofFiles = filesJson.getValuesAs(JsonObject.class).size(); + + for (JsonObject fileJson : filesJson.getValuesAs(JsonObject.class)) { + + OptionalFileParams optionalFileParams = null; + try { + optionalFileParams = new OptionalFileParams(fileJson.toString()); + + String newFilename = null; + String newFileContentType = null; + String newStorageIdentifier = null; + if (optionalFileParams.hasStorageIdentifier()) { + newStorageIdentifier = optionalFileParams.getStorageIdentifier(); + if (optionalFileParams.hasFileName()) { + newFilename = optionalFileParams.getFileName(); + if (optionalFileParams.hasMimetype()) { + newFileContentType = optionalFileParams.getMimeType(); + } + } + + msgt("ADD! = " + newFilename); + + runAddFileByDataset(dataset, + newFilename, + newFileContentType, + newStorageIdentifier, + null, + optionalFileParams, true); + + if (hasError()) { + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("storageIdentifier", newStorageIdentifier) + .add("errorMessage", getHttpErrorCode().toString() +":"+ getErrorMessagesAsString("\n")) + .add("fileDetails", fileJson); + jarr.add(fileoutput); + } else { + JsonObject successresult = getSuccessResultAsJsonObjectBuilder().build(); + String duplicateWarning = getDuplicateFileWarning(); + + if (duplicateWarning != null && !duplicateWarning.isEmpty()) { + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("storageIdentifier", newStorageIdentifier) + .add("warningMessage", getDuplicateFileWarning()) + .add("fileDetails", successresult.getJsonArray("files").getJsonObject(0)); + jarr.add(fileoutput); + } else { + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("storageIdentifier", newStorageIdentifier) + .add("successMessage", "Added successfully to the dataset") + .add("fileDetails", successresult.getJsonArray("files").getJsonObject(0)); + jarr.add(fileoutput); + } + } + successNumberofFiles = successNumberofFiles + 1; + } else { + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("errorMessage", "You must provide a storageidentifier, filename, and mimetype.") + .add("fileDetails", fileJson); + + jarr.add(fileoutput); + } + + } catch (DataFileTagException ex) { + Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("errorCode", Response.Status.BAD_REQUEST.getStatusCode()) + .add("message", ex.getMessage()) + .add("fileDetails", fileJson); + jarr.add(fileoutput); + + } + catch (NoFilesException ex) { + Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); + JsonObjectBuilder fileoutput = Json.createObjectBuilder() + .add("errorCode", Response.Status.BAD_REQUEST.getStatusCode()) + .add("message", BundleUtil.getStringFromBundle("NoFileException! Serious Error! See administrator!")) + .add("fileDetails", fileJson); + jarr.add(fileoutput); + } + + }// End of adding files + + DatasetLock eipLock = dataset.getLockFor(DatasetLock.Reason.EditInProgress); + if (eipLock == null) { + logger.log(Level.WARNING, "Dataset not locked for EditInProgress "); + } else { + datasetService.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); + logger.log(Level.INFO, "Removed EditInProgress lock "); + } + + try { + Command cmd = new UpdateDatasetVersionCommand(dataset, dvRequest); + ((UpdateDatasetVersionCommand) cmd).setValidateLenient(true); + commandEngine.submit(cmd); + } catch (CommandException ex) { + return error(Response.Status.INTERNAL_SERVER_ERROR, "CommandException updating DatasetVersion from addFiles job: " + ex.getMessage()); + } + + dataset = datasetService.find(dataset.getId()); + + List s = dataset.getFiles(); + for (DataFile dataFile : s) { + } + //ingest job + ingestService.startIngestJobsForDataset(dataset, (AuthenticatedUser) authUser); + + } + } + catch ( javax.json.stream.JsonParsingException ex) { + ex.printStackTrace(); + return error(BAD_REQUEST, "Json Parsing Exception :" + ex.getMessage()); + } + catch (Exception e) { + e.printStackTrace(); + return error(BAD_REQUEST, e.getMessage()); + } + + JsonObjectBuilder result = Json.createObjectBuilder() + .add("Total number of files", totalNumberofFiles) + .add("Number of files successfully added", successNumberofFiles); + + + return Response.ok().entity(Json.createObjectBuilder() + .add("status", STATUS_OK) + .add("data", Json.createObjectBuilder().add("Files", jarr).add("Result", result)).build() ).build(); + } + + protected static Response error(Response.Status sts, String msg ) { + return Response.status(sts) + .entity( NullSafeJsonBuilder.jsonObjectBuilder() + .add("status", STATUS_ERROR) + .add( "message", msg ).build() + ).type(MediaType.APPLICATION_JSON_TYPE).build(); + } } // end class /* diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java index 13d4ed96815..9484a412913 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java @@ -57,7 +57,7 @@ public class IngestUtil { * * @param version the dataset version * @param newFiles the list of new data files to add to it - * @param fileToReplace + * @param fileToReplace */ public static void checkForDuplicateFileNamesFinal(DatasetVersion version, List newFiles, DataFile fileToReplace) { @@ -257,7 +257,7 @@ public static Set existingPathNamesAsSet(DatasetVersion version, FileMet // #6942 added proxy for existing files to a boolean set when dataset version copy is done for (Iterator fmIt = version.getFileMetadatas().iterator(); fmIt.hasNext();) { FileMetadata fm = fmIt.next(); - if((fm.isInPriorVersion() || fm.getId() != null) && (replacedFmd==null) || (!fm.getDataFile().equals(replacedFmd.getDataFile()))) { + if((fm.isInPriorVersion() || fm.getId() != null) && (replacedFmd==null || !fm.getDataFile().equals(replacedFmd.getDataFile()))) { String existingName = fm.getLabel(); String existingDir = fm.getDirectoryLabel(); From cdb7db464b0ccbf450d8dfa6a2ee75fddd62471f Mon Sep 17 00:00:00 2001 From: "don.sizemore" Date: Fri, 28 May 2021 15:13:18 -0400 Subject: [PATCH 171/354] #7897 add phil's authoritative list of ITs for docker-aio and dataverse-ansible --- conf/docker-aio/run-test-suite.sh | 4 +++- tests/integration-tests.txt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 tests/integration-tests.txt diff --git a/conf/docker-aio/run-test-suite.sh b/conf/docker-aio/run-test-suite.sh index bf0683fdbd4..19d05154a17 100755 --- a/conf/docker-aio/run-test-suite.sh +++ b/conf/docker-aio/run-test-suite.sh @@ -6,6 +6,8 @@ if [ -z "$dvurl" ]; then dvurl="http://localhost:8084" fi +integrationtests=$(<../../tests/integration-tests.txt) + # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment. # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml. -source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT -Ddataverse.test.baseurl=$dvurl +mvn test -Dtest=$integrationtests -Ddataverse.test.baseurl=$dvurl diff --git a/tests/integration-tests.txt b/tests/integration-tests.txt new file mode 100644 index 00000000000..d6edfa4d70e --- /dev/null +++ b/tests/integration-tests.txt @@ -0,0 +1 @@ +DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT From f2a4d46ac4799c76eb39221540956b27679bdc16 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 28 May 2021 15:40:32 -0400 Subject: [PATCH 172/354] mention new IT test list in dev guide #7897 Also remove reference to phoenix server, which has been decommissioned. We'll finished cleaning up phoenix-related docs in #7031. --- doc/sphinx-guides/source/developers/testing.rst | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index bbfac33fcda..e7bc6b2fed5 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -246,7 +246,7 @@ Once installed, you may run commands with ``mvn [options] [] [`. +If you are adding a new test class, be sure to add it to :download:`tests/integration-tests.txt <../../../../tests/integration-tests.txt>` so that our automated testing knows about it. Writing and Using a Testcontainers Test @@ -438,13 +438,6 @@ How to Run the Phoenix Tests - Log into Jenkins and click "Build Now" at https://build.hmdc.harvard.edu:8443/job/phoenix.dataverse.org-build-develop/ - Wait for all three chained Jenkins jobs to complete and note if they passed or failed. If you see a failure, open a GitHub issue or at least get the attention of some developers. -List of Tests Run Against the Phoenix Server -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -We haven't thought much about a good way to publicly list the "IT" classes that are executed against the phoenix server. (Currently your best bet is to look at the ``Executing Maven`` line at the top of the "Full Log" of "Console Output" of ``phoenix.dataverse.org-apitest-develop`` Jenkins job mentioned above.) We endeavor to keep the list of tests in the "all-in-one" Docker environment described above in sync with the list of tests configured in Jenkins. That is to say, refer to :download:`run-test-suite.sh <../../../../conf/docker-aio/run-test-suite.sh>` mentioned in ``conf/docker-aio/readme.md`` for the current list of IT tests that are expected to pass. Here's a dump of that file: - -.. literalinclude:: ../../../../conf/docker-aio/run-test-suite.sh - Accessibility Testing --------------------- From 8f297802b5cd7ee98fea7c89ec2f26b1a601ea1e Mon Sep 17 00:00:00 2001 From: chenganj Date: Fri, 28 May 2021 16:42:41 -0400 Subject: [PATCH 173/354] style correction --- .../iq/dataverse/datasetutility/AddReplaceFileHelper.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 6dbe33abc05..21f2aca1eba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -1990,8 +1990,7 @@ public void setDuplicateFileWarning(String duplicateFileWarning) { this.duplicateFileWarning = duplicateFileWarning; } - public Response addFiles(String jsonData, Dataset dataset, User authUser) - { + public Response addFiles(String jsonData, Dataset dataset, User authUser) { msgt("(addFilesToDataset) jsonData: " + jsonData.toString()); JsonArrayBuilder jarr = Json.createArrayBuilder(); From e8ca18a10af6ae4593d6a6cd53edf2e78c31962d Mon Sep 17 00:00:00 2001 From: chenganj Date: Tue, 1 Jun 2021 10:36:22 -0400 Subject: [PATCH 174/354] /addFiles API doc --- doc/sphinx-guides/source/api/native-api.rst | 38 +++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 3a3f2fe5cec..45f693818fa 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2086,6 +2086,44 @@ The fully expanded example above (without environment variables) looks like this Note: The ``id`` returned in the json response is the id of the file metadata version. + + +Adding File Metadata +~~~~~~~~~~~~~~~~~~~~ + +This API call requires a ``jsonString`` expressing the metadata of multiple files. It adds file metadata to the database table where the file already copied to the storage. + +The jsonData object includes values for: + +* "description" - A description of the file +* "directoryLabel" - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset +* "storageIdentifier" - String +* "fileName" - String +* "mimeType" - String +* "fixity/checksum" either: + + * "md5Hash" - String with MD5 hash value, or + * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings + + +A curl example using an ``PERSISTENT_ID`` + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/7U7YBV + export JSON_DATA="[{'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42', 'fileName':'file1.txt', 'mimeType':'text/plain', 'checksum': {'@type': 'SHA-1', '@value': '123456'}}, \ + {'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53', 'fileName':'file2.txt', 'mimeType':'text/plain', 'checksum': {'@type': 'SHA-1', '@value': '123789'}}]" + + curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/addFiles?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/datasets/:persistentId/addFiles?persistentId=doi:10.5072/FK2/7U7YBV -F jsonData='[{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123456"}}, {"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53", "fileName":"file2.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123789"}}]' + Updating File Metadata ~~~~~~~~~~~~~~~~~~~~~~ From 80a8f955955fd97a6756669c09dc4cff58b96fd1 Mon Sep 17 00:00:00 2001 From: Benjamin Peuch Date: Wed, 2 Jun 2021 09:10:17 +0200 Subject: [PATCH 175/354] Update 5.5-release-notes.md Fixing broken link --- doc/release-notes/5.5-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/5.5-release-notes.md b/doc/release-notes/5.5-release-notes.md index eb12070d2d7..7c4726995f8 100644 --- a/doc/release-notes/5.5-release-notes.md +++ b/doc/release-notes/5.5-release-notes.md @@ -16,7 +16,7 @@ Users trying to download a zip file larger than the Dataverse installation's :Zi ### Guidelines on Depositing Code -The Software Metadata Working Group has created guidelines on depositing research code in a Dataverse installation. Learn more in the [Dataset Management section](https://guides.dataverse.org/en/latest/dataset-management) of the Dataverse Guides. +The Software Metadata Working Group has created guidelines on depositing research code in a Dataverse installation. Learn more in the [Dataset Management section](https://guides.dataverse.org/en/latest/user/dataset-management.html#research-code) of the Dataverse Guides. ### New Metrics API From 2e3f69803ab0cd7d19dce8e79d4bc3e39dd361fc Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 2 Jun 2021 10:33:29 -0400 Subject: [PATCH 176/354] #7767 code cleanup --- .../harvard/iq/dataverse/api/Dataverses.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 2498641045d..99199a3e653 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -851,34 +851,29 @@ public Response getGroupByOwnerAndAliasInOwner(@PathParam("identifier") String d @Produces({"application/download"}) public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String dvIdtf, @QueryParam("guestbookId") Long gbId) { - Dataverse dv = findDataverse(dvIdtf); - - try { - - String filename = dv.getAlias() + "GBResponses.csv"; - String contentDispositionString = "attachment;filename=" + filename; - File file = new File(filename); - System.out.print("filename: " + filename); - ResponseBuilder response = Response.ok((Object) file); - response.header("Content-Disposition", contentDispositionString); + Dataverse dv = findDataverse(dvIdtf); + + try { + String filename = dv.getAlias() + "GBResponses.csv"; + String contentDispositionString = "attachment;filename=" + filename; + File file = new File(filename); + ResponseBuilder response = Response.ok((Object) file); + response.header("Content-Disposition", contentDispositionString); FileOutputStream outputStream = new FileOutputStream(filename); Map customQandAs = guestbookResponseService.mapCustomQuestionAnswersAsStrings(dv.getId(), gbId); - List guestbookResults = guestbookResponseService.getGuestbookResults(dv.getId(), gbId); + List guestbookResults = guestbookResponseService.getGuestbookResults(dv.getId(), gbId); outputStream.write("Guestbook, Dataset, Dataset PID, Date, Type, File Name, File Id, File PID, User Name, Email, Institution, Position, Custom Questions\n".getBytes()); for (Object[] result : guestbookResults) { StringBuilder sb = guestbookResponseService.convertGuestbookResponsesToCSV(customQandAs, result); outputStream.write(sb.toString().getBytes()); outputStream.flush(); } - return response.build(); } catch (IOException io) { - return error(Status.BAD_REQUEST, "exception: " + io.getMessage()); - + return error(Status.BAD_REQUEST, "Failed to produce response file. Exception: " + io.getMessage()); } - // return error(Status.BAD_REQUEST, "filler"); } @PUT From e888ed86fc2e74fb78d5509311e77a6214029a70 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 2 Jun 2021 11:33:51 -0400 Subject: [PATCH 177/354] #7767 add "OrDie" to find dv --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 99199a3e653..fa4f6f0f2f1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -851,9 +851,9 @@ public Response getGroupByOwnerAndAliasInOwner(@PathParam("identifier") String d @Produces({"application/download"}) public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String dvIdtf, @QueryParam("guestbookId") Long gbId) { - Dataverse dv = findDataverse(dvIdtf); - + try { + Dataverse dv = findDataverseOrDie(dvIdtf); String filename = dv.getAlias() + "GBResponses.csv"; String contentDispositionString = "attachment;filename=" + filename; File file = new File(filename); @@ -872,6 +872,8 @@ public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String return response.build(); } catch (IOException io) { return error(Status.BAD_REQUEST, "Failed to produce response file. Exception: " + io.getMessage()); + } catch (WrappedResponse wr) { + return wr.getResponse(); } } From 02fc565ba1b7bf6909851385e65d7553a15d5b4e Mon Sep 17 00:00:00 2001 From: JayanthyChengan Date: Wed, 2 Jun 2021 16:27:52 -0400 Subject: [PATCH 178/354] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 45f693818fa..ac937f04456 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2091,7 +2091,7 @@ Note: The ``id`` returned in the json response is the id of the file metadata ve Adding File Metadata ~~~~~~~~~~~~~~~~~~~~ -This API call requires a ``jsonString`` expressing the metadata of multiple files. It adds file metadata to the database table where the file already copied to the storage. +This API call requires a ``jsonString`` expressing the metadata of multiple files. It adds file metadata to the database table where the file has already been copied to the storage. The jsonData object includes values for: From 1ca650589fbbea48e5c2c27ffc60a2ca22084057 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 3 Jun 2021 09:54:07 -0400 Subject: [PATCH 179/354] #7767 add perms to download gb api --- .../java/edu/harvard/iq/dataverse/api/Dataverses.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index fa4f6f0f2f1..fc5213a2d79 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -854,6 +854,15 @@ public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String try { Dataverse dv = findDataverseOrDie(dvIdtf); + User u = findUserOrDie(); + DataverseRequest req = createDataverseRequest(u); + if (permissionSvc.request(req) + .on(dv) + .has(Permission.EditDataverse)) { + } else { + return error(Status.FORBIDDEN, "Not authorized"); + } + String filename = dv.getAlias() + "GBResponses.csv"; String contentDispositionString = "attachment;filename=" + filename; File file = new File(filename); From f47b268fb83f81b09ef99514ec26c6806a70c47a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 3 Jun 2021 12:47:38 -0400 Subject: [PATCH 180/354] update StringUtils package --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 72e6a3b21ee..79a34e6999a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -30,7 +30,7 @@ import javax.json.stream.JsonGenerator; import javax.ws.rs.BadRequestException; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import com.apicatalog.jsonld.JsonLd; import com.apicatalog.jsonld.api.JsonLdError; From 3864150dfbdb225e6d6f5c701bbc5a4c2e8e28d6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 3 Jun 2021 12:49:34 -0400 Subject: [PATCH 181/354] checkstyle fixes --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 6 +++--- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index e80c944f908..4f4034e7723 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -456,10 +456,10 @@ public Response recreateDataset(String jsonLDBody, @PathParam("identifier") Stri (ds.getAuthority().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Authority))&& ds.getProtocol().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Protocol))&& ds.getIdentifier().startsWith(settingsService.getValueForKey(SettingsServiceBean.Key.Shoulder)))) { - throw new BadRequestException("Cannot recreate a dataset that has a PID that doesn't match the server's settings"); + throw new BadRequestException("Cannot recreate a dataset that has a PID that doesn't match the server's settings"); } if(!datasetSvc.isIdentifierLocallyUnique(ds)) { - throw new BadRequestException("Cannot recreate a dataset whose PID is already in use"); + throw new BadRequestException("Cannot recreate a dataset whose PID is already in use"); } @@ -470,7 +470,7 @@ public Response recreateDataset(String jsonLDBody, @PathParam("identifier") Stri DatasetVersion version = ds.getVersions().get(0); if (!version.isPublished()) { - throw new BadRequestException("Cannot recreate a dataset that hasn't been published."); + throw new BadRequestException("Cannot recreate a dataset that hasn't been published."); } //While the datasetversion whose metadata we're importing has been published, we consider it in draft until the API caller adds files and then completes the migration version.setVersionState(DatasetVersion.VersionState.DRAFT); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 9a57e97ffdf..09811410606 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -30,7 +30,7 @@ import javax.json.stream.JsonGenerator; import javax.ws.rs.BadRequestException; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import com.apicatalog.jsonld.JsonLd; import com.apicatalog.jsonld.api.JsonLdError; From 9acea6b97fea28457d19009bb6e5a6e87202b440 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Sun, 6 Jun 2021 23:36:39 -0400 Subject: [PATCH 182/354] Do not count thumbnails and prep downloads, when redirecting to S3 (similarly to how these downloads are treated when done internally, without redirecting to the remote bucket, in line 457). #7924 --- .../edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index 7f3b652af33..71d653e5ae3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -208,7 +208,7 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] } if (redirect_uri != null) { // increment the download count, if necessary: - if (di.getGbr() != null) { + if (di.getGbr() != null && !(isThumbnailDownload(di) || isPreprocessedMetadataDownload(di))) { try { logger.fine("writing guestbook response, for an S3 download redirect."); Command cmd = new CreateGuestbookResponseCommand(di.getDataverseRequestService().getDataverseRequest(), di.getGbr(), di.getGbr().getDataFile().getOwner()); From 558395426e0153e44205cb5fbcb5879c52d475f2 Mon Sep 17 00:00:00 2001 From: Robert Treacy Date: Mon, 7 Jun 2021 12:00:32 -0400 Subject: [PATCH 183/354] LazyFileMetadataDataModel had code which was affected by a breaking change in the latest version of PrimeFaces. The class was not being used in the code and it was decided to remove it after talking to Steve and Gustavo. There was also an instance declared in the DatasetPage, but again it was not serving any function --- .../edu/harvard/iq/dataverse/DatasetPage.java | 10 ----- .../dataverse/LazyFileMetadataDataModel.java | 39 ------------------- 2 files changed, 49 deletions(-) delete mode 100644 src/main/java/edu/harvard/iq/dataverse/LazyFileMetadataDataModel.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index dbd762bba6e..97fc14a5485 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -438,16 +438,6 @@ public void setRemoveUnusedTags(boolean removeUnusedTags) { private String fileSortField; private String fileSortOrder; - private LazyFileMetadataDataModel lazyModel; - - public LazyFileMetadataDataModel getLazyModel() { - return lazyModel; - } - - public void setLazyModel(LazyFileMetadataDataModel lazyModel) { - this.lazyModel = lazyModel; - } - public List> getCartList() { if (session.getUser() instanceof AuthenticatedUser) { return ((AuthenticatedUser) session.getUser()).getCart().getContents(); diff --git a/src/main/java/edu/harvard/iq/dataverse/LazyFileMetadataDataModel.java b/src/main/java/edu/harvard/iq/dataverse/LazyFileMetadataDataModel.java deleted file mode 100644 index c23babc3660..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/LazyFileMetadataDataModel.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package edu.harvard.iq.dataverse; - -import java.util.List; -import java.util.Map; -import org.primefaces.model.FilterMeta; -import org.primefaces.model.LazyDataModel; -import org.primefaces.model.SortOrder; - -/** - * - * @author skraffmi - */ -public class LazyFileMetadataDataModel /* extends LazyDataModel */ { - - private final DataFileServiceBean fileServiceBean; - private final Long datasetVersionId; - - public LazyFileMetadataDataModel(Long datasetVersionId, DataFileServiceBean fileServiceBean) { - this.fileServiceBean = fileServiceBean; - this.datasetVersionId = datasetVersionId; - } - - -// @Override - public List load(int first, int pageSize, String sortField, - SortOrder sortOrder, Map filters) { - - List listFileMetadata = null; //fileServiceBean.findFileMetadataByDatasetVersionIdLazy(datasetVersionId, pageSize, sortField, sortField, first); - //this.setRowCount(fileServiceBean.findCountByDatasetVersionId(datasetVersionId).intValue()); - return listFileMetadata; - } - - -} From 645c40cc2787607e226aea813c0a6a01908b58d3 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 7 Jun 2021 15:30:45 -0400 Subject: [PATCH 184/354] remove references to Freenode IRC Network #7928 We've moved to Matrix: https://groups.google.com/g/dataverse-community/c/KU0FDNb8ckM/m/bO8ZXv2WAQAJ Even though we're on Matrix now, we don't explictly mention this. We just point people to https://chat.dataverse.org --- CONTRIBUTING.md | 8 +++----- doc/sphinx-guides/source/developers/intro.rst | 2 +- doc/sphinx-guides/source/installation/intro.rst | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2227286d4d1..cb7e5f9d123 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,12 +6,11 @@ We aren't just looking for developers. There are many ways to contribute to Data ## Ideas/Feature Requests -Your idea or feature request might already be captured in the Dataverse [issue tracker] on GitHub but if not, the best way to bring it to the community's attention is by posting on the [dataverse-community Google Group][] or bringing it up on a [Community Call][]. You're also welcome make some noise in the [#dataverse IRC channel][] (which is [logged][]) or cram your idea into 280 characters and mention [@dataverseorg][] on Twitter. To discuss your idea privately, please email it to support@dataverse.org +Your idea or feature request might already be captured in the Dataverse [issue tracker] on GitHub but if not, the best way to bring it to the community's attention is by posting on the [dataverse-community Google Group][] or bringing it up on a [Community Call][]. You're also welcome to make some noise in [chat.dataverse.org][] or cram your idea into 280 characters and mention [@dataverseorg][] on Twitter. To discuss your idea privately, please email it to support@dataverse.org There's a chance your idea is already on our roadmap, which is available at https://www.iq.harvard.edu/roadmap-dataverse-project -[#dataverse IRC channel]: http://chat.dataverse.org -[logged]: http://irclog.iq.harvard.edu/dataverse/today +[chat.dataverse.org]: http://chat.dataverse.org [issue tracker]: https://github.com/IQSS/dataverse/issues [@dataverseorg]: https://twitter.com/dataverseorg @@ -55,7 +54,7 @@ We love code contributions. Developers are not limited to the main Dataverse cod [API Guide]: http://guides.dataverse.org/en/latest/api [Installation Guide]: http://guides.dataverse.org/en/latest/installation -If you are interested in working on the main Dataverse code, great! Before you start coding, please reach out to us either on the [dataverse-community Google Group][], the [dataverse-dev Google Group][], [IRC][] (#dataverse on freenode), or via support@dataverse.org to make sure the effort is well coordinated and we avoid merge conflicts. We maintain a list of [community contributors][] and [dev efforts][] the community is working on so please let us know if you'd like to be added or removed from either list. +If you are interested in working on the main Dataverse code, great! Before you start coding, please reach out to us either on the [dataverse-community Google Group][], the [dataverse-dev Google Group][], [chat.dataverse.org][], or via support@dataverse.org to make sure the effort is well coordinated and we avoid merge conflicts. We maintain a list of [community contributors][] and [dev efforts][] the community is working on so please let us know if you'd like to be added or removed from either list. Please read http://guides.dataverse.org/en/latest/developers/version-control.html to understand how we use the "git flow" model of development and how we will encourage you to create a GitHub issue (if it doesn't exist already) to associate with your pull request. That page also includes tips on making a pull request. @@ -66,6 +65,5 @@ Thanks for your contribution! [dataverse-community Google Group]: https://groups.google.com/group/dataverse-community [Community Call]: https://dataverse.org/community-calls [dataverse-dev Google Group]: https://groups.google.com/group/dataverse-dev -[IRC]: http://chat.dataverse.org [community contributors]: https://docs.google.com/spreadsheets/d/1o9DD-MQ0WkrYaEFTD5rF_NtyL8aUISgURsAXSL7Budk/edit?usp=sharing [dev efforts]: https://github.com/orgs/IQSS/projects/2#column-5298405 diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index 29be6ab1d93..8fc0c679a8b 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -19,7 +19,7 @@ To get started, you'll want to set up your :doc:`dev-environment` and make sure Getting Help ------------ -If you have any questions at all, please reach out to other developers via the channels listed in https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md such as http://chat.dataverse.org (#dataverse on freenode), the `dataverse-dev `_ mailing list, `community calls `_, or support@dataverse.org. +If you have any questions at all, please reach out to other developers via the channels listed in https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md such as http://chat.dataverse.org, the `dataverse-dev `_ mailing list, `community calls `_, or support@dataverse.org. Core Technologies ----------------- diff --git a/doc/sphinx-guides/source/installation/intro.rst b/doc/sphinx-guides/source/installation/intro.rst index 6c6199af02d..4dd5f9e8795 100644 --- a/doc/sphinx-guides/source/installation/intro.rst +++ b/doc/sphinx-guides/source/installation/intro.rst @@ -36,7 +36,7 @@ Getting Help To get help installing or configuring a Dataverse installation, please try one or more of: - posting to the `dataverse-community `_ Google Group. -- asking at http://chat.dataverse.org (#dataverse on the freenode IRC network) +- asking at http://chat.dataverse.org - emailing support@dataverse.org to open a private ticket at https://help.hmdc.harvard.edu Improving this Guide From bf72ff9c0a42ed2f1b79a3ae7ba4141acbbf4de2 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 8 Jun 2021 10:27:25 +0200 Subject: [PATCH 185/354] chore(appserver): update to Payara 5.2021.4 #7700 --- conf/docker-aio/0prep_deps.sh | 4 ++-- conf/docker-aio/c8.dockerfile | 2 +- doc/release-notes/7700-upgrade-payara.md | 6 +++--- doc/sphinx-guides/source/developers/dev-environment.rst | 4 ++-- doc/sphinx-guides/source/installation/prerequisites.rst | 6 +++--- downloads/download.sh | 2 +- pom.xml | 2 +- scripts/vagrant/setup.sh | 2 +- .../edu/harvard/iq/dataverse/util/DataSourceProducer.java | 2 +- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/conf/docker-aio/0prep_deps.sh b/conf/docker-aio/0prep_deps.sh index 3f4e3d53e81..c26492b2d25 100755 --- a/conf/docker-aio/0prep_deps.sh +++ b/conf/docker-aio/0prep_deps.sh @@ -4,10 +4,10 @@ if [ ! -d dv/deps ]; then fi wdir=`pwd` -if [ ! -e dv/deps/payara-5.2021.3.zip ]; then +if [ ! -e dv/deps/payara-5.2021.4.zip ]; then echo "payara dependency prep" # no more fiddly patching :) - wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.3/payara-5.2021.3.zip -O dv/deps/payara-5.2021.3.zip + wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.4/payara-5.2021.4.zip -O dv/deps/payara-5.2021.4.zip fi if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then diff --git a/conf/docker-aio/c8.dockerfile b/conf/docker-aio/c8.dockerfile index 328ef98d342..4fe12201fc3 100644 --- a/conf/docker-aio/c8.dockerfile +++ b/conf/docker-aio/c8.dockerfile @@ -23,7 +23,7 @@ COPY disableipv6.conf /etc/sysctl.d/ RUN rm /etc/httpd/conf/* COPY httpd.conf /etc/httpd/conf RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz -RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.3.zip ; ln -s /opt/payara5 /opt/glassfish4 +RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.4.zip ; ln -s /opt/payara5 /opt/glassfish4 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm` # this dies under Java 11, do we keep it? diff --git a/doc/release-notes/7700-upgrade-payara.md b/doc/release-notes/7700-upgrade-payara.md index 7f76d993b18..de7c58bf963 100644 --- a/doc/release-notes/7700-upgrade-payara.md +++ b/doc/release-notes/7700-upgrade-payara.md @@ -1,9 +1,9 @@ -### Payara 5.2021.3 (or Higher) Required +### Payara 5.2021.4 (or Higher) Required -Some changes in this release require an upgrade to Payara 5.2021.3 or higher. +Some changes in this release require an upgrade to Payara 5.2021.4 or higher. Instructions on how to update can be found in the -[Payara documentation](https://docs.payara.fish/community/docs/5.2021.3/documentation/user-guides/upgrade-payara.html) +[Payara documentation](https://docs.payara.fish/community/docs/5.2021.4/documentation/user-guides/upgrade-payara.html) It would likely be safer to upgrade Payara first, while still running Dataverse 5.6, and then proceed with the steps below. Upgrading from an earlier version of Payara should be a straightforward process: diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index fe95d0d85c3..41e986c0ea3 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -85,9 +85,9 @@ To install Payara, run the following commands: ``cd /usr/local`` -``sudo curl -O -L https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.3/payara-5.2021.3.zip`` +``sudo curl -O -L https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.4/payara-5.2021.4.zip`` -``sudo unzip payara-5.2021.3.zip`` +``sudo unzip payara-5.2021.4.zip`` ``sudo chown -R $USER /usr/local/payara5`` diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 8e886212a73..4f04024607f 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -44,7 +44,7 @@ On RHEL/derivative you can make Java 11 the default with the ``alternatives`` co Payara ------ -Payara 5.2021.3 is recommended. Newer versions might work fine, regular updates are recommended. +Payara 5.2021.4 is recommended. Newer versions might work fine, regular updates are recommended. Installing Payara ================= @@ -55,8 +55,8 @@ Installing Payara - Download and install Payara (installed in ``/usr/local/payara5`` in the example commands below):: - # wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.3/payara-5.2021.3.zip - # unzip payara-5.2021.3.zip + # wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.4/payara-5.2021.4.zip + # unzip payara-5.2021.4.zip # mv payara5 /usr/local If you intend to install and run Payara under a service account (and we hope you do), chown -R the Payara hierarchy to root to protect it but give the service account access to the below directories: diff --git a/downloads/download.sh b/downloads/download.sh index 957f54dd599..c9cd2942295 100755 --- a/downloads/download.sh +++ b/downloads/download.sh @@ -1,5 +1,5 @@ #!/bin/sh -curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.3/payara-5.2021.3.zip +curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.4/payara-5.2021.4.zip curl -L -O https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar diff --git a/pom.xml b/pom.xml index 4a175af124d..0eb489b4c15 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ false 8.0.0 - 5.2021.3 + 5.2021.4 42.2.19 1.11.762 1.2 diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh index 84c9a094ca9..fdce9210ef4 100644 --- a/scripts/vagrant/setup.sh +++ b/scripts/vagrant/setup.sh @@ -53,7 +53,7 @@ SOLR_USER=solr echo "Ensuring Unix user '$SOLR_USER' exists" useradd $SOLR_USER || : DOWNLOAD_DIR='/dataverse/downloads' -PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2021.3.zip" +PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2021.4.zip" SOLR_TGZ="$DOWNLOAD_DIR/solr-8.8.1.tgz" if [ ! -f $PAYARA_ZIP ] || [ ! -f $SOLR_TGZ ]; then echo "Couldn't find $PAYARA_ZIP or $SOLR_TGZ! Running download script...." diff --git a/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java b/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java index e4f2cf7fa7b..b393ca4a605 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java @@ -38,7 +38,7 @@ //}) // // ... but at this time we don't think we need any. The full list -// of properties can be found at https://docs.payara.fish/community/docs/5.2021.3/documentation/payara-server/jdbc/advanced-connection-pool-properties.html#full-list-of-properties +// of properties can be found at https://docs.payara.fish/community/docs/5.2021.4/documentation/payara-server/jdbc/advanced-connection-pool-properties.html#full-list-of-properties // // All these properties cannot be configured via MPCONFIG as Payara doesn't support this (yet). To be enhanced. // See also https://github.com/payara/Payara/issues/5024 From e711fb3beebe18394ad1c828f5b1f82787f1e07b Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 8 Jun 2021 11:07:36 -0400 Subject: [PATCH 186/354] #7767 update response building --- .../harvard/iq/dataverse/api/Dataverses.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index fc5213a2d79..6a7631e5de9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -100,15 +100,15 @@ import javax.ws.rs.core.Response.Status; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.toJsonArray; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.text.MessageFormat; import java.util.Arrays; import java.util.Date; import java.util.Map; import java.util.Optional; -import javax.ws.rs.core.Response.ResponseBuilder; +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.Context; import javax.xml.stream.XMLStreamException; /** @@ -850,7 +850,7 @@ public Response getGroupByOwnerAndAliasInOwner(@PathParam("identifier") String d @Path("{identifier}/guestbookResponses/") @Produces({"application/download"}) public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String dvIdtf, - @QueryParam("guestbookId") Long gbId) { + @QueryParam("guestbookId") Long gbId, @Context HttpServletResponse response) { try { Dataverse dv = findDataverseOrDie(dvIdtf); @@ -863,12 +863,12 @@ public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String return error(Status.FORBIDDEN, "Not authorized"); } - String filename = dv.getAlias() + "GBResponses.csv"; - String contentDispositionString = "attachment;filename=" + filename; - File file = new File(filename); - ResponseBuilder response = Response.ok((Object) file); - response.header("Content-Disposition", contentDispositionString); - FileOutputStream outputStream = new FileOutputStream(filename); + String filename = dv.getAlias() + "_GBResponses.csv"; + + response.setHeader("Content-Disposition", "attachment; filename=" + + filename); + ServletOutputStream outputStream = response.getOutputStream(); + Map customQandAs = guestbookResponseService.mapCustomQuestionAnswersAsStrings(dv.getId(), gbId); List guestbookResults = guestbookResponseService.getGuestbookResults(dv.getId(), gbId); @@ -878,7 +878,7 @@ public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String outputStream.write(sb.toString().getBytes()); outputStream.flush(); } - return response.build(); + return Response.ok().build(); } catch (IOException io) { return error(Status.BAD_REQUEST, "Failed to produce response file. Exception: " + io.getMessage()); } catch (WrappedResponse wr) { @@ -886,7 +886,7 @@ public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String } } - + @PUT @Path("{identifier}/groups/{aliasInOwner}") public Response updateGroup(ExplicitGroupDTO groupDto, From dc5e29e1be14afd69dfeb528fb8e66b21178ceeb Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 8 Jun 2021 14:06:53 -0400 Subject: [PATCH 187/354] #7767 add timestamp to file name --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 6a7631e5de9..ebc52785637 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -102,6 +102,7 @@ import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; import java.io.IOException; import java.text.MessageFormat; +import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Date; import java.util.Map; @@ -121,6 +122,7 @@ public class Dataverses extends AbstractApiBean { private static final Logger logger = Logger.getLogger(Dataverses.class.getCanonicalName()); + private static final SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss"); @EJB ExplicitGroupServiceBean explicitGroupSvc; @@ -863,7 +865,8 @@ public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String return error(Status.FORBIDDEN, "Not authorized"); } - String filename = dv.getAlias() + "_GBResponses.csv"; + String fileTimestamp = dateFormatter.format(new Date()); + String filename = dv.getAlias() + "_GBResponses_" + fileTimestamp + ".csv"; response.setHeader("Content-Disposition", "attachment; filename=" + filename); From 6288b5e059eeb0357f008adfc492ebfb9b69e128 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 8 Jun 2021 14:57:58 -0400 Subject: [PATCH 188/354] Add docs for GB response api --- doc/sphinx-guides/source/api/native-api.rst | 23 ++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 3a3f2fe5cec..f5317ac2a47 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -580,10 +580,31 @@ The fully expanded example above (without environment variables) looks like this .. code-block:: bash - curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root/actions/:publish + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root/actions/:publish You should expect a 200 ("OK") response and JSON output. +Retrieve Guestbook Responses for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In order to retrieve a file containing a list of Guestbook Responses in csv format for Dataverse collection, you must know either its "alias" (which the GUI calls an "identifier") or its database ID. If the Dataverse collection has more than one guestbook you may provide the id of a single guestbook as an optional parameter. If the guestbook id is provided then only those responses from that guestbook will be included in the file. + +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -O -J -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/guestbookResponses?guestbookId=1 + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -O -J -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - https://demo.dataverse.org/api/dataverses/root/guestbookResponses?guestbookId=1 + Datasets -------- From 4087673a92701f9352fa3e9dfd9a207f93df0a2f Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 8 Jun 2021 15:35:53 -0400 Subject: [PATCH 189/354] #7766 consolidate adding linking paths --- .../harvard/iq/dataverse/search/IndexServiceBean.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index ec6e0f0a84e..d72e2a7f642 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1496,15 +1496,9 @@ private List retrieveDVOPaths(DvObject dvo) { } List dataversePaths = getDataversePathsFromSegments(dataverseSegments); /* - First get all linking dataverses - Then get their respective paths - And finally add to the "normal" path + add linking paths */ - List linkingDataverses = findAllLinkingDataverses(dvo); - List linkingDataversePaths = findLinkingDataversePaths(linkingDataverses); - for (String dvPath : linkingDataversePaths) { - dataversePaths.add(dvPath); - } + dataversePaths.addAll(findLinkingDataversePaths(findAllLinkingDataverses(dvo))); return dataversePaths; } From 6e7d2db9ae4b5f5128abdadcb12ab9a9d87f0c4b Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 8 Jun 2021 16:40:44 -0400 Subject: [PATCH 190/354] Put in export for GUESTBOOK_ID --- doc/sphinx-guides/source/api/native-api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index f5317ac2a47..51d10d50718 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -596,8 +596,9 @@ In order to retrieve a file containing a list of Guestbook Responses in csv form export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export ID=root + export GUESTBOOK_ID=root - curl -O -J -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/guestbookResponses?guestbookId=1 + curl -O -J -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/guestbookResponses?guestbookId=$GUESTBOOK_ID The fully expanded example above (without environment variables) looks like this: From 5186396538c67fa995cd76996d2b46ead235eb06 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 8 Jun 2021 16:43:07 -0400 Subject: [PATCH 191/354] typo(copy/paste-o) --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 51d10d50718..d990d18aec5 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -596,7 +596,7 @@ In order to retrieve a file containing a list of Guestbook Responses in csv form export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export ID=root - export GUESTBOOK_ID=root + export GUESTBOOK_ID=1 curl -O -J -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/guestbookResponses?guestbookId=$GUESTBOOK_ID From fd37485001bd18b88f07dfb3066a79dd9cfe9e4f Mon Sep 17 00:00:00 2001 From: "don.sizemore" Date: Fri, 11 Jun 2021 16:12:25 -0400 Subject: [PATCH 192/354] #7660 bump Postgres to version 13 --- conf/docker-aio/c8.dockerfile | 11 ++++++----- conf/docker-aio/entrypoint.bash | 6 +++--- scripts/vagrant/setup.sh | 15 +++++++-------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/conf/docker-aio/c8.dockerfile b/conf/docker-aio/c8.dockerfile index 72643d0e566..1ccc37e048c 100644 --- a/conf/docker-aio/c8.dockerfile +++ b/conf/docker-aio/c8.dockerfile @@ -1,10 +1,11 @@ FROM centos:8 # OS dependencies -# PG 10 is the default in centos8; keep the repo comment for when we bump to 11+ -#RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm +# IQSS now recommends Postgres 13. +RUN dnf -qy module disable postgresql +RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf -RUN yum install -y java-11-openjdk-devel postgresql-server sudo epel-release unzip curl httpd python2 diffutils +RUN yum install -y java-11-openjdk-devel postgresql13-server sudo epel-release unzip curl httpd python2 diffutils RUN yum install -y jq lsof awscli # for older search scripts @@ -29,10 +30,10 @@ RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2020.6.zip ; ln -s /opt/payara5 /opt/g # this dies under Java 11, do we keep it? #COPY domain-restmonitor.xml /opt/payara5/glassfish/domains/domain1/config/domain.xml -RUN sudo -u postgres /usr/bin/initdb /var/lib/pgsql/data +RUN sudo -u postgres /usr/pgsql-13/bin/initdb -D /var/lib/pgsql/13/data # copy configuration related files -RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/ +RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/13/data/ RUN cp -r /opt/solr-8.8.1/server/solr/configsets/_default /opt/solr-8.8.1/server/solr/collection1 RUN cp /tmp/dv/schema*.xml /opt/solr-8.8.1/server/solr/collection1/conf/ RUN cp /tmp/dv/solrconfig.xml /opt/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml diff --git a/conf/docker-aio/entrypoint.bash b/conf/docker-aio/entrypoint.bash index 6cb5463cd1a..90f60da20f2 100755 --- a/conf/docker-aio/entrypoint.bash +++ b/conf/docker-aio/entrypoint.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash export LANG=en_US.UTF-8 -sudo -u postgres /usr/bin/pg_ctl start -D /var/lib/pgsql/data & +sudo -u postgres /usr/pgsql-13/bin/pg_ctl start -D /var/lib/pgsql/13/data & cd /opt/solr-8.8.1/ # TODO: Run Solr as non-root and remove "-force". bin/solr start -force @@ -9,8 +9,8 @@ bin/solr create_core -c collection1 -d server/solr/collection1/conf -force # start apache, in both foreground and background... apachectl -DFOREGROUND & -# TODO: Run Glassfish as non-root. -cd /opt/glassfish4 +# TODO: Run Payara as non-root. +cd /opt/payara5 bin/asadmin start-domain --debug sleep infinity diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh index 14f12cea692..1c075fde447 100644 --- a/scripts/vagrant/setup.sh +++ b/scripts/vagrant/setup.sh @@ -35,16 +35,15 @@ echo "export MAVEN_HOME=/opt/maven" >> /etc/profile.d/maven.sh echo "export PATH=/opt/maven/bin:${PATH}" >> /etc/profile.d/maven.sh chmod 0755 /etc/profile.d/maven.sh -# disable centos8 postgresql module and install postgresql10-server -# note: postgresql10 because 9.6 isn't backwards compatible with python3-psycopg2 +# disable centos8 postgresql module and install postgresql13-server dnf -qy module disable postgresql dnf install -qy https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm -dnf install -qy postgresql10-server -/usr/pgsql-10/bin/postgresql-10-setup initdb -/usr/bin/systemctl stop postgresql-10 -cp /dataverse/conf/vagrant/var/lib/pgsql/data/pg_hba.conf /var/lib/pgsql/10/data/pg_hba.conf -/usr/bin/systemctl start postgresql-10 -/usr/bin/systemctl enable postgresql-10 +dnf install -qy postgresql13-server +/usr/pgsql-13/bin/postgresql-13-setup initdb +/usr/bin/systemctl stop postgresql-13 +cp /dataverse/conf/vagrant/var/lib/pgsql/data/pg_hba.conf /var/lib/pgsql/13/data/pg_hba.conf +/usr/bin/systemctl start postgresql-13 +/usr/bin/systemctl enable postgresql-13 PAYARA_USER=dataverse echo "Ensuring Unix user '$PAYARA_USER' exists" From e7f794f655ce2bfb17c8cd49e7c007f8f43c5b4c Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 15 Jun 2021 09:56:48 +0200 Subject: [PATCH 193/354] fix(ci): remove broken Travis CI config #7876 --- .../workflows/cypress_ui.yml.future | 17 +++++++++-------- .travis.yml | 5 ----- 2 files changed, 9 insertions(+), 13 deletions(-) rename .travis.yml.future => .github/workflows/cypress_ui.yml.future (74%) delete mode 100644 .travis.yml diff --git a/.travis.yml.future b/.github/workflows/cypress_ui.yml.future similarity index 74% rename from .travis.yml.future rename to .github/workflows/cypress_ui.yml.future index 8bd747625e4..b38ae2f9558 100644 --- a/.travis.yml.future +++ b/.github/workflows/cypress_ui.yml.future @@ -1,16 +1,17 @@ +############################################################################### +# +# THIS IS AN OLD TRAVIS-CI.ORG JOB FILE +# To be used with Github Actions, it would be necessary to refactor it. +# Keeping it as the future example it has been before. +# See also #5846 +# +############################################################################### + services: - docker jobs: include: - # Execute java unit- and integration tests - - stage: test - language: java - jdk: - - oraclejdk8 - script: mvn -DcompilerArgument=-Xlint:unchecked test -P all-unit-tests - after_success: mvn jacoco:report coveralls:report - # Execute Cypress for UI testing # see https://docs.cypress.io/guides/guides/continuous-integration.html - stage: test diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 67de6619add..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,5 +0,0 @@ -language: java -jdk: - - openjdk11 -script: mvn -DcompilerArgument=-Xlint:unchecked test -P all-unit-tests -after_success: mvn jacoco:report coveralls:report From efb4537bb70e38f2fd556c0c10e52cf014ff379a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 15 Jun 2021 09:57:42 +0200 Subject: [PATCH 194/354] fix(ci): readd unit tests + code cov with Github Action #7876 --- .github/workflows/maven_unit_test.yml | 45 +++++++++++++++++++++++++++ pom.xml | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/maven_unit_test.yml diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml new file mode 100644 index 00000000000..916f0e7ecbb --- /dev/null +++ b/.github/workflows/maven_unit_test.yml @@ -0,0 +1,45 @@ +name: Maven Unit Tests + +on: + push: + paths: + - "**.java" + pull_request: + paths: + - "**.java" + +jobs: + unittest: + name: (JDK ${{ matrix.jdk }} / ${{ matrix.os }}) Unit Tests + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + jdk: [ '11' ] + #include: + # - os: ubuntu-latest + # jdk: '16' + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: ${{ matrix.jdk }} + distribution: 'adopt' + - name: Cache Maven packages + uses: actions/cache@v2 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + - name: Build with Maven + run: mvn -DcompilerArgument=-Xlint:unchecked -P all-unit-tests clean test + - name: Maven Code Coverage + env: + CI_NAME: github + BRANCH_NAME_OR_REF: ${{ github.head_ref || github.ref }} + CI_BUILD_NUMBER: ${{ github.run_id }} + CI_BUILD_URL: https://github.com/${{ github.repository }}/commit/${{ github.event.after }}/checks + COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }} + run: mvn -V -B jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} \ No newline at end of file diff --git a/pom.xml b/pom.xml index ca2d3570956..d3aaf2fc90e 100644 --- a/pom.xml +++ b/pom.xml @@ -783,7 +783,7 @@ org.eluder.coveralls coveralls-maven-plugin - 4.0.0 + 4.3.0 javax.xml.bind From b337fe8db39aef9e9545aa42ac81b6be1e08d795 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 16 Jun 2021 12:11:20 +0200 Subject: [PATCH 195/354] fix(ci): switch status badge from Travis to Github CI #7876 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f3b8c2de90..6fd11374353 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi [![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/) [![API Test Coverage](https://img.shields.io/jenkins/coverage/jacoco?jobUrl=https%3A%2F%2Fjenkins.dataverse.org%2Fjob%2FIQSS-dataverse-develop&label=API%20Test%20Coverage)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/ws/target/coverage-it/index.html) -[![Unit Test Status](https://img.shields.io/travis/IQSS/dataverse?label=Unit%20Test%20Status)](https://travis-ci.org/IQSS/dataverse) +[![Unit Test Status](https://github.com/IQSS/dataverse/actions/workflows/maven_unit_test.yml/badge.svg?branch=develop)](https://github.com/IQSS/dataverse/actions/workflows/maven_unit_test.yml) [![Unit Test Coverage](https://img.shields.io/coveralls/github/IQSS/dataverse?label=Unit%20Test%20Coverage)](https://coveralls.io/github/IQSS/dataverse?branch=develop) [![Guides Build Status](https://github.com/IQSS/dataverse/actions/workflows/guides_build_sphinx.yml/badge.svg)](https://github.com/IQSS/dataverse/actions/workflows/guides_build_sphinx.yml) From 879c1993082d3834b16bafd80ac0d5202885e1c9 Mon Sep 17 00:00:00 2001 From: Ouahalou Youssef <46802308+youssefOuahalou@users.noreply.github.com> Date: Thu, 17 Jun 2021 18:58:38 +0200 Subject: [PATCH 196/354] Update 5.4-release-notes.md --- doc/release-notes/5.4-release-notes.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/release-notes/5.4-release-notes.md b/doc/release-notes/5.4-release-notes.md index 0f34db254fa..fff9de78e34 100644 --- a/doc/release-notes/5.4-release-notes.md +++ b/doc/release-notes/5.4-release-notes.md @@ -94,6 +94,11 @@ There are some things to note and keep in mind regarding the move to Java 11: `$ sudo yum install java-11-openjdk-devel` The `remove` command may provide an error message if -headless isn't installed. + + - Select the latest version of Java + + `$ sudo alternatives --config java` + - We targeted and tested Java 11, but 11+ will likely work. Java 11 was targeted because of its long term support. - If you're moving from a Dataverse installation that was previously running Glassfish 4.x (typically this would be Dataverse Software 4.x), you will need to adjust some JVM options in domain.xml as part of the upgrade process. We've provided these optional steps below. These steps are not required if your first installed Dataverse Software version was running Payara 5.x (typically Dataverse Software 5.x). From f7055a373e3e80ea808d38af677897115563ed71 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 16 Jun 2021 16:12:40 -0400 Subject: [PATCH 197/354] don't show citation downloads in file table menu/file page --- src/main/webapp/file-download-button-fragment.xhtml | 2 +- src/main/webapp/file.xhtml | 1 + src/main/webapp/filesFragment.xhtml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index ac8f7c28c40..255865a0c6d 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -197,7 +197,7 @@ -
  • +
  • #{bundle['file.downloadBtn.format.citation']}
    • diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index c1a649cff87..7086217606a 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -197,6 +197,7 @@ + diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 44d1119fa88..ca14dc29b53 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -500,6 +500,7 @@ + From 79ea29d6585c5735cfd54c49ed96df67ada0f9df Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 17 Jun 2021 09:58:56 -0400 Subject: [PATCH 198/354] typo --- src/main/webapp/file-download-button-fragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index 255865a0c6d..84ddb4b0005 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -197,7 +197,7 @@ -
    • #{bundle['file.downloadBtn.format.citation']}
      • From 1b6ae4893f7bc1603e22a7f639e28b5e4098cd86 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 17 Jun 2021 15:10:21 -0400 Subject: [PATCH 199/354] block most api calls for anonymized --- .../edu/harvard/iq/dataverse/api/AbstractApiBean.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index d21524a7150..74c5dc6933d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -362,7 +362,16 @@ protected User findUserOrDie() throws WrappedResponse { return GuestUser.get(); } PrivateUrlUser privateUrlUser = privateUrlSvc.getPrivateUrlUserFromToken(requestApiKey); + // For privateUrlUsers restricted to anonymized access, all api calls are off-limits except for those used in the UI + // to download the file or image thumbs if (privateUrlUser != null) { + if (privateUrlUser.hasAnonymizedAccess()) { + String pathInfo = httpRequest.getPathInfo(); + if (!(pathInfo.startsWith("/access/datafile/") && !pathInfo.substring(17).contains("/"))) { + logger.info("Anonymized access request for " + pathInfo); + return GuestUser.get(); + } + } return privateUrlUser; } return findAuthenticatedUserOrDie(requestApiKey, requestWFKey); From 9d2a6793f0d74de73359fb582b1df67dee7c6cd3 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 17 Jun 2021 15:28:16 -0400 Subject: [PATCH 200/354] Send unauth error --- src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 74c5dc6933d..5fef7af56cf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -369,7 +369,7 @@ protected User findUserOrDie() throws WrappedResponse { String pathInfo = httpRequest.getPathInfo(); if (!(pathInfo.startsWith("/access/datafile/") && !pathInfo.substring(17).contains("/"))) { logger.info("Anonymized access request for " + pathInfo); - return GuestUser.get(); + throw new WrappedResponse(error(Status.UNAUTHORIZED, "API Access not allowed with this Key")); } } return privateUrlUser; From 0a5c85810e39cde863323be0737ff9c22256e127 Mon Sep 17 00:00:00 2001 From: Robert Verkerk Date: Fri, 18 Jun 2021 11:34:00 +0200 Subject: [PATCH 201/354] Implement usage of user supplied handle for authentication. This is an optional parameter. --- doc/sphinx-guides/source/installation/config.rst | 11 +++++++++++ .../harvard/iq/dataverse/HandlenetServiceBean.java | 4 +++- .../iq/dataverse/settings/SettingsServiceBean.java | 5 +++++ .../edu/harvard/iq/dataverse/util/SystemConfig.java | 5 +++++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 39f27f749dc..6e4214ae90c 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -205,6 +205,7 @@ Here are the configuration options for handles: - :ref:`:IdentifierGenerationStyle <:IdentifierGenerationStyle>` (optional) - :ref:`:DataFilePIDFormat <:DataFilePIDFormat>` (optional) - :ref:`:IndependentHandleService <:IndependentHandleService>` (optional) +- :ref:`:HandleAuthHandle <:HandleAuthHandle>` (optional) Note: If you are **minting your own handles** and plan to set up your own handle service, please refer to `Handle.Net documentation `_. @@ -1543,6 +1544,16 @@ By default this setting is absent and the Dataverse Software assumes it to be fa ``curl -X PUT -d 'true' http://localhost:8080/api/admin/settings/:IndependentHandleService`` +.. _:HandleAuthHandle: + +:HandleAuthHandle ++++++++++++++++++++++++++ + +Specific for Handle PIDs. Set this setting to to be used on a global handle service when the public key is NOT stored in the default handle. +By default this setting is absent and the Dataverse Software assumes it to be not set. + +``curl -X PUT -d '' http://localhost:8080/api/admin/settings/:HandleAuthHandle`` + .. _:FileValidationOnPublishEnabled: :FileValidationOnPublishEnabled diff --git a/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java index defb884fd3e..3881b691ed8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java @@ -314,7 +314,9 @@ private String getAuthenticationHandle(DvObject dvObject){ private String getAuthenticationHandle(String handlePrefix) { logger.log(Level.FINE,"getAuthenticationHandle"); - if (systemConfig.isIndependentHandleService()) { + if (systemConfig.getHandleAuthHandle()) { + return systemConfig.getHandleAuthHandle(); + } else if (systemConfig.isIndependentHandleService()) { return handlePrefix + "/ADMIN"; } else { return "0.NA/" + handlePrefix; diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 02637bfa8df..684fd895aab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -356,6 +356,11 @@ Whether Harvesting (OAI) service is enabled */ IndependentHandleService, + /** + Handle to use for authentication if the default is not being used + */ + HandleAuthHandle, + /** * Archiving can be configured by providing an Archiver class name (class must extend AstractSubmitToArchiverCommand) * and a list of settings that should be passed to the Archiver. diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 71accdaa8c8..8a282f2dfa6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -1045,6 +1045,11 @@ public boolean isIndependentHandleService() { } + public string getHandleAuthHandle() { + string HandleAuthHandle = settingsService.getValueForKey(SettingsServiceBean.Key.HandleAuthHandle, null); + return HandleAuthHandle + } + public String getMDCLogPath() { String mDCLogPath = settingsService.getValueForKey(SettingsServiceBean.Key.MDCLogPath, null); return mDCLogPath; From b18637848682950ef764a0695697e02d6afb84c9 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Jun 2021 13:49:28 -0400 Subject: [PATCH 202/354] expand /traces API --- .../edu/harvard/iq/dataverse/api/Users.java | 23 +- .../command/impl/GetUserTracesCommand.java | 319 +++++++++--------- 2 files changed, 190 insertions(+), 152 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index ce226ea14b8..bc67980927b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -14,9 +14,14 @@ import edu.harvard.iq.dataverse.engine.command.impl.MergeInAccountCommand; import edu.harvard.iq.dataverse.engine.command.impl.RevokeAllRolesCommand; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.logging.Logger; import javax.ejb.Stateless; import javax.json.JsonObjectBuilder; +import javax.ws.rs.BadRequestException; import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -214,7 +219,23 @@ public Response removeUserRoles(@PathParam("identifier") String identifier) { public Response getTraces(@PathParam("identifier") String identifier) { try { AuthenticatedUser userToQuery = authSvc.getAuthenticatedUser(identifier); - JsonObjectBuilder jsonObj = execCommand(new GetUserTracesCommand(createDataverseRequest(findUserOrDie()), userToQuery)); + JsonObjectBuilder jsonObj = execCommand(new GetUserTracesCommand(createDataverseRequest(findUserOrDie()), userToQuery, null)); + return ok(jsonObj); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + + private List elements = Arrays.asList("roleAssignments","dataversCreator", "dataversePublisher","datasetCreator", "datasetPublisher","dataFileCreator","dataFilePublisher","datasetVersionUsers","explicitGroups","guestbookEntries", "savedSearches"); + @GET + @Path("{identifier}/traces/{element}") + public Response getTraces(@PathParam("identifier") String identifier, @PathParam("element") String element) { + try { + AuthenticatedUser userToQuery = authSvc.getAuthenticatedUser(identifier); + if(!elements.contains(element)) { + throw new BadRequestException("Not a valid element"); + } + JsonObjectBuilder jsonObj = execCommand(new GetUserTracesCommand(createDataverseRequest(findUserOrDie()), userToQuery, null)); return ok(jsonObj); } catch (WrappedResponse ex) { return ex.getResponse(); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java index 41a1708e4c5..5b6edcb6d8f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java @@ -30,11 +30,13 @@ public class GetUserTracesCommand extends AbstractCommand { private DataverseRequest request; private AuthenticatedUser user; + private String element; - public GetUserTracesCommand(DataverseRequest request, AuthenticatedUser user) { + public GetUserTracesCommand(DataverseRequest request, AuthenticatedUser user, String element) { super(request, (DvObject) null); this.request = request; this.user = user; + this.element = element; } @Override @@ -47,180 +49,195 @@ public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { } Long userId = user.getId(); JsonObjectBuilder traces = Json.createObjectBuilder(); -// List roleAssignments = ctxt.permissions().getDvObjectsUserHasRoleOn(user); - List roleAssignments = ctxt.roleAssignees().getAssignmentsFor(user.getIdentifier()); - if (roleAssignments != null && !roleAssignments.isEmpty()) { - JsonObjectBuilder job = Json.createObjectBuilder(); - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (RoleAssignment roleAssignment : roleAssignments) { - jab.add(NullSafeJsonBuilder.jsonObjectBuilder() - .add("id", roleAssignment.getId()) - .add("definitionPointName", roleAssignment.getDefinitionPoint().getCurrentName()) - .add("definitionPointIdentifier", roleAssignment.getDefinitionPoint().getIdentifier()) - .add("definitionPointId", roleAssignment.getDefinitionPoint().getId()) - .add("roleAlias", roleAssignment.getRole().getAlias()) - .add("roleName", roleAssignment.getRole().getName()) - ); + if (element == null || element.equals("roleAssignments")) { + // List roleAssignments = + // ctxt.permissions().getDvObjectsUserHasRoleOn(user); + List roleAssignments = ctxt.roleAssignees().getAssignmentsFor(user.getIdentifier()); + if (roleAssignments != null && !roleAssignments.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (RoleAssignment roleAssignment : roleAssignments) { + jab.add(NullSafeJsonBuilder.jsonObjectBuilder() + .add("id", roleAssignment.getId()) + .add("definitionPointName", roleAssignment.getDefinitionPoint().getCurrentName()) + .add("definitionPointIdentifier", roleAssignment.getDefinitionPoint().getIdentifier()) + .add("definitionPointId", roleAssignment.getDefinitionPoint().getId()) + .add("roleAlias", roleAssignment.getRole().getAlias()) + .add("roleName", roleAssignment.getRole().getName())); + } + job.add("count", roleAssignments.size()); + job.add("items", jab); + traces.add("roleAssignments", job); } - job.add("count", roleAssignments.size()); - job.add("items", jab); - traces.add("roleAssignments", job); } - List dataversesCreated = ctxt.dataverses().findByCreatorId(userId); - if (dataversesCreated != null && !dataversesCreated.isEmpty()) { - JsonObjectBuilder job = Json.createObjectBuilder(); - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (Dataverse dataverse : dataversesCreated) { - jab.add(Json.createObjectBuilder() - .add("id", dataverse.getId()) - .add("alias", dataverse.getAlias()) - ); + if (element == null || element.equals("dataverseCreator")) { + List dataversesCreated = ctxt.dataverses().findByCreatorId(userId); + if (dataversesCreated != null && !dataversesCreated.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (Dataverse dataverse : dataversesCreated) { + jab.add(Json.createObjectBuilder() + .add("id", dataverse.getId()) + .add("alias", dataverse.getAlias())); + } + job.add("count", dataversesCreated.size()); + job.add("items", jab); + traces.add("dataverseCreator", job); } - job.add("count", dataversesCreated.size()); - job.add("items", jab); - traces.add("dataverseCreator", job); } - List dataversesPublished = ctxt.dataverses().findByReleaseUserId(userId); - if (dataversesPublished != null && !dataversesPublished.isEmpty()) { - JsonObjectBuilder job = Json.createObjectBuilder(); - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (Dataverse dataverse : dataversesPublished) { - jab.add(Json.createObjectBuilder() - .add("id", dataverse.getId()) - .add("alias", dataverse.getAlias()) - ); + if (element == null || element.equals("dataversePublisher")) { + List dataversesPublished = ctxt.dataverses().findByReleaseUserId(userId); + if (dataversesPublished != null && !dataversesPublished.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (Dataverse dataverse : dataversesPublished) { + jab.add(Json.createObjectBuilder() + .add("id", dataverse.getId()) + .add("alias", dataverse.getAlias())); + } + job.add("count", dataversesPublished.size()); + job.add("items", jab); + traces.add("dataversePublisher", job); } - job.add("count", dataversesPublished.size()); - job.add("items", jab); - traces.add("dataversePublisher", job); } - List datasetsCreated = ctxt.datasets().findByCreatorId(userId); - if (datasetsCreated != null && !datasetsCreated.isEmpty()) { - JsonObjectBuilder job = Json.createObjectBuilder(); - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (Dataset dataset : datasetsCreated) { - jab.add(Json.createObjectBuilder() - .add("id", dataset.getId()) - .add("pid", dataset.getGlobalId().asString()) - ); + if (element == null || element.equals("datasetCreator")) { + List datasetsCreated = ctxt.datasets().findByCreatorId(userId); + if (datasetsCreated != null && !datasetsCreated.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (Dataset dataset : datasetsCreated) { + jab.add(Json.createObjectBuilder() + .add("id", dataset.getId()) + .add("pid", dataset.getGlobalId().asString())); + } + job.add("count", datasetsCreated.size()); + job.add("items", jab); + traces.add("datasetCreator", job); } - job.add("count", datasetsCreated.size()); - job.add("items", jab); - traces.add("datasetCreator", job); } - List datasetsPublished = ctxt.datasets().findByReleaseUserId(userId); - if (datasetsPublished != null && !datasetsPublished.isEmpty()) { - JsonObjectBuilder job = Json.createObjectBuilder(); - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (Dataset dataset : datasetsPublished) { - jab.add(Json.createObjectBuilder() - .add("id", dataset.getId()) - .add("pid", dataset.getGlobalId().asString()) - ); + if (element == null || element.equals("datasetPublisher")) { + List datasetsPublished = ctxt.datasets().findByReleaseUserId(userId); + if (datasetsPublished != null && !datasetsPublished.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (Dataset dataset : datasetsPublished) { + jab.add(Json.createObjectBuilder() + .add("id", dataset.getId()) + .add("pid", dataset.getGlobalId().asString())); + } + job.add("count", datasetsPublished.size()); + job.add("items", jab); + traces.add("datasetPublisher", job); } - job.add("count", datasetsPublished.size()); - job.add("items", jab); - traces.add("datasetPublisher", job); } - List dataFilesCreated = ctxt.files().findByCreatorId(userId); - if (dataFilesCreated != null && !dataFilesCreated.isEmpty()) { - JsonObjectBuilder job = Json.createObjectBuilder(); - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (DataFile dataFile : dataFilesCreated) { - jab.add(Json.createObjectBuilder() - .add("id", dataFile.getId()) - .add("filename", dataFile.getCurrentName()) - .add("datasetPid", dataFile.getOwner().getGlobalId().asString()) - ); + if (element == null || element.equals("dataFileCreator")) { + List dataFilesCreated = ctxt.files().findByCreatorId(userId); + if (dataFilesCreated != null && !dataFilesCreated.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (DataFile dataFile : dataFilesCreated) { + jab.add(Json.createObjectBuilder() + .add("id", dataFile.getId()) + .add("filename", dataFile.getCurrentName()) + .add("datasetPid", dataFile.getOwner().getGlobalId().asString())); + } + job.add("count", dataFilesCreated.size()); + job.add("items", jab); + traces.add("dataFileCreator", job); } - job.add("count", dataFilesCreated.size()); - job.add("items", jab); - traces.add("dataFileCreator", job); } - // TODO: Consider removing this because we don't seem to populate releaseuser_id for files. - List dataFilesPublished = ctxt.files().findByReleaseUserId(userId); - if (dataFilesPublished != null && !dataFilesPublished.isEmpty()) { - JsonObjectBuilder job = Json.createObjectBuilder(); - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (DataFile dataFile : dataFilesPublished) { - jab.add(Json.createObjectBuilder() - .add("id", dataFile.getId()) - .add("filename", dataFile.getCurrentName()) - .add("datasetPid", dataFile.getOwner().getGlobalId().asString()) - ); + if (element == null || element.equals("dataFilePublisher")) { + // TODO: Consider removing this because we don't seem to populate releaseuser_id + // for files. + List dataFilesPublished = ctxt.files().findByReleaseUserId(userId); + if (dataFilesPublished != null && !dataFilesPublished.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (DataFile dataFile : dataFilesPublished) { + jab.add(Json.createObjectBuilder() + .add("id", dataFile.getId()) + .add("filename", dataFile.getCurrentName()) + .add("datasetPid", dataFile.getOwner().getGlobalId().asString())); + } + job.add("count", dataFilesPublished.size()); + job.add("items", jab); + traces.add("dataFilePublisher", job); } - job.add("count", dataFilesPublished.size()); - job.add("items", jab); - traces.add("dataFileCreator", job); } - // These are the users who have published a version (or created a draft). - List datasetVersionUsers = ctxt.datasetVersion().getDatasetVersionUsersByAuthenticatedUser(user); - if (datasetVersionUsers != null && !datasetVersionUsers.isEmpty()) { - JsonObjectBuilder job = Json.createObjectBuilder(); - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (DatasetVersionUser datasetVersionUser : datasetVersionUsers) { - jab.add(Json.createObjectBuilder() - .add("id", datasetVersionUser.getId()) - .add("dataset", datasetVersionUser.getDatasetVersion().getDataset().getGlobalId().asString()) - .add("version", datasetVersionUser.getDatasetVersion().getSemanticVersion()) - ); + if (element == null || element.equals("datasetVersionUsers")) { + // These are the users who have published a version (or created a draft). + List datasetVersionUsers = ctxt.datasetVersion().getDatasetVersionUsersByAuthenticatedUser(user); + if (datasetVersionUsers != null && !datasetVersionUsers.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (DatasetVersionUser datasetVersionUser : datasetVersionUsers) { + jab.add(Json.createObjectBuilder() + .add("id", datasetVersionUser.getId()) + .add("dataset", datasetVersionUser.getDatasetVersion().getDataset().getGlobalId().asString()) + .add("version", datasetVersionUser.getDatasetVersion().getSemanticVersion())); + } + job.add("count", datasetVersionUsers.size()); + job.add("items", jab); + traces.add("datasetVersionUsers", job); } - job.add("count", datasetVersionUsers.size()); - job.add("items", jab); - traces.add("datasetVersionUsers", job); } - Set explicitGroups = ctxt.explicitGroups().findDirectlyContainingGroups(user); - if (explicitGroups != null && !explicitGroups.isEmpty()) { - JsonObjectBuilder job = Json.createObjectBuilder(); - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (ExplicitGroup explicitGroup : explicitGroups) { - jab.add(Json.createObjectBuilder() - .add("id", explicitGroup.getId()) - .add("name", explicitGroup.getDisplayName()) - ); + if (element == null || element.equals("explicitGroups")) { + Set explicitGroups = ctxt.explicitGroups().findDirectlyContainingGroups(user); + if (explicitGroups != null && !explicitGroups.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (ExplicitGroup explicitGroup : explicitGroups) { + jab.add(Json.createObjectBuilder() + .add("id", explicitGroup.getId()) + .add("name", explicitGroup.getDisplayName())); + } + job.add("count", explicitGroups.size()); + job.add("items", jab); + traces.add("explicitGroups", job); } - job.add("count", explicitGroups.size()); - job.add("items", jab); - traces.add("explicitGroups", job); } - List guestbookResponses = ctxt.responses().findByAuthenticatedUserId(user); - if (guestbookResponses != null && !guestbookResponses.isEmpty()) { - JsonObjectBuilder job = Json.createObjectBuilder(); - // The feeling is that this is too much detail for now so we only show a count. -// JsonArrayBuilder jab = Json.createArrayBuilder(); -// for (GuestbookResponse guestbookResponse : guestbookResponses) { -// jab.add(Json.createObjectBuilder() -// .add("id", guestbookResponse.getId()) -// .add("downloadType", guestbookResponse.getDownloadtype()) -// .add("filename", guestbookResponse.getDataFile().getCurrentName()) -// .add("date", guestbookResponse.getResponseDate()) -// .add("guestbookName", guestbookResponse.getGuestbook().getName()) -// .add("dataset", guestbookResponse.getDatasetVersion().getDataset().getGlobalId().asString()) -// .add("version", guestbookResponse.getDatasetVersion().getSemanticVersion()) -// ); -// } - job.add("count", guestbookResponses.size()); -// job.add("items", jab); - traces.add("guestbookEntries", job); + if (element == null || element.equals("guestbookEntries")) { + List guestbookResponses = ctxt.responses().findByAuthenticatedUserId(user); + if (guestbookResponses != null && !guestbookResponses.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + // The feeling is that this is too much detail for the call for all elements so + // we only show a count in that case. + if (element != null) { + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (GuestbookResponse guestbookResponse : guestbookResponses) { + jab.add(Json.createObjectBuilder() + .add("id", guestbookResponse.getId()) + .add("downloadType", guestbookResponse.getDownloadtype()) + .add("filename", guestbookResponse.getDataFile().getCurrentName()) + .add("date", guestbookResponse.getResponseDate()) + .add("guestbookName", guestbookResponse.getGuestbook().getName()) + .add("dataset", guestbookResponse.getDatasetVersion().getDataset().getGlobalId().asString()) + .add("version", guestbookResponse.getDatasetVersion().getSemanticVersion())); + } + } + job.add("count", guestbookResponses.size()); + // job.add("items", jab); + traces.add("guestbookEntries", job); + } } - List savedSearchs = ctxt.savedSearches().findByAuthenticatedUser(user); - if (savedSearchs != null && !savedSearchs.isEmpty()) { - JsonObjectBuilder job = Json.createObjectBuilder(); - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (SavedSearch savedSearch : savedSearchs) { - jab.add(Json.createObjectBuilder() - .add("id", savedSearch.getId()) - ); + if (element == null || element.equals("savedSearches")) { + List savedSearchs = ctxt.savedSearches().findByAuthenticatedUser(user); + if (savedSearchs != null && !savedSearchs.isEmpty()) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (SavedSearch savedSearch : savedSearchs) { + jab.add(Json.createObjectBuilder() + .add("id", savedSearch.getId())); + } + job.add("count", savedSearchs.size()); + job.add("items", jab); + traces.add("savedSearches", job); } - job.add("count", savedSearchs.size()); - job.add("items", jab); - traces.add("savedSearches", job); } JsonObjectBuilder result = Json.createObjectBuilder(); result.add("user", Json.createObjectBuilder() .add("identifier", user.getIdentifier()) - .add("name", user.getName()) - ); + .add("name", user.getName())); result.add("traces", traces); return result; } From ea2efb130521c14293aa9c7a5deeb70749848223 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Jun 2021 14:16:01 -0400 Subject: [PATCH 203/354] actually add array to response --- .../iq/dataverse/engine/command/impl/GetUserTracesCommand.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java index 5b6edcb6d8f..a7ee8e1b8f1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java @@ -214,6 +214,7 @@ public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { .add("dataset", guestbookResponse.getDatasetVersion().getDataset().getGlobalId().asString()) .add("version", guestbookResponse.getDatasetVersion().getSemanticVersion())); } + job.add("items", jab); } job.add("count", guestbookResponses.size()); // job.add("items", jab); From 79f2d0de96a90cb633e9e01a0feefc7b6443c6f0 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Jun 2021 14:16:10 -0400 Subject: [PATCH 204/354] add csv option --- .../edu/harvard/iq/dataverse/api/Users.java | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index bc67980927b..7dfda1e5222 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -13,6 +13,9 @@ import edu.harvard.iq.dataverse.engine.command.impl.GetUserTracesCommand; import edu.harvard.iq.dataverse.engine.command.impl.MergeInAccountCommand; import edu.harvard.iq.dataverse.engine.command.impl.RevokeAllRolesCommand; +import edu.harvard.iq.dataverse.metrics.MetricsUtil; +import edu.harvard.iq.dataverse.util.FileUtil; + import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; import java.util.ArrayList; @@ -20,6 +23,7 @@ import java.util.List; import java.util.logging.Logger; import javax.ejb.Stateless; +import javax.json.JsonArray; import javax.json.JsonObjectBuilder; import javax.ws.rs.BadRequestException; import javax.ws.rs.DELETE; @@ -27,7 +31,12 @@ import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Request; import javax.ws.rs.core.Response; +import javax.ws.rs.core.Variant; /** * @@ -227,16 +236,34 @@ public Response getTraces(@PathParam("identifier") String identifier) { } private List elements = Arrays.asList("roleAssignments","dataversCreator", "dataversePublisher","datasetCreator", "datasetPublisher","dataFileCreator","dataFilePublisher","datasetVersionUsers","explicitGroups","guestbookEntries", "savedSearches"); + @GET @Path("{identifier}/traces/{element}") - public Response getTraces(@PathParam("identifier") String identifier, @PathParam("element") String element) { + @Produces("text/csv, application/json") + public Response getTraces(@Context Request req, @PathParam("identifier") String identifier, @PathParam("element") String element) { try { AuthenticatedUser userToQuery = authSvc.getAuthenticatedUser(identifier); if(!elements.contains(element)) { throw new BadRequestException("Not a valid element"); } JsonObjectBuilder jsonObj = execCommand(new GetUserTracesCommand(createDataverseRequest(findUserOrDie()), userToQuery, null)); - return ok(jsonObj); + + List vars = Variant + .mediaTypes(MediaType.valueOf(FileUtil.MIME_TYPE_CSV), MediaType.APPLICATION_JSON_TYPE) + .add() + .build(); + MediaType requestedType = req.selectVariant(vars).getMediaType(); + if ((requestedType != null) && (requestedType.equals(MediaType.APPLICATION_JSON_TYPE))) { + return ok(jsonObj); + + } + JsonArray items=null; + try { + items = jsonObj.build().getJsonObject(element).getJsonArray("items"); + } catch(Exception e) { + return ok(jsonObj); + } + return ok(FileUtil.jsonArrayOfObjectsToCSV(items, items.getJsonObject(0).keySet().toArray(new String[0])), MediaType.valueOf(FileUtil.MIME_TYPE_CSV), "filedownloads.csv"); } catch (WrappedResponse ex) { return ex.getResponse(); } From 1a3300b29f30ebf99049c781d7767bba147da5e8 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Jun 2021 14:45:23 -0400 Subject: [PATCH 205/354] typos --- src/main/java/edu/harvard/iq/dataverse/api/Users.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index 7dfda1e5222..8b3a9d2e298 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -246,7 +246,7 @@ public Response getTraces(@Context Request req, @PathParam("identifier") String if(!elements.contains(element)) { throw new BadRequestException("Not a valid element"); } - JsonObjectBuilder jsonObj = execCommand(new GetUserTracesCommand(createDataverseRequest(findUserOrDie()), userToQuery, null)); + JsonObjectBuilder jsonObj = execCommand(new GetUserTracesCommand(createDataverseRequest(findUserOrDie()), userToQuery, element)); List vars = Variant .mediaTypes(MediaType.valueOf(FileUtil.MIME_TYPE_CSV), MediaType.APPLICATION_JSON_TYPE) @@ -263,7 +263,7 @@ public Response getTraces(@Context Request req, @PathParam("identifier") String } catch(Exception e) { return ok(jsonObj); } - return ok(FileUtil.jsonArrayOfObjectsToCSV(items, items.getJsonObject(0).keySet().toArray(new String[0])), MediaType.valueOf(FileUtil.MIME_TYPE_CSV), "filedownloads.csv"); + return ok(FileUtil.jsonArrayOfObjectsToCSV(items, items.getJsonObject(0).keySet().toArray(new String[0])), MediaType.valueOf(FileUtil.MIME_TYPE_CSV), element + ".csv"); } catch (WrappedResponse ex) { return ex.getResponse(); } From edefdaebaf83f4b377959f7b1b66e01cb1a6ca1f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Jun 2021 15:01:15 -0400 Subject: [PATCH 206/354] handle old/bad db entries --- .../engine/command/impl/GetUserTracesCommand.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java index a7ee8e1b8f1..bfb441f7767 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java @@ -20,6 +20,8 @@ import java.math.BigDecimal; import java.util.List; import java.util.Set; +import java.util.logging.Logger; + import javax.json.Json; import javax.json.JsonArrayBuilder; import javax.json.JsonObjectBuilder; @@ -28,6 +30,8 @@ @RequiredPermissions({}) public class GetUserTracesCommand extends AbstractCommand { + private static final Logger logger = Logger.getLogger(GetUserTracesCommand.class.getCanonicalName()); + private DataverseRequest request; private AuthenticatedUser user; private String element; @@ -205,6 +209,7 @@ public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { if (element != null) { JsonArrayBuilder jab = Json.createArrayBuilder(); for (GuestbookResponse guestbookResponse : guestbookResponses) { + try { jab.add(Json.createObjectBuilder() .add("id", guestbookResponse.getId()) .add("downloadType", guestbookResponse.getDownloadtype()) @@ -213,6 +218,10 @@ public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { .add("guestbookName", guestbookResponse.getGuestbook().getName()) .add("dataset", guestbookResponse.getDatasetVersion().getDataset().getGlobalId().asString()) .add("version", guestbookResponse.getDatasetVersion().getSemanticVersion())); + } catch (NullPointerException npe) { + //Legacy/bad db entries + logger.warning("Dataset id:" + guestbookResponse.getDatasetVersion().getDataset().getId() + " does not have required info."); + } } job.add("items", jab); } From df455694d6bc9d93443a531d3927cfc1b1b436e3 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Jun 2021 15:08:53 -0400 Subject: [PATCH 207/354] don't expect version or dataset to exist --- .../iq/dataverse/engine/command/impl/GetUserTracesCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java index bfb441f7767..009dafc2d51 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java @@ -220,7 +220,7 @@ public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { .add("version", guestbookResponse.getDatasetVersion().getSemanticVersion())); } catch (NullPointerException npe) { //Legacy/bad db entries - logger.warning("Dataset id:" + guestbookResponse.getDatasetVersion().getDataset().getId() + " does not have required info."); + logger.warning("Guestbook id:" + guestbookResponse.getId() + " does not have required info."); } } job.add("items", jab); From 80c442dd1a85fe423a79dd1dcb3c6f1dcb9257d6 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Jun 2021 15:31:51 -0400 Subject: [PATCH 208/354] bug fixes --- .../edu/harvard/iq/dataverse/api/Users.java | 2 +- .../command/impl/GetUserTracesCommand.java | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index 8b3a9d2e298..9642551f921 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -259,7 +259,7 @@ public Response getTraces(@Context Request req, @PathParam("identifier") String } JsonArray items=null; try { - items = jsonObj.build().getJsonObject(element).getJsonArray("items"); + items = jsonObj.build().getJsonObject("traces").getJsonObject(element).getJsonArray("items"); } catch(Exception e) { return ok(jsonObj); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java index 009dafc2d51..9259b2ddf6b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java @@ -210,14 +210,17 @@ public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { JsonArrayBuilder jab = Json.createArrayBuilder(); for (GuestbookResponse guestbookResponse : guestbookResponses) { try { - jab.add(Json.createObjectBuilder() - .add("id", guestbookResponse.getId()) - .add("downloadType", guestbookResponse.getDownloadtype()) - .add("filename", guestbookResponse.getDataFile().getCurrentName()) - .add("date", guestbookResponse.getResponseDate()) - .add("guestbookName", guestbookResponse.getGuestbook().getName()) - .add("dataset", guestbookResponse.getDatasetVersion().getDataset().getGlobalId().asString()) - .add("version", guestbookResponse.getDatasetVersion().getSemanticVersion())); + JsonObjectBuilder gbe = Json.createObjectBuilder() + .add("id", guestbookResponse.getId()) + .add("downloadType", guestbookResponse.getDownloadtype()) + .add("filename", guestbookResponse.getDataFile().getCurrentName()) + .add("date", guestbookResponse.getResponseDate()) + .add("guestbookName", guestbookResponse.getGuestbook().getName()); + if (guestbookResponse.getDatasetVersion().getDataset().getGlobalId() != null) { + gbe.add("dataset", guestbookResponse.getDatasetVersion().getDataset().getGlobalId().asString()); + } + gbe.add("version", guestbookResponse.getDatasetVersion().getSemanticVersion()); + jab.add(gbe); } catch (NullPointerException npe) { //Legacy/bad db entries logger.warning("Guestbook id:" + guestbookResponse.getId() + " does not have required info."); From e914984b770d5923ddbaf428fa64d45460ce0947 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Jun 2021 16:09:19 -0400 Subject: [PATCH 209/354] handle no datasetversion --- .../engine/command/impl/GetUserTracesCommand.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java index 9259b2ddf6b..f3324ba6f2e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java @@ -216,10 +216,12 @@ public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { .add("filename", guestbookResponse.getDataFile().getCurrentName()) .add("date", guestbookResponse.getResponseDate()) .add("guestbookName", guestbookResponse.getGuestbook().getName()); - if (guestbookResponse.getDatasetVersion().getDataset().getGlobalId() != null) { - gbe.add("dataset", guestbookResponse.getDatasetVersion().getDataset().getGlobalId().asString()); + if(guestbookResponse.getDataset().getGlobalId()!=null) { + gbe.add("dataset", guestbookResponse.getDataset().getGlobalId().asString()); + } + if (guestbookResponse.getDatasetVersion() != null) { + gbe.add("version", guestbookResponse.getDatasetVersion().getSemanticVersion()); } - gbe.add("version", guestbookResponse.getDatasetVersion().getSemanticVersion()); jab.add(gbe); } catch (NullPointerException npe) { //Legacy/bad db entries From 0b3fa6d88340196030724c9cc6d058f6ea9122a1 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 18 Jun 2021 13:04:32 -0400 Subject: [PATCH 210/354] typo --- src/main/java/edu/harvard/iq/dataverse/api/Users.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index 9642551f921..b1177531874 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -235,7 +235,7 @@ public Response getTraces(@PathParam("identifier") String identifier) { } } - private List elements = Arrays.asList("roleAssignments","dataversCreator", "dataversePublisher","datasetCreator", "datasetPublisher","dataFileCreator","dataFilePublisher","datasetVersionUsers","explicitGroups","guestbookEntries", "savedSearches"); + private List elements = Arrays.asList("roleAssignments","dataverseCreator", "dataversePublisher","datasetCreator", "datasetPublisher","dataFileCreator","dataFilePublisher","datasetVersionUsers","explicitGroups","guestbookEntries", "savedSearches"); @GET @Path("{identifier}/traces/{element}") From c2c5968c79d4690b45ed0857cdd49352bdaef3a9 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 18 Jun 2021 14:36:33 -0400 Subject: [PATCH 211/354] fix style issue --- src/main/webapp/resources/css/structure.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 0ffc336387e..63bb19f5589 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -655,7 +655,7 @@ div.edit-field div.ui-message {margin:6px 0;} /* TRUNCATION */ div.more-block {text-align:center; padding-top:250px; width:100%; position:absolute; bottom:0; background:linear-gradient(180deg,hsla(0,0%,100%,0),#fff 80%);} button.desc-more-link {margin:0; padding:0;} -div.less-block {text-align:center;}; +div.less-block {text-align:center;} /* Primefaces vs Bootstrap selectOneMenu fix */ div.ui-selectonemenu.form-control {padding:0px;} From ea2545c9028c610600e48686b5023c8f32966e3c Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 21 Jun 2021 10:23:07 -0400 Subject: [PATCH 212/354] update CURL command for error response --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index d990d18aec5..695b2b47974 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -598,13 +598,13 @@ In order to retrieve a file containing a list of Guestbook Responses in csv form export ID=root export GUESTBOOK_ID=1 - curl -O -J -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/guestbookResponses?guestbookId=$GUESTBOOK_ID + curl -O -J -f -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/guestbookResponses?guestbookId=$GUESTBOOK_ID The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -O -J -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - https://demo.dataverse.org/api/dataverses/root/guestbookResponses?guestbookId=1 + curl -O -J -f -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/guestbookResponses?guestbookId=1 Datasets -------- From d47f7de3a9121da59e8f43f62c282cf0640b745a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 21 Jun 2021 14:52:24 -0400 Subject: [PATCH 213/354] missing import --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 547171d2ad1..52636de5976 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -139,6 +139,7 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; From 2d22e262168d90ddc7acc2d62496754693e72dfc Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 21 Jun 2021 15:03:46 -0400 Subject: [PATCH 214/354] add missing citation updates --- .../java/edu/harvard/iq/dataverse/DataCitation.java | 11 +++++++++-- src/main/java/edu/harvard/iq/dataverse/Dataset.java | 6 +++++- .../java/edu/harvard/iq/dataverse/DatasetVersion.java | 6 +++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java index 4dcfbc035a5..bf89598ccfb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java @@ -185,10 +185,17 @@ public String toString() { } public String toString(boolean html) { - // first add comma separated parts + return toString(html, false); + } + public String toString(boolean html, boolean anonymized) { + // first add comma separated parts String separator = ", "; List citationList = new ArrayList<>(); - citationList.add(formatString(getAuthorsString(), html)); + if(anonymized) { + citationList.add("Author name(s) withheld"); + } else { + citationList.add(formatString(getAuthorsString(), html)); + } citationList.add(year); if ((fileTitle != null) && isDirect()) { citationList.add(formatString(fileTitle, html, "\"")); diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index cd40e76a304..56694e48f88 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -649,7 +649,11 @@ public String getCitation(DatasetVersion version) { } public String getCitation(boolean isOnlineVersion, DatasetVersion version) { - return version.getCitation(isOnlineVersion); + return getCitation(isOnlineVersion, version, false); + } + + public String getCitation(boolean isOnlineVersion, DatasetVersion version, boolean anonymized) { + return version.getCitation(isOnlineVersion, anonymized); } public String getPublicationDateFormattedYYYYMMDD() { diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 1f0467577a2..5083b778d2e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1340,7 +1340,11 @@ public String getCitation() { } public String getCitation(boolean html) { - return new DataCitation(this).toString(html); + return getCitation(html, false); + } + + public String getCitation(boolean html, boolean anonymized) { + return new DataCitation(this).toString(html, anonymized); } public Date getCitationDate() { From 66dd55d6d247e2a24e890a3007b8e6206f7e1fd9 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 21 Jun 2021 15:25:33 -0400 Subject: [PATCH 215/354] Add 'Author name(s) withheld' to bundle --- src/main/java/edu/harvard/iq/dataverse/DataCitation.java | 2 +- src/main/java/propertyFiles/Bundle.properties | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java index bf89598ccfb..6bf47c515fe 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java @@ -192,7 +192,7 @@ public String toString(boolean html, boolean anonymized) { String separator = ", "; List citationList = new ArrayList<>(); if(anonymized) { - citationList.add("Author name(s) withheld"); + citationList.add(BundleUtil.getStringFromBundle("file.anonymized.authorsWithheld")); } else { citationList.add(formatString(getAuthorsString(), html)); } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 7838e543144..db97e972684 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1132,6 +1132,7 @@ dataset.message.manageTemplates.label=Manage Dataset Templates dataset.message.manageTemplates.message=Create a template prefilled with metadata fields standard values, such as Author Affiliation, or add instructions in the metadata fields to give depositors more information on what metadata is expected. # metadataFragment.xhtml +dataset.anonymized.withheld=withheld # template.xhtml dataset.template.name.tip=The name of the dataset template. @@ -1939,6 +1940,7 @@ file.versionDifferences.fileRestricted=Restricted file.versionDifferences.fileUnrestricted=Unrestricted file.versionDifferences.fileGroupTitle=File +file.anonymized.authorsWithheld=Author name(s) withheld # File Ingest ingest.csv.invalidHeader=Invalid header row. One of the cells is empty. ingest.csv.lineMismatch=Mismatch between line counts in first and final passes!, {0} found on first pass, but {1} found on second. From 863449bf2d8d8857adb1888a475f40ba830491cf Mon Sep 17 00:00:00 2001 From: Robert Verkerk Date: Tue, 22 Jun 2021 08:35:43 +0200 Subject: [PATCH 216/354] Fix mising ";" coding error. --- src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 8a282f2dfa6..7dd2248eafe 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -1047,7 +1047,7 @@ public boolean isIndependentHandleService() { public string getHandleAuthHandle() { string HandleAuthHandle = settingsService.getValueForKey(SettingsServiceBean.Key.HandleAuthHandle, null); - return HandleAuthHandle + return HandleAuthHandle; } public String getMDCLogPath() { From 593b3d655b6d9a9919d7e327b8320ea62ff83c9d Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 23 Jun 2021 09:28:34 -0400 Subject: [PATCH 217/354] #7767 sort responses for gb download --- .../edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java index 701ee6aa7f2..f2d290215da 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java @@ -149,7 +149,7 @@ public List getGuestbookResults(Long dataverseId, Long guestbookId ){ queryString+= (" and r.guestbook_id = " + guestbookId.toString()); } - queryString += ";"; + queryString += " ORDER by r.id DESC;"; logger.fine("stream responses query: " + queryString); return em.createNativeQuery(queryString).getResultList(); From e7f26de33ec75585e7a495029a2808e62b270ae4 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 23 Jun 2021 09:40:00 -0400 Subject: [PATCH 218/354] adding clarification to docs --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 695b2b47974..85fd0232e1d 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -587,7 +587,7 @@ You should expect a 200 ("OK") response and JSON output. Retrieve Guestbook Responses for a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to retrieve a file containing a list of Guestbook Responses in csv format for Dataverse collection, you must know either its "alias" (which the GUI calls an "identifier") or its database ID. If the Dataverse collection has more than one guestbook you may provide the id of a single guestbook as an optional parameter. If the guestbook id is provided then only those responses from that guestbook will be included in the file. +In order to retrieve a file containing a list of Guestbook Responses in csv format for Dataverse collection, you must know either its "alias" (which the GUI calls an "identifier") or its database ID. If the Dataverse collection has more than one guestbook you may provide the id of a single guestbook as an optional parameter. If no guestbook id is provided the results returned will be the same as pressing the "Download All Responses" button on the Manage Dataset Guestbook page. If the guestbook id is provided then only those responses from that guestbook will be included in the file. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. From 8020d0b96b20c20dd61161722a84a8dcb789c6b9 Mon Sep 17 00:00:00 2001 From: "don.sizemore" Date: Wed, 23 Jun 2021 11:31:56 -0400 Subject: [PATCH 219/354] #7965 switch docker-aio to Rocky, correct run-test-suite.sh path --- conf/docker-aio/c8.dockerfile | 2 +- conf/docker-aio/run-test-suite.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/docker-aio/c8.dockerfile b/conf/docker-aio/c8.dockerfile index 1ccc37e048c..1609cb904aa 100644 --- a/conf/docker-aio/c8.dockerfile +++ b/conf/docker-aio/c8.dockerfile @@ -1,4 +1,4 @@ -FROM centos:8 +FROM rockylinux/rockylinux:latest # OS dependencies # IQSS now recommends Postgres 13. RUN dnf -qy module disable postgresql diff --git a/conf/docker-aio/run-test-suite.sh b/conf/docker-aio/run-test-suite.sh index 19d05154a17..39809a7a50e 100755 --- a/conf/docker-aio/run-test-suite.sh +++ b/conf/docker-aio/run-test-suite.sh @@ -6,7 +6,7 @@ if [ -z "$dvurl" ]; then dvurl="http://localhost:8084" fi -integrationtests=$(<../../tests/integration-tests.txt) +integrationtests=$( Date: Wed, 23 Jun 2021 13:36:43 -0400 Subject: [PATCH 220/354] move metadataOnOrig out of citation block which makes it optional --- scripts/api/data/metadatablocks/citation.tsv | 1 - scripts/api/data/metadatablocks/migration.tsv | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 scripts/api/data/metadatablocks/migration.tsv diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index 1996302ccb0..375a8c67cec 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -79,7 +79,6 @@ originOfSources Origin of Sources For historical materials, information about the origin of the sources and the rules followed in establishing the sources should be specified. textbox 75 FALSE FALSE FALSE FALSE FALSE FALSE citation characteristicOfSources Characteristic of Sources Noted Assessment of characteristics and source material. textbox 76 FALSE FALSE FALSE FALSE FALSE FALSE citation accessToSources Documentation and Access to Sources Level of documentation of the original sources. textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation - metadataOnOrig Metadata on the original source of migrated datasets. textbox 78 FALSE FALSE FALSE FALSE FALSE FALSE citation #controlledVocabulary DatasetField Value identifier displayOrder subject Agricultural Sciences D01 0 subject Arts and Humanities D0 1 diff --git a/scripts/api/data/metadatablocks/migration.tsv b/scripts/api/data/metadatablocks/migration.tsv new file mode 100644 index 00000000000..1f9a7a327f5 --- /dev/null +++ b/scripts/api/data/metadatablocks/migration.tsv @@ -0,0 +1,5 @@ +#metadataBlock name dataverseAlias displayName blockURI + migration Migrated Metadata https://dataverse.org/schema/migration/ +#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id termURI + metadataOnOrig Metadata on the original source of migrated datasets. textbox 1 FALSE FALSE FALSE FALSE FALSE FALSE migration https://dataverse.org/schema/core#metadataOnOrig +#controlledVocabulary DatasetField Value identifier displayOrder \ No newline at end of file From c6c7d408a481c5a5df5b483e620517400304f51b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 23 Jun 2021 13:36:43 -0400 Subject: [PATCH 221/354] move metadataOnOrig out of citation block which makes it optional Conflicts: scripts/api/data/metadatablocks/citation.tsv --- scripts/api/data/metadatablocks/citation.tsv | 1 - scripts/api/data/metadatablocks/migration.tsv | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 scripts/api/data/metadatablocks/migration.tsv diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index b908c246dbe..375a8c67cec 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -79,7 +79,6 @@ originOfSources Origin of Sources For historical materials, information about the origin of the sources and the rules followed in establishing the sources should be specified. textbox 75 FALSE FALSE FALSE FALSE FALSE FALSE citation characteristicOfSources Characteristic of Sources Noted Assessment of characteristics and source material. textbox 76 FALSE FALSE FALSE FALSE FALSE FALSE citation accessToSources Documentation and Access to Sources Level of documentation of the original sources. textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation - metadataOnOrig External Metadata Metadata that isn't supported in Dataverse, as provided by external applications textbox 78 FALSE FALSE FALSE FALSE FALSE FALSE citation #controlledVocabulary DatasetField Value identifier displayOrder subject Agricultural Sciences D01 0 subject Arts and Humanities D0 1 diff --git a/scripts/api/data/metadatablocks/migration.tsv b/scripts/api/data/metadatablocks/migration.tsv new file mode 100644 index 00000000000..1f9a7a327f5 --- /dev/null +++ b/scripts/api/data/metadatablocks/migration.tsv @@ -0,0 +1,5 @@ +#metadataBlock name dataverseAlias displayName blockURI + migration Migrated Metadata https://dataverse.org/schema/migration/ +#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id termURI + metadataOnOrig Metadata on the original source of migrated datasets. textbox 1 FALSE FALSE FALSE FALSE FALSE FALSE migration https://dataverse.org/schema/core#metadataOnOrig +#controlledVocabulary DatasetField Value identifier displayOrder \ No newline at end of file From d91edb2eefd3a8cfda4ede31607e1026312093f8 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 24 Jun 2021 11:26:19 -0400 Subject: [PATCH 222/354] #7858 add setup warnings --- .../edu/harvard/iq/dataverse/DatasetPage.java | 8 +++-- .../iq/dataverse/EditDatafilesPage.java | 34 ++++++++++++++++++- .../impl/RequestRsyncScriptCommand.java | 2 +- src/main/java/propertyFiles/Bundle.properties | 4 ++- 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index d441faf7ec6..b3d93fce0f8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1899,7 +1899,8 @@ private String init(boolean initFull) { this.guestbookResponse = guestbookResponseService.initGuestbookResponseForFragment(workingVersion, null, session); logger.fine("Checking if rsync support is enabled."); if (DataCaptureModuleUtil.rsyncSupportEnabled(settingsWrapper.getValueForKey(SettingsServiceBean.Key.UploadMethods)) - && dataset.getFiles().isEmpty()) { //only check for rsync if no files exist + && dataset.getFiles().isEmpty() && this.canUpdateDataset() ) { //only check for rsync if no files exist + //and user can update dataset try { ScriptRequestResponse scriptRequestResponse = commandEngine.submit(new RequestRsyncScriptCommand(dvRequestService.getDataverseRequest(), dataset)); logger.fine("script: " + scriptRequestResponse.getScript()); @@ -1913,8 +1914,11 @@ private String init(boolean initFull) { } } catch (RuntimeException ex) { logger.warning("Problem getting rsync script: " + ex.getLocalizedMessage()); + FacesContext.getCurrentInstance().addMessage("Dataset Page Runtime", new FacesMessage("message") + ); } catch (CommandException cex) { - logger.warning("Problem getting rsync script (Command Exception): " + cex.getLocalizedMessage()); + logger.warning("Problem getting rsync script (Command Exception): -catching in page.." + cex.getLocalizedMessage()); + FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_ERROR, "Problem getting rsync script:", cex.getLocalizedMessage())); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 7d190618330..e11ab0d30c8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -473,6 +473,7 @@ public String initCreateMode(String modeToken, DatasetVersion version, MutableBo public String init() { + System.out.print("edit data files page init..."); // default mode should be EDIT if (mode == null) { mode = FileEditMode.EDIT; @@ -598,9 +599,12 @@ public String init() { } if (mode == FileEditMode.UPLOAD) { + System.out.print("file edit mode " + mode); if (settingsWrapper.getUploadMethodsCount() == 1){ + System.out.print("settingsWrapper.getUploadMethodsCount() == 1 " ); JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.uploadFiles.label"), BundleUtil.getStringFromBundle("dataset.message.uploadFilesSingle.message", Arrays.asList(systemConfig.getGuidesBaseUrl(), systemConfig.getGuidesVersion()))); } else if (settingsWrapper.getUploadMethodsCount() > 1) { + System.out.print("settingsWrapper.getUploadMethodsCount() > 1 " ); JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.uploadFiles.label"), BundleUtil.getStringFromBundle("dataset.message.uploadFilesMultiple.message", Arrays.asList(systemConfig.getGuidesBaseUrl(), systemConfig.getGuidesVersion()))); } @@ -1292,6 +1296,7 @@ public boolean showFileUploadFragment(){ public boolean showFileUploadComponent(){ if (mode == FileEditMode.UPLOAD || mode == FileEditMode.CREATE) { + System.out.print("showfileuploadfragement == true"); return true; } @@ -1512,6 +1517,7 @@ public void setHasRsyncScript(Boolean hasRsyncScript) { } private void setUpRsync() { + System.out.print("setUpRsync called..."); logger.fine("setUpRsync called..."); if (DataCaptureModuleUtil.rsyncSupportEnabled(settingsWrapper.getValueForKey(SettingsServiceBean.Key.UploadMethods)) && dataset.getFiles().isEmpty()) { //only check for rsync if no files exist @@ -1527,10 +1533,24 @@ private void setUpRsync() { } } catch (EJBException ex) { logger.warning("Problem getting rsync script (EJBException): " + EjbUtil.ejbExceptionToString(ex)); + FacesContext.getCurrentInstance().addMessage(uploadComponentId, + new FacesMessage(FacesMessage.SEVERITY_ERROR, + "Problem getting rsync script (EJBException): " + EjbUtil.ejbExceptionToString(ex), + "Problem getting rsync script (EJBException):")); } catch (RuntimeException ex) { + System.out.print("RuntimeException..."); logger.warning("Problem getting rsync script (RuntimeException): " + ex.getLocalizedMessage()); + FacesContext.getCurrentInstance().addMessage(uploadComponentId, + new FacesMessage(FacesMessage.SEVERITY_ERROR, + "Problem getting rsync script (RuntimeException): " +ex.getMessage(), + "Problem getting rsync script (RuntimeException):")); } catch (CommandException cex) { + System.out.print("commandException..."); logger.warning("Problem getting rsync script (Command Exception): " + cex.getLocalizedMessage()); + FacesContext.getCurrentInstance().addMessage(uploadComponentId, + new FacesMessage(FacesMessage.SEVERITY_ERROR, + "Problem getting rsync script (Command Exception): " +cex.getMessage(), + "Problem getting rsync script (Command Exception):")); } } } @@ -1989,6 +2009,7 @@ public void handleFileUpload(FileUploadEvent event) throws IOException { * @param event */ public void handleExternalUpload() { + System.out.print("in handle external upload..."); Map paramMap = FacesContext.getCurrentInstance().getExternalContext().getRequestParameterMap(); this.uploadComponentId = paramMap.get("uploadComponentId"); @@ -2967,8 +2988,19 @@ public boolean rsyncUploadSupported() { // ToDo - rsync was written before multiple store support and currently is hardcoded to use the "s3" store. // When those restrictions are lifted/rsync can be configured per store, the test in the // Dataset Util method should be updated + System.out.print(" IN rsyncUploadSupported() "); + System.out.print("settingsWrapper.isRsyncUpload() " + settingsWrapper.isRsyncUpload()); + System.out.print("DatasetUtil.isAppropriateStorageDriver(dataset) " + DatasetUtil.isAppropriateStorageDriver(dataset)); + if(settingsWrapper.isRsyncUpload() && !DatasetUtil.isAppropriateStorageDriver(dataset) ){ + System.out.print("setuprsync failed message adding..."); + //dataset.file.upload.setUp.rsync.failed.detail + FacesMessage message = new FacesMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("dataset.file.upload.setUp.rsync.failed"), BundleUtil.getStringFromBundle("dataset.file.upload.setUp.rsync.failed.detail")); + FacesContext.getCurrentInstance().addMessage(null, message); + } + - return settingsWrapper.isRsyncUpload() && DatasetUtil.isAppropriateStorageDriver(dataset); + + return settingsWrapper.isRsyncUpload() && DatasetUtil.isAppropriateStorageDriver(dataset); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java index 2a6d7216aa5..a29e7fdd59c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java @@ -51,7 +51,7 @@ public ScriptRequestResponse execute(CommandContext ctxt) throws CommandExceptio } String dcmBaseUrl = ctxt.settings().getValueForKey(DataCaptureModuleUrl); if (dcmBaseUrl == null) { - throw new RuntimeException(DataCaptureModuleUrl + " is null!"); + throw new CommandException("DataCaptureModuleUrl is null!", this); } User user = request.getUser(); if (!(user instanceof AuthenticatedUser)) { diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 77248d76ea8..903e6905bc2 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2189,7 +2189,9 @@ dataset.file.exist.test={0, choice, 1#File |2#Files |} {1} {0, choice, 1#has |2# dataset.files.duplicate=Files {0} have the same content as {1} that have already been uploaded. dataset.file.duplicate=File {0} has the same content as {1} that has already been uploaded. dataset.file.inline.message= This file has the same content as {0}. -dataset.file.upload=Succesful {0} is uploaded. +dataset.file.upload=Successful {0} is uploaded. +dataset.file.upload.setUp.rsync.failed=Rsync upload setup failed! +dataset.file.upload.setUp.rsync.failed.detail=Unable to find appropriate storage driver. dataset.file.uploadFailure=upload failure dataset.file.uploadFailure.detailmsg=the file {0} failed to upload! dataset.file.uploadWarning=upload warning From d0d78bba86d37be74b7ab663dbd7e08c23726362 Mon Sep 17 00:00:00 2001 From: pkiraly Date: Thu, 24 Jun 2021 19:53:23 +0200 Subject: [PATCH 223/354] #7431 remove XML prolog from the individual records of OAI-PMH ListRecords response --- .../dataverse/harvest/server/web/servlet/OAIServlet.java | 4 ---- .../iq/dataverse/harvest/server/xoai/XlistRecords.java | 1 - .../iq/dataverse/harvest/server/xoai/Xrecord.java | 9 ++++++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java index 8e436eae8c1..d8619c42dfa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java @@ -46,13 +46,9 @@ import java.io.IOException; import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; import java.util.Date; import java.util.HashMap; import java.util.logging.Logger; -import java.util.zip.DeflaterOutputStream; -import java.util.zip.GZIPOutputStream; import javax.ejb.EJB; import javax.mail.internet.InternetAddress; import javax.servlet.ServletConfig; diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecords.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecords.java index e2366119f54..15bd005cacf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecords.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecords.java @@ -1,4 +1,3 @@ - package edu.harvard.iq.dataverse.harvest.server.xoai; import com.lyncode.xml.exceptions.XmlWriteException; diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xrecord.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xrecord.java index 538f0fe7219..7e115c78f06 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xrecord.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xrecord.java @@ -1,4 +1,3 @@ - package edu.harvard.iq.dataverse.harvest.server.xoai; import com.lyncode.xoai.model.oaipmh.Header; @@ -17,6 +16,7 @@ import java.io.OutputStream; import java.net.InetAddress; import java.net.UnknownHostException; +import org.apache.poi.util.ReplacingInputStream; /** * @@ -85,7 +85,11 @@ public void writeToStream(OutputStream outputStream) throws IOException { if (dataset != null && formatName != null) { InputStream inputStream = null; try { - inputStream = ExportService.getInstance().getExport(dataset, formatName); + inputStream = new ReplacingInputStream( + ExportService.getInstance().getExport(dataset, formatName), + "", + "" + ); } catch (ExportException ex) { inputStream = null; } @@ -101,7 +105,6 @@ public void writeToStream(OutputStream outputStream) throws IOException { } } outputStream.flush(); - } private String itemHeaderToString(Header header) { From bccaa93e33ace1a184ce4e07be8ba7d2a732235c Mon Sep 17 00:00:00 2001 From: Kaitlin Newson Date: Thu, 24 Jun 2021 14:04:07 -0400 Subject: [PATCH 224/354] reformat SWORD API code --- doc/sphinx-guides/source/api/sword.rst | 49 +++++++++++++++++++------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index 23d6588bd86..e4ffd5a0b24 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -94,14 +94,18 @@ Retrieve SWORD service document The service document enumerates the Dataverse collections (also "collections" from a SWORD perspective) the user can deposit data into. The "collectionPolicy" element for each Dataverse collections contains the Terms of Use. Any user with an API token can use this API endpoint. Institution-wide Shibboleth groups are not respected because membership in such a group can only be set via a browser. -``curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/service-document`` +.. code-block:: bash + + curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/service-document Create a dataset with an Atom entry ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To create a dataset, you must have the "Dataset Creator" role (the ``AddDataset`` permission) on a Dataverse collection. Practically speaking, you should first retrieve the service document to list the Dataverse collections into which you are authorized to deposit data. -``curl -u $API_TOKEN: --data-binary "@path/to/atom-entry-study.xml" -H "Content-Type: application/atom+xml" https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/collection/dataverse/$DATAVERSE_ALIAS`` +.. code-block:: bash + + curl -u $API_TOKEN: --data-binary "@path/to/atom-entry-study.xml" -H "Content-Type: application/atom+xml" https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/collection/dataverse/$DATAVERSE_ALIAS Example Atom entry (XML) @@ -146,14 +150,18 @@ List datasets in a Dataverse Collection You must have permission to add datasets in a Dataverse collection (the Dataverse collection should appear in the service document) to list the datasets inside. Institution-wide Shibboleth groups are not respected because membership in such a group can only be set via a browser. -``curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/collection/dataverse/$DATAVERSE_ALIAS`` +.. code-block:: bash + + curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/collection/dataverse/$DATAVERSE_ALIAS Add files to a dataset with a zip file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You must have ``EditDataset`` permission (Contributor role or above such as Curator or Admin) on the dataset to add files. -``curl -u $API_TOKEN: --data-binary @path/to/example.zip -H "Content-Disposition: filename=example.zip" -H "Content-Type: application/zip" -H "Packaging: http://purl.org/net/sword/package/SimpleZip" https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit-media/study/doi:TEST/12345`` +.. code-block:: bash + + curl -u $API_TOKEN: --data-binary @path/to/example.zip -H "Content-Disposition: filename=example.zip" -H "Content-Type: application/zip" -H "Packaging: http://purl.org/net/sword/package/SimpleZip" https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit-media/study/doi:TEST/12345 Display a dataset atom entry ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -162,56 +170,72 @@ You must have ``ViewUnpublishedDataset`` permission (Contributor role or above s Contains data citation (bibliographicCitation), alternate URI (persistent URI of study), edit URI, edit media URI, statement URI. -``curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/study/doi:TEST/12345`` +.. code-block:: bash + + curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/study/doi:TEST/12345 Display a dataset statement ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Contains title, author, feed of file entries, latestVersionState, locked boolean, updated timestamp. You must have ``ViewUnpublishedDataset`` permission (Contributor role or above such as Curator or Admin) on the dataset to display the statement. -``curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/statement/study/doi:TEST/12345`` +.. code-block:: bash + + curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/statement/study/doi:TEST/12345 Delete a file by database id ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You must have ``EditDataset`` permission (Contributor role or above such as Curator or Admin) on the dataset to delete files. -``curl -u $API_TOKEN: -X DELETE https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit-media/file/123`` +.. code-block:: bash + + curl -u $API_TOKEN: -X DELETE https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit-media/file/123 Replacing metadata for a dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Please note that **ALL** metadata (title, author, etc.) will be replaced, including fields that can not be expressed with "dcterms" fields. You must have ``EditDataset`` permission (Contributor role or above such as Curator or Admin) on the dataset to replace metadata. -``curl -u $API_TOKEN: --upload-file "path/to/atom-entry-study2.xml" -H "Content-Type: application/atom+xml" https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/study/doi:TEST/12345`` +.. code-block:: bash + + curl -u $API_TOKEN: --upload-file "path/to/atom-entry-study2.xml" -H "Content-Type: application/atom+xml" https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/study/doi:TEST/12345`` Delete a dataset ~~~~~~~~~~~~~~~~ You must have the ``DeleteDatasetDraft`` permission (Contributor role or above such as Curator or Admin) on the dataset to delete it. Please note that if the dataset has never been published you will be able to delete it completely but if the dataset has already been published you will only be able to delete post-publication drafts, never a published version. -``curl -u $API_TOKEN: -i -X DELETE https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/study/doi:TEST/12345`` +.. code-block:: bash + + curl -u $API_TOKEN: -i -X DELETE https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/study/doi:TEST/12345 Determine if a Dataverse Collection has been published ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This API endpoint is the same as the "list datasets in a Dataverse collection" endpoint documented above and the same permissions apply but it is documented here separately to point out that you can look for a boolean called ``dataverseHasBeenReleased`` to know if a Dataverse collection has been released, which is required for publishing a dataset. -``curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/collection/dataverse/$DATAVERSE_ALIAS`` +.. code-block:: bash + + curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/collection/dataverse/$DATAVERSE_ALIAS Publish a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``cat /dev/null`` and ``--data-binary @-`` arguments are used to send zero-length content to the API, which is required by the upstream library to process the ``In-Progress: false`` header. You must have the ``PublishDataverse`` permission (Admin role) on the Dataverse collection to publish it. -``cat /dev/null | curl -u $API_TOKEN: -X POST -H "In-Progress: false" --data-binary @- https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/dataverse/$DATAVERSE_ALIAS`` +.. code-block:: bash + + cat /dev/null | curl -u $API_TOKEN: -X POST -H "In-Progress: false" --data-binary @- https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/dataverse/$DATAVERSE_ALIAS Publish a dataset ~~~~~~~~~~~~~~~~~ The ``cat /dev/null`` and ``--data-binary @-`` arguments are used to send zero-length content to the API, which is required by the upstream library to process the ``In-Progress: false`` header. You must have the ``PublishDataset`` permission (Curator or Admin role) on the dataset to publish it. -``cat /dev/null | curl -u $API_TOKEN: -X POST -H "In-Progress: false" --data-binary @- https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/study/doi:TEST/12345`` +.. code-block:: bash + + cat /dev/null | curl -u $API_TOKEN: -X POST -H "In-Progress: false" --data-binary @- https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/study/doi:TEST/12345 .. _known-issues: @@ -249,4 +273,3 @@ Client libraries - R: https://github.com/IQSS/dataverse-client-r - Ruby: https://github.com/swordapp/sword2ruby - PHP: https://github.com/swordapp/swordappv2-php-library - From 2dc0c825f03e2127e061a29f4b2fb4cfa17437bd Mon Sep 17 00:00:00 2001 From: Kaitlin Newson Date: Thu, 24 Jun 2021 14:09:17 -0400 Subject: [PATCH 225/354] use code block formatting in sword api docs --- doc/sphinx-guides/source/api/sword.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index e4ffd5a0b24..778a6a4c8a7 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -34,11 +34,15 @@ HTTP Basic Authentication commonly makes use of both a username and a password b Clients such as ``curl`` expect both a username and a password separated by a colon. With ``curl``, the way to indicate that the password should be blank or empty is to include the colon at the end of the username (the API token) like this: -``curl -u 54b143b5-d001-4254-afc0-a1c0f6a5b5a7:`` +.. code-block:: bash + + curl -u 54b143b5-d001-4254-afc0-a1c0f6a5b5a7: All the curl examples below take this form but instead of showing an API token like above, a Bash environment variable called ``$API_TOKEN`` is shown instead like this: -``curl -u $API_TOKEN:`` +.. code-block:: bash + + curl -u $API_TOKEN: .. _RFC 7617: https://tools.ietf.org/html/rfc7617 From d31219d23d7763e295bc7ede66772809b365cde0 Mon Sep 17 00:00:00 2001 From: Kaitlin Newson Date: Thu, 24 Jun 2021 14:15:31 -0400 Subject: [PATCH 226/354] xml syntax highlighting, formatting for in-line XML in sword api docs --- doc/sphinx-guides/source/api/sword.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index 778a6a4c8a7..d853994f073 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -57,9 +57,9 @@ Differences in Dataverse Software 4 from DVN 3.x lead to a few minor backward in - Newly required fields when creating/editing datasets for compliance with the `Joint Declaration for Data Citation principles `_. - - dcterms:creator (maps to authorName) + - ``dcterms:creator`` (maps to authorName) - - dcterms:description + - ``dcterms:description`` - Deaccessioning is no longer supported. An alternative will be developed at https://github.com/IQSS/dataverse/issues/778 @@ -76,11 +76,11 @@ New features as of v1.1 - Datasets versions will only be increased to the next minor version (i.e. 1.1) rather than a major version (2.0) if possible. This depends on the nature of the change. Adding or removing a file, for example, requires a major version bump. -- "Author Affiliation" can now be populated with an XML attribute. For example: Stumptown, Jane +- "Author Affiliation" can now be populated with an XML attribute. For example: ``Stumptown, Jane`` -- "Contributor" can now be populated and the "Type" (Editor, Funder, Researcher, etc.) can be specified with an XML attribute. For example: CaffeineForAll +- "Contributor" can now be populated and the "Type" (Editor, Funder, Researcher, etc.) can be specified with an XML attribute. For example: ``CaffeineForAll`` -- "License" can now be set with dcterms:license and the possible values are "CC0" and "NONE". "License" interacts with "Terms of Use" (dcterms:rights) in that if you include dcterms:rights in the XML, the license will be set to "NONE". If you don't include dcterms:rights, the license will default to "CC0". It is invalid to specify "CC0" as a license and also include dcterms:rights; an error will be returned. For backwards compatibility, dcterms:rights is allowed to be blank (i.e. ) but blank values will not be persisted to the database and the license will be set to "NONE". +- "License" can now be set with ``dcterms:license`` and the possible values are "CC0" and "NONE". "License" interacts with "Terms of Use" (``dcterms:rights``) in that if you include ``dcterms:rights`` in the XML, the license will be set to "NONE". If you don't include ``dcterms:rights``, the license will default to "CC0". It is invalid to specify "CC0" as a license and also include ``dcterms:rights``; an error will be returned. For backwards compatibility, ``dcterms:rights`` is allowed to be blank (i.e. ````) but blank values will not be persisted to the database and the license will be set to "NONE". - "Contact E-mail" is automatically populated from dataset owner's email. @@ -114,6 +114,7 @@ To create a dataset, you must have the "Dataset Creator" role (the ``AddDataset` Example Atom entry (XML) .. literalinclude:: sword-atom-entry.xml + :language: xml Dublin Core Terms (DC Terms) Qualified Mapping - Dataverse Project DB Element Crosswalk ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From e211c55564c562260811a04dc2b69ad661807b3e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jun 2021 15:34:29 -0400 Subject: [PATCH 227/354] missing property --- src/main/java/propertyFiles/Bundle.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index db97e972684..d62496d93e2 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1740,6 +1740,7 @@ file.dataFilesTab.versions=Versions file.dataFilesTab.versions.headers.dataset=Dataset file.dataFilesTab.versions.headers.summary=Summary file.dataFilesTab.versions.headers.contributors=Contributors +file.dataFilesTab.versions.headers.contributors=Contributor name(s) withheld file.dataFilesTab.versions.headers.published=Published file.dataFilesTab.versions.viewDiffBtn=View Differences file.dataFilesTab.versions.citationMetadata=Citation Metadata: From 560bebbc1255cafc220eff7ce2e8e6a0d4a93fc0 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jun 2021 16:04:35 -0400 Subject: [PATCH 228/354] disable button when dataset has been published already --- src/main/java/propertyFiles/Bundle.properties | 1 + src/main/webapp/dataset.xhtml | 2 +- src/main/webapp/resources/css/structure.css | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index d62496d93e2..ba93d821e17 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1503,6 +1503,7 @@ dataset.privateurl.tip=Use a Private URL to allow those without Dataverse accoun dataset.privateurl.absent=Private URL has not been created. dataset.privateurl.createPrivateUrl=Create Private URL dataset.privateurl.createPrivateUrl.anonymized=Create URL for Anonymized Access +dataset.privateurl.createPrivateUrl.anonymized.unavailable=Anonymized Access is not available once a version of the dataset has been published dataset.privateurl.disablePrivateUrl=Disable Private URL dataset.privateurl.disablePrivateUrlConfirm=Yes, Disable Private URL dataset.privateurl.disableConfirmationText=Are you sure you want to disable the Private URL? If you have shared the Private URL with others they will no longer be able to use it to access your unpublished dataset. diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 4592c87fc55..61ecac4565f 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1010,7 +1010,7 @@
    • - + diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 0ffc336387e..10f65bca1db 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -375,6 +375,9 @@ form.btn-form {display:inline-block;} .button-block span.ui-state-disabled:last-child, .button-block.pull-right .btn:last-child, .button-block .btn:only-child, .button-block div.btn-group:only-child {margin-right:0;} +.disabledTitle { + pointer-events: auto; +} /* DIASABLED LINK-BTN TITLE PF COMMANDLINK HACK */ /* TODO - REVIEW DISABLED CLASS LINKS WHICH ARE NOW FUNCTIONAL DUE TO HACK?? From a9ddcb3ceeccd99135e12fc2c143f016811eec25 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 24 Jun 2021 16:32:04 -0400 Subject: [PATCH 229/354] don't maintain dynamic homepage, link to Harvard page #7968 --- .../branding/custom-homepage-dynamic.html | 360 ------------------ .../source/installation/config.rst | 4 +- 2 files changed, 2 insertions(+), 362 deletions(-) delete mode 100644 doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-homepage-dynamic.html diff --git a/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-homepage-dynamic.html b/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-homepage-dynamic.html deleted file mode 100644 index 4a6408c7b38..00000000000 --- a/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-homepage-dynamic.html +++ /dev/null @@ -1,360 +0,0 @@ - - - - -
    -
    -
    Deposit and share your data. Get academic credit.
    -

    Harvard Dataverse is a repository for research data. Deposit data and code here.

    - -

    - datasets -    - downloads -

    - - - Add a dataset - -
    -
    -
    Organize datasets and gather metrics in your own repository.
    -

    A dataverse is a container for all your datasets, files, and metadata.

    - -

    - dataverses -

    - - - Add a dataverse - -
    -
    - -
    - - -
    - - - -
    -
    Find data across research fields, preview metadata, and download files
    -
    -
    -
    - - - - -
    -
    -
    -

    VIEW ALL DATA

    -
    -
    -
    - - -
    -

    Browse by subject

    -
    - -
    -
    - - - -
    -

    ALL DATA

    -
    -
    -
    - - -
    -
    -
    -
    Datasets from journal dataverses
    - -
    - - -

    Loading...

    - - - - -
    - - - -
    -
    Datasets from other dataverses
    - -
    - - -

    Loading...

    -
    -
    - -
    -
    - - - -
    -
    - - -
    -

    ALL RECENT ACTIVITY

    -
    -
    -
    -
    -
    - -
    - - -
    -
    -
    Activity
    -
    - -
    - -
    -
    -
    Datasets
    -
    All Activity
    -
    Past 30 Days
    -
    -
    -
    Total
    -
    ...
    -
    ...
    -
    -
    -
    Deposited
    -
    ...
    -
    ...
    -
    -
    -
    Harvested
    -
    ...
    -
    ...
    -
    -
    - - - -
    -
    -
    Files
    -
    All Activity
    -
    Past 30 Days
    -
    -
    -
    Downloaded
    -
    ...
    -
    ...
    -
    -
    -
    Deposited
    -
    ...
    -
    ...
    -
    -
    - -
    -
    - -
    - - -
    -
    -
    Looking for other online repositories at Harvard?
    -
    -
    -

    Harvard DASH central, open-access repository

    -
    -
    -

    Other data at Harvard

    -
    -
    -
    -
    - - diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 39f27f749dc..d6c3b47ee24 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -604,9 +604,9 @@ Once you have the location of your custom homepage HTML file, run this curl comm ``curl -X PUT -d '/var/www/dataverse/branding/custom-homepage.html' http://localhost:8080/api/admin/settings/:HomePageCustomizationFile`` -If you prefer to start with less of a blank slate, you can download the :download:`custom-homepage-dynamic.html ` template which was built for the Harvard Dataverse Repository, and includes branding messaging, action buttons, search input, subject links, and recent dataset links. This page was built to utilize the :doc:`/api/metrics` to deliver dynamic content to the page via javascript. +If you prefer to start with less of a blank slate, you can review the custom homepage used by the Harvard Dataverse Repository, which includes branding messaging, action buttons, search input, subject links, and recent dataset links. This page was built to utilize the :doc:`/api/metrics` to deliver dynamic content to the page via Javascript. The files can be found at https://github.com/IQSS/dataverse.harvard.edu -Note that the ``custom-homepage.html`` and ``custom-homepage-dynamic.html`` files provided have multiple elements that assume your root Dataverse collection still has an alias of "root". While you were branding your root Dataverse collection, you may have changed the alias to "harvard" or "librascholar" or whatever and you should adjust the custom homepage code as needed. +Note that the ``custom-homepage.html`` file provided has multiple elements that assume your root Dataverse collection still has an alias of "root". While you were branding your root Dataverse collection, you may have changed the alias to "harvard" or "librascholar" or whatever and you should adjust the custom homepage code as needed. For more background on what this curl command above is doing, see the "Database Settings" section below. If you decide you'd like to remove this setting, use the following curl command: From 2adfa437365253584ae2e1260b0cdf6054cd2fc8 Mon Sep 17 00:00:00 2001 From: pkiraly Date: Thu, 24 Jun 2021 23:00:03 +0200 Subject: [PATCH 230/354] #7431 adding integration tests --- .../iq/dataverse/api/HarvestingServerIT.java | 15 +++++++++++++++ .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 7 ++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 4b3f35379c5..a1b70cc0f45 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -18,6 +18,7 @@ import static com.jayway.restassured.RestAssured.given; import java.util.List; import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; /** @@ -163,6 +164,20 @@ public void testOaiFunctionality() throws InterruptedException { // And the record should be the dataset we have just created: assertEquals(datasetPersistentId, listIdentifiersResponse.getBody().xmlPath().getString("OAI-PMH.ListIdentifiers.header.identifier")); + Response listRecordsResponse = UtilIT.getOaiListRecords(setName, "oai_dc"); + assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode()); + List listRecords = listRecordsResponse.getBody().xmlPath().getList("OAI-PMH.ListRecords.record"); + + assertNotNull(listRecords); + assertEquals(1, listRecords.size()); + assertEquals(datasetPersistentId, listRecordsResponse.getBody().xmlPath().getString("OAI-PMH.ListRecords.record[0].header.identifier")); + + // assert that Datacite format does not contain the XML prolog + Response listRecordsResponseDatacite = UtilIT.getOaiListRecords(setName, "Datacite"); + assertEquals(OK.getStatusCode(), listRecordsResponseDatacite.getStatusCode()); + String body = listRecordsResponseDatacite.getBody().asString(); + assertFalse(body.contains("")); + // And now run GetRecord on the OAI record for the dataset: Response getRecordResponse = UtilIT.getOaiRecord(datasetPersistentId, "oai_dc"); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index dc471a76a04..bb4d3fe3c5a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2359,7 +2359,12 @@ static Response getOaiListIdentifiers(String setName, String metadataFormat) { String apiPath = String.format("/oai?verb=ListIdentifiers&set=%s&metadataPrefix=%s", setName, metadataFormat); return given().get(apiPath); } - + + static Response getOaiListRecords(String setName, String metadataFormat) { + String apiPath = String.format("/oai?verb=ListRecords&set=%s&metadataPrefix=%s", setName, metadataFormat); + return given().get(apiPath); + } + static Response changeAuthenticatedUserIdentifier(String oldIdentifier, String newIdentifier, String apiToken) { Response response; String path = String.format("/api/users/%s/changeIdentifier/%s", oldIdentifier, newIdentifier ); From 7eb1a5e174157b754e8effe15ddd1f66dc8c2541 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Jun 2021 21:06:12 +0000 Subject: [PATCH 231/354] Bump httpclient from 4.5.5 to 4.5.13 Bumps httpclient from 4.5.5 to 4.5.13. --- updated-dependencies: - dependency-name: org.apache.httpcomponents:httpclient dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8579403a109..baadc21b63d 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ 1.11.762 1.2 3.12.0 - 4.5.5 + 4.5.13 4.13.1 5.7.0 ${junit.jupiter.version} From 0828366b055285386c45c06f200dbfd6e08ca321 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jun 2021 17:50:10 -0400 Subject: [PATCH 232/354] add anonymized request and released check, i18n messages --- .../command/impl/CreatePrivateUrlCommand.java | 30 ++++++++----------- src/main/java/propertyFiles/Bundle.properties | 5 ++++ 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommand.java index d86f60b4807..3f5a9b042f0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommand.java @@ -13,6 +13,8 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; +import edu.harvard.iq.dataverse.util.BundleUtil; + import java.util.UUID; import java.util.logging.Logger; @@ -34,30 +36,22 @@ public CreatePrivateUrlCommand(DataverseRequest dataverseRequest, Dataset theDat public PrivateUrl execute(CommandContext ctxt) throws CommandException { logger.fine("Executing CreatePrivateUrlCommand..."); if (dataset == null) { - /** - * @todo Internationalize this. - */ - String message = "Can't create Private URL. Dataset is null."; - logger.info(message); - throw new IllegalCommandException(message, this); + logger.info("Can't create Private URL. Dataset is null."); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.privateurl.error.datasetnotfound"), this); } PrivateUrl existing = ctxt.privateUrl().getPrivateUrlFromDatasetId(dataset.getId()); if (existing != null) { - /** - * @todo Internationalize this. - */ - String message = "Private URL already exists for dataset id " + dataset.getId() + "."; - logger.info(message); - throw new IllegalCommandException(message, this); + logger.info("Private URL already exists for dataset id " + dataset.getId() + "."); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.privateurl.error.alreadyexists"), this); } DatasetVersion latestVersion = dataset.getLatestVersion(); if (!latestVersion.isDraft()) { - /** - * @todo Internationalize this. - */ - String message = "Can't create Private URL because the latest version of dataset id " + dataset.getId() + " is not a draft."; - logger.info(message); - throw new IllegalCommandException(message, this); + logger.info("Can't create Private URL because the latest version of dataset id" + dataset.getId() + " is not a draft."); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.privateurl.error.notdraft"), this); + } + if (dataset.isReleased() && anonymizedAccess) { + logger.info("Can't create anonymized access Private URL because the dataset id" + dataset.getId() + " has a published version."); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.privateurl.anonymized.error.released"), this); } PrivateUrlUser privateUrlUser = new PrivateUrlUser(dataset.getId(), anonymizedAccess); DataverseRole memberRole = ctxt.roles().findBuiltinRoleByAlias(DataverseRole.MEMBER); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index ba93d821e17..ca2466a98a7 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2378,6 +2378,11 @@ datasets.api.grant.role.cant.create.assignment.error=Cannot create assignment: { datasets.api.grant.role.assignee.not.found.error=Assignee not found datasets.api.revoke.role.not.found.error="Role assignment {0} not found" datasets.api.revoke.role.success=Role {0} revoked for assignee {1} in {2} +datasets.api.privateurl.error.datasetnotfound=Could not find dataset. +datasets.api.privateurl.error.alreadyexists=Private URL already exists for this dataset. +datasets.api.privateurl.error.notdraft=Can't create Private URL because the latest version of this dataset is not a draft. +datasets.api.privateurl.anonymized.error.released=Can't create a URL for anonymized access because this dataset has been published. + #Dataverses.java dataverses.api.update.default.contributor.role.failure.role.not.found=Role {0} not found. From 603c6f1de1b3053d559b9818cd2c94de14c3d5a6 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jun 2021 17:50:22 -0400 Subject: [PATCH 233/354] add to notes --- .../iq/dataverse/privateurl/package-info.java | 47 ++++++++++++++----- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/privateurl/package-info.java b/src/main/java/edu/harvard/iq/dataverse/privateurl/package-info.java index 16ba6e5bf4c..6e939c1bb6d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/privateurl/package-info.java +++ b/src/main/java/edu/harvard/iq/dataverse/privateurl/package-info.java @@ -6,6 +6,13 @@ * all files (regardless of if the files are restricted or not) of a draft * version of a dataset. *

    + * As of this note, a second option - to create a Private URL that provides an + * anonymized view of the dataset has been added. This option works the same as + * the original except that it hides author names in the citation block, hides + * the values for an admin specified list of metadata fields, disables citation + * downloads, and disables API access (except for file and file thumbnail + * downloads which are used by the UI). + *

    * The primary use case for a Private URL is for journal editors to send a link * to reviewers of a dataset before publication. In most cases, these journal * editors do not permit depositors to publish on their own, which is to say @@ -26,12 +33,20 @@ * assign roles to them directly, rather than using a Private URL which requires * no username or password. *

    + * As of this note, a second option aimed specifically at the review use case - + * to create a Private URL that provides an anonymized view of the dataset - has + * been added. This option works the same as the original except that it hides + * author names in the citation block, hides the values for an admin specified + * list of metadata fields, disables citation downloads, and disables API access + * (except for file and file thumbnail downloads which are used by the UI). + *

    * The token associated with the Private URL role assignment that can be used - * either in the GUI or via the API to elevate privileges beyond what a "Guest" - * can see. The ability to use a Private URL token via API was added mostly to - * facilitate automated testing of the feature but the far more common case is - * expected to be use of the Private URL token in a link that is clicked to open - * a browser, similar to links shared via Dropbox, Google, etc. + * either in the GUI or, for the non-anonymized-access option, via the API to + * elevate privileges beyond what a "Guest" can see. The ability to use a + * Private URL token via API was added mostly to facilitate automated testing of + * the feature but the far more common case is expected to be use of the Private + * URL token in a link that is clicked to open a browser, similar to links + * shared via Dropbox, Google, etc. *

    * When reviewers click a Private URL their browser sessions are set to the * "{@link edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser}" that @@ -65,16 +80,24 @@ * using the "PrivateUrlUser" that has the "Member" role only on the dataset in * question. This means that read-only operations such as downloads of the * dataset's files are permitted. The Search API does not respect the Private - * URL token but you can download unpublished metadata using the Native API and - * download files using the Access API. + * URL token but you can download files using the Access API, and, with the + * non-anonymized-access option, download unpublished metadata using the Native + * API. *

    * A Private URL cannot be created for a published version of a dataset. In the * GUI, you will be reminded of this fact with a popup. The API will explain * this as well. *

    - * If a draft dataset containing a Private URL is published, the Private URL is - * deleted. This means that reviewers who click the link after publication will - * see a 404. + * An anonymized-access Private URL can't be created if any published dataset + * version exists. The primary reason for this is that, since datasets have + * DOIs, the full metadata about published versions is available directly from + * the DOI provider. (While the metadata for that version could be somewhat + * different, in practice it would probably provide a means of identifying + * some/all of the authors). + *

    + * If a draft dataset containing a Private URL is + * published, the Private URL is deleted. This means that reviewers who click + * the link after publication will see a 404. *

    * If a post-publication draft containing a Private URL is deleted, the Private * URL is deleted. This is to ensure that if a new draft is created in the @@ -106,8 +129,8 @@ *

  • {@link edu.harvard.iq.dataverse.engine.command.impl.CreatePrivateUrlCommand}
    • *
  • {@link edu.harvard.iq.dataverse.engine.command.impl.DeletePrivateUrlCommand}
    • * - * See also the Private URL To Unpublished Dataset BRD at - * + * See also the Private URL To Unpublished Dataset BRD at * https://docs.google.com/document/d/1FT47QkZKcmjSgRnePaJO2g1nzcotLyN3Yb2ORvBr6cs/edit?usp=sharing */ package edu.harvard.iq.dataverse.privateurl; From 2d23fe786d5f404f244b219496c51a2ef301bc29 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jun 2021 17:58:58 -0400 Subject: [PATCH 234/354] update tests, add test, drop exact message checks --- .../impl/CreatePrivateUrlCommandTest.java | 41 +++++++++++++------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java index 46c372ad313..aafad58654e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java @@ -28,7 +28,9 @@ public class CreatePrivateUrlCommandTest { private final Long privateUrlAlreadyExists = 1l; private final Long latestVersionIsNotDraft = 2l; private final Long createDatasetLong = 3l; - + private final Long versionIsReleased = 4l; + + @Before public void setUp() { dataset = new Dataset(); @@ -94,30 +96,25 @@ public String getDataverseSiteUrl() { @Test public void testDatasetNull() { dataset = null; - String expected = "Can't create Private URL. Dataset is null."; - String actual = null; PrivateUrl privateUrl = null; try { privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset, false)); + //Should not get here - exception expected + assertTrue(false); } catch (CommandException ex) { - actual = ex.getMessage(); } - assertEquals(expected, actual); assertNull(privateUrl); } @Test public void testAlreadyExists() { dataset.setId(privateUrlAlreadyExists); - String expected = "Private URL already exists for dataset id " + privateUrlAlreadyExists + "."; - String actual = null; PrivateUrl privateUrl = null; try { privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset, false)); + assertTrue(false); } catch (CommandException ex) { - actual = ex.getMessage(); } - assertEquals(expected, actual); assertNull(privateUrl); } @@ -130,15 +127,12 @@ public void testAttemptCreatePrivateUrlOnNonDraft() { versions.add(datasetVersion); dataset.setVersions(versions); dataset.setId(latestVersionIsNotDraft); - String expected = "Can't create Private URL because the latest version of dataset id " + latestVersionIsNotDraft + " is not a draft."; - String actual = null; PrivateUrl privateUrl = null; try { privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset, false)); + assertTrue(false); } catch (CommandException ex) { - actual = ex.getMessage(); } - assertEquals(expected, actual); assertNull(privateUrl); } @@ -176,5 +170,26 @@ public void testCreateAnonymizedAccessPrivateUrlSuccessfully() throws CommandExc assertTrue(privateUrl.isAnonymizedAccess()); assertEquals("https://dataverse.example.edu/privateurl.xhtml?token=" + privateUrl.getToken(), privateUrl.getLink()); } + + @Test + public void testAttemptCreateAnonymizedAccessPrivateUrlOnReleased() { + dataset = new Dataset(); + List versions = new ArrayList<>(); + DatasetVersion datasetVersion = new DatasetVersion(); + datasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED); + DatasetVersion datasetVersion2 = new DatasetVersion(); + + versions.add(datasetVersion); + versions.add(datasetVersion2); + dataset.setVersions(versions); + dataset.setId(versionIsReleased); + PrivateUrl privateUrl = null; + try { + privateUrl = testEngine.submit(new CreatePrivateUrlCommand(null, dataset, true)); + assertTrue(false); + } catch (CommandException ex) { + } + assertNull(privateUrl); + } } From f2e41583880179a1e45dd936a6b177098fbd9491 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jun 2021 09:58:04 -0400 Subject: [PATCH 235/354] flyway script --- .../resources/db/migration/V5.5.0.3__1724-anonymizedaccess.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src/main/resources/db/migration/V5.5.0.3__1724-anonymizedaccess.sql diff --git a/src/main/resources/db/migration/V5.5.0.3__1724-anonymizedaccess.sql b/src/main/resources/db/migration/V5.5.0.3__1724-anonymizedaccess.sql new file mode 100644 index 00000000000..da016361596 --- /dev/null +++ b/src/main/resources/db/migration/V5.5.0.3__1724-anonymizedaccess.sql @@ -0,0 +1,2 @@ +ALTER TABLE roleassignment ADD COLUMN IF NOT EXISTS privateurlanonymizedaccess boolean; + From a4d297ec6c71a35d8ac39a1268cd981f5d95b368 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 25 Jun 2021 10:52:47 -0400 Subject: [PATCH 236/354] explain backward incompatible chane to Metrics API #7968 --- doc/release-notes/5.5-release-notes.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/5.5-release-notes.md b/doc/release-notes/5.5-release-notes.md index 7c4726995f8..93383749578 100644 --- a/doc/release-notes/5.5-release-notes.md +++ b/doc/release-notes/5.5-release-notes.md @@ -20,7 +20,14 @@ The Software Metadata Working Group has created guidelines on depositing researc ### New Metrics API -Users can retrieve new types of metrics and per-collection metrics. The new capabilities are described in the guides. A new version of the [Dataverse Metrics web app](https://github.com/IQSS/dataverse-metrics) adds interactive graphs to display these metrics. Anyone running the existing Dataverse Metrics app will need to upgrade or apply [a small patch](https://github.com/IQSS/dataverse-metrics/pull/60) to continue retrieving metrics from Dataverse instances upgrading to this release. +Users can retrieve new types of metrics and per-collection metrics. The new capabilities are [described](https://guides.dataverse.org/en/5.5/api/metrics.html) in the guides. A new version of the [Dataverse Metrics web app](https://github.com/IQSS/dataverse-metrics) adds interactive graphs to display these metrics. + +There is a backward incompatible change to the Metrics API whereby you must now send the header `Accept: application/json` if you want to continue receiving JSON output. As of this release, absence of this header results in CSV rather than JSON for time series metrics. For details, see [Return Formats](https://guides.dataverse.org/en/5.5/api/metrics.html#return-formats) in the Metrics API section. + +Because of this change from JSON to CSV, adjustments have been made to the software that makes use of the Metrics API: + +- Anyone running the existing Dataverse Metrics app will need to upgrade to 0.2.8 (or higher) or apply [a small patch](https://github.com/IQSS/dataverse-metrics/pull/60) to continue retrieving metrics from Dataverse instances upgrading to this release. +- Anyone who is using the Harvard Dataverse Repository custom homepage as a model might want to look at [changes](https://github.com/IQSS/dataverse.harvard.edu/pull/116) to get elements such as "browse by subject" working again. ## Major Use Cases @@ -86,12 +93,20 @@ Most of your API users have likely figured it out already, since you enabled S3 The "deactivated" field on the Authenticated User table has been updated to be a non-nullable field. When the field was added in version 5.3 it was set to 'false' in an update script. If for whatever reason that update failed in the 5.3 deploy you will need to re-run it before deploying 5.5. The update query you may need to run is: UPDATE authenticateduser SET deactivated = false WHERE deactivated IS NULL; +### Metrics API Change + +See "New Metrics API" above for a backward incompatible change. + ## Notes for Tool Developers and Integrators ### S3 Download Redirects See above note about download redirects. If your application integrates with the the Dataverse software using the APIs, you may need to change how redirects are handled in your tool or integration. +### Metrics API Change + +See "New Metrics API" above for a backward incompatible change. + ## Complete List of Changes For the complete list of code changes in this release, see the [5.5 Milestone](https://github.com/IQSS/dataverse/milestone/96?closed=1) in Github. From 870389e9454b31a503fa9910b2b247cf56a3506b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 25 Jun 2021 12:08:14 -0400 Subject: [PATCH 237/354] remove references to Travis from docs #7876 --- .../source/developers/testing.rst | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index e7bc6b2fed5..7bde4055e33 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -37,11 +37,9 @@ A unit test should execute an operation of your code in a controlled fashion. Yo Unit Test Automation Overview ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -We use a variety of tools to write, execute, and measure the code coverage of unit tests, including Maven, JUnit, Jacoco, GitHub, Travis, and Coveralls. We'll explain the role of each tool below, but here's an overview of what you can expect from the automation we've set up. +We use a variety of tools to write, execute, and measure the code coverage of unit tests, including Maven, JUnit, Jacoco, GitHub, and Coveralls. We'll explain the role of each tool below, but here's an overview of what you can expect from the automation we've set up. -As you prepare to make a pull request, as described in the :doc:`version-control` section, you will be working on a new branch you create from the "develop" branch. Let's say your branch is called ``1012-private-url``. As you work, you are constantly invoking Maven to build the war file. When you do a "clean and build" in Netbeans, Maven runs all the unit tests (anything ending with ``Test.java``) and the runs the results through a tool called Jacoco that calculates code coverage. When you push your branch to GitHub and make a pull request, a web service called Travis CI runs Maven and Jacoco on your branch and pushes the results to Coveralls, which is a web service that tracks changes to code coverage over time. - -To make this more concrete, observe that https://github.com/IQSS/dataverse/pull/3111 has comments from a GitHub user called ``coveralls`` saying things like "Coverage increased (+0.5%) to 5.547% when pulling dd6ceb1 on 1012-private-url into be5b26e on develop." Clicking on the comment should lead you to a URL such as https://coveralls.io/builds/7013870 which shows how code coverage has gone up or down. That page links to a page such as https://travis-ci.org/IQSS/dataverse/builds/144840165 which shows the build on the Travis side that pushed the results to Coveralls. Note that we have configured Coveralls to not mark small decreases in code coverage as a failure. +As you prepare to make a pull request, as described in the :doc:`version-control` section, you will be working on a new branch you create from the "develop" branch. Let's say your branch is called ``1012-private-url``. As you work, you are constantly invoking Maven to build the war file. When you do a "clean and build" in Netbeans, Maven runs all the unit tests (anything ending with ``Test.java``) and then runs the results through a tool called Jacoco that calculates code coverage. When you push your branch to GitHub and make a pull request, GitHub Actions runs Maven and Jacoco on your branch and pushes the results to Coveralls, which is a web service that tracks changes to code coverage over time. Note that we have configured Coveralls to not mark small decreases in code coverage as a failure. You can find the Coveralls reports at https://coveralls.io/github/IQSS/dataverse The main takeaway should be that we care about unit testing enough to measure the changes to code coverage over time using automation. Now let's talk about how you can help keep our code coverage up by writing unit tests with JUnit. @@ -102,12 +100,10 @@ In addition, there is a writeup on "The Testable Command" at https://github.com/ Running Non-Essential (Excluded) Unit Tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -You should be aware that some unit tests have been deemed "non-essential" and have been annotated with ``@Category(NonEssentialTests.class)`` and are excluded from the "dev" Maven profile, which is the default profile. All unit tests (that have not been annotated with ``@Ignore``), including these non-essential tests, are run from continuous integration systems such as Jenkins and Travis CI with the following ``mvn`` command that invokes a non-default profile: +You should be aware that some unit tests have been deemed "non-essential" and have been annotated with ``@Category(NonEssentialTests.class)`` and are excluded from the "dev" Maven profile, which is the default profile. All unit tests (that have not been annotated with ``@Ignore``), including these non-essential tests, are run from continuous integration systems such as Jenkins and GitHub Actions with the following ``mvn`` command that invokes a non-default profile: ``mvn test -P all-unit-tests`` -Typically https://travis-ci.org/IQSS/dataverse will show a higher number of unit tests executed because it uses the profile above. - Generally speaking, unit tests have been flagged as non-essential because they are slow or because they require an Internet connection. You should not feel obligated to run these tests continuously but you can use the ``mvn`` command above to run them. To iterate on the unit test in Netbeans and execute it with "Run -> Test File", you must temporarily comment out the annotation flagging the test as non-essential. Integration Tests @@ -393,12 +389,12 @@ The script requires a file called ``files.txt`` to operate and database IDs for Continuous Integration ---------------------- -The Dataverse Project currently makes use of two Continuous Integration platforms, Travis and Jenkins. - -Travis builds are configured via :download:`.travis.yml <../../../../.travis.yml>` and a `GitHub webhook `; build output is viewable at https://travis-ci.org/IQSS/dataverse/builds +The Dataverse Project currently makes use of two Continuous Integration platforms, Jenkins and GitHub Actions. Our Jenkins config is a work in progress and may be viewed at https://github.com/IQSS/dataverse-jenkins/ A corresponding GitHub webhook is required. Build output is viewable at https://jenkins.dataverse.org/ +GitHub Actions jobs can be found in ``.github/workflows``. + As always, pull requests to improve our continuous integration configurations are welcome. Enhance build time by caching dependencies @@ -475,7 +471,6 @@ Future Work on Integration Tests - Automate testing of dataverse-client-python: https://github.com/IQSS/dataverse-client-python/issues/10 - Work with @leeper on testing the R client: https://github.com/IQSS/dataverse-client-r - Review and attempt to implement "API Test Checklist" from @kcondon at https://docs.google.com/document/d/199Oq1YwQ4pYCguaeW48bIN28QAitSk63NbPYxJHCCAE/edit?usp=sharing -- Attempt to use @openscholar approach for running integration tests using Travis https://github.com/openscholar/openscholar/blob/SCHOLAR-3.x/.travis.yml (probably requires using Ubuntu rather than CentOS) - Generate code coverage reports for **integration** tests: https://github.com/pkainulainen/maven-examples/issues/3 and http://www.petrikainulainen.net/programming/maven/creating-code-coverage-reports-for-unit-and-integration-tests-with-the-jacoco-maven-plugin/ - Consistent logging of API Tests. Show test name at the beginning and end and status codes returned. - expected passing and known/expected failing integration tests: https://github.com/IQSS/dataverse/issues/4438 @@ -488,15 +483,14 @@ Browser-Based Testing Installation Testing ~~~~~~~~~~~~~~~~~~~~ -- Run `vagrant up` on a server to test the installer: http://guides.dataverse.org/en/latest/developers/tools.html#vagrant . We haven't been able to get this working in Travis: https://travis-ci.org/IQSS/dataverse/builds/96292683 . Perhaps it would be possible to use AWS as a provider from Vagrant judging from https://circleci.com/gh/critical-alert/circleci-vagrant/6 . -- Work with @lwo to automate testing of https://github.com/IQSS/dataverse-puppet . Consider using Travis: https://github.com/IQSS/dataverse-puppet/issues/10 -- Work with @donsizemore to automate testing of https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible with Travis or similar. +- Run `vagrant up` on a server to test the installer +- Work with @donsizemore to automate testing of https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible Future Work on Load/Performance Testing ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Clean up and copy stress tests code, config, and docs into main repo from https://github.com/IQSS/dataverse-helper-scripts/tree/master/src/stress_tests -- Marcel Duran created a command-line wrapper for the WebPagetest API that can be used to test performance in your continuous integration pipeline (TAP, Jenkins, Travis-CI, etc): https://github.com/marcelduran/webpagetest-api/wiki/Test-Specs#jenkins-integration +- Marcel Duran created a command-line wrapper for the WebPagetest API that can be used to test performance in your continuous integration pipeline (TAP, Jenkins, etc.): https://github.com/marcelduran/webpagetest-api/wiki/Test-Specs#jenkins-integration - Create top-down checklist, building off the "API Test Coverage" spreadsheet at https://github.com/IQSS/dataverse/issues/3358#issuecomment-256400776 Future Work on Accessibility Testing From 1ef71be4578e5dda0a2c0d1e3b0394b0b2e06f27 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 25 Jun 2021 12:12:09 -0400 Subject: [PATCH 238/354] remove last (?) mention of Travis #7876 --- doc/theTestableCommand/TheTestableCommand.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/theTestableCommand/TheTestableCommand.md b/doc/theTestableCommand/TheTestableCommand.md index e60faa313f0..5a9fc259d4a 100644 --- a/doc/theTestableCommand/TheTestableCommand.md +++ b/doc/theTestableCommand/TheTestableCommand.md @@ -21,7 +21,7 @@ While they can't replace end-to-end tests, unit tests are a great way to validat Because unit tests are easy to create (Java only, no configuration needed) and quick to run, it is possible to write many of them, such that many aspects of the code are tested. Normally, a single unit test would test a single use case of the unit. This way, when a unit test fails, the failure describes exactly what part stopped functioning. Other unit tests are not blocked by the failure, and so by running the entire test suite, the developer can get a good overview of which parts are broken and which parts are functioning well. -Because unit tests are easy to execute, it is recommended to get in the habit of running them prior to committing code changes to the repository. These tests are also integrated into Dataverse's automatic build processes (on [Travis-ci](https://travis-ci.org/IQSS/dataverse)). A failed test halts the build. Dataverse's build process also collects data about code coverage during the unit tests, using [Coveralls](https://coveralls.io/github/IQSS/dataverse). While code coverage is a problematic measure for Java EE applications (and has some inherent problems as well), generally speaking larger coverage means better testing. +Because unit tests are easy to execute, it is recommended to get in the habit of running them prior to committing code changes to the repository. These tests are also integrated into Dataverse's automatic build processes. A failed test halts the build. Dataverse's build process also collects data about code coverage during the unit tests, using [Coveralls](https://coveralls.io/github/IQSS/dataverse). While code coverage is a problematic measure for Java EE applications (and has some inherent problems as well), generally speaking larger coverage means better testing. Unit Testing of application logic in Java EE applications is normally hard to do, as the application logic lives in the service beans, which rely on dependency injections. Writing unit tests for service beans is possible, but as it involves a test container, and a persistent context (read: in-memory database) these unit tests are not very unit-y. From 12aa7368f845fc8b5db69044197607eda07a8054 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 25 Jun 2021 12:33:26 -0400 Subject: [PATCH 239/354] doc fixes, update for ld+json type --- doc/sphinx-guides/source/admin/dashboard.rst | 2 +- .../dataset-semantic-metadata-api.rst | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/admin/dashboard.rst b/doc/sphinx-guides/source/admin/dashboard.rst index 5129552b706..3f77729d0ab 100644 --- a/doc/sphinx-guides/source/admin/dashboard.rst +++ b/doc/sphinx-guides/source/admin/dashboard.rst @@ -22,7 +22,7 @@ This dashboard tool allows you to define sets of local datasets to make availabl Metadata Export --------------- -This part of the Dashboard is simply a reminder message that metadata export happens through the Dataverse Software API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. +This part of the Dashboard is simply a reminder message that metadata export happens through the Dataverse Software API. See the :doc:`/admin/metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. Users ----- diff --git a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst index ddabedca1a5..00876d0887a 100644 --- a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst @@ -1,11 +1,13 @@ Dataset Semantic Metadata API ============================= -The OAI_ORE metadata export format represents Dataset metadata using json-ld (see the :doc:`metadataexport` section). As part of an RDA-supported effort to allow import of Datasets exported as Bags with an included OAI_ORE metadata file, +The OAI_ORE metadata export format represents Dataset metadata using json-ld (see the :doc:`/admin/metadataexport` section). As part of an RDA-supported effort to allow import of Datasets exported as Bags with an included OAI_ORE metadata file, an experimental API has been created that provides a json-ld alternative the the v1.0 API calls to get/set/delete Dataset metadata in the :doc:`/api/native-api`. You may prefer to work with this API if you are building a tool to import from a Bag/OAI-ORE source or already work with json-ld representations of metadata, or if you prefer the flatter json-ld representation to Dataverse software's json representation (which includes structure related to the metadata blocks involved and the type/multiplicity of the metadata fields.) You may not want to use this API if you need stability and backward compatibility (the 'experimental' designation for this API implies that community feedback is desired and that, in future Dataverse software versions, the API may be modified based on that feedback). + +Note: The examples use the 'application/ld+json' mimetype. For compatibility reasons, the APIs also be used with mimetype "application/json-ld" Get Dataset Metadata -------------------- @@ -22,11 +24,11 @@ To get the json-ld formatted metadata for a Dataset, specify the Dataset ID (DAT Example 1: Get metadata for version '1.0' - curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' "$SERVER_URL/api/datasets/$DATASET_ID/versions/$VERSION/metadata" + curl -H X-Dataverse-key:$API_TOKEN -H 'Accept: application/ld+json' "$SERVER_URL/api/datasets/$DATASET_ID/versions/$VERSION/metadata" Example 2: Get metadata for the latest version using the DATASET PID - curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID" + curl -H X-Dataverse-key:$API_TOKEN -H 'Accept: application/ld+json' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID" You should expect a 200 ("OK") response and JSON-LD mirroring the OAI-ORE representation in the returned 'data' object. @@ -46,11 +48,11 @@ To add json-ld formatted metadata for a Dataset, specify the Dataset ID (DATASET Example: Change the Dataset title - curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"Title": "Submit menu test", "@context":{"Title": "http://purl.org/dc/terms/title"}}' "$SERVER_URL/api/datasets/$DATASET_ID/metadata?replace=true" + curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -d '{"Title": "Submit menu test", "@context":{"Title": "http://purl.org/dc/terms/title"}}' "$SERVER_URL/api/datasets/$DATASET_ID/metadata?replace=true" Example 2: Add a description using the DATASET PID - curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"citation:Description": {"dsDescription:Text": "New description"}, "@context":{"citation": "https://dataverse.org/schema/citation/","dsDescription": "https://dataverse.org/schema/citation/dsDescription#"}}' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID" + curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -d '{"citation:Description": {"dsDescription:Text": "New description"}, "@context":{"citation": "https://dataverse.org/schema/citation/","dsDescription": "https://dataverse.org/schema/citation/dsDescription#"}}' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID" You should expect a 200 ("OK") response indicating whether a draft Dataset version was created or an existing draft was updated. @@ -70,7 +72,7 @@ To delete metadata for a Dataset, send a json-ld representation of the fields to Example: Delete the TermsOfUseAndAccess 'restrictions' value 'No restrictions' for the latest version using the DATASET PID - curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/json-ld' -d '{"https://dataverse.org/schema/core#restrictions":"No restrictions"}' "$SERVER_URL/api/datasets/:persistentId/metadata/delete?persistentId=$DATASET_PID" + curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -d '{"https://dataverse.org/schema/core#restrictions":"No restrictions"}' "$SERVER_URL/api/datasets/:persistentId/metadata/delete?persistentId=$DATASET_PID" Note, this example uses the term URI directly rather than adding an '@context' element. You can use either form in any of these API calls. @@ -80,10 +82,10 @@ You should expect a 200 ("OK") response indicating whether a draft Dataset versi Create a Dataset ---------------- -Specifying the Content-Type as application/json-ld with the existing /api/dataverses/{id}/datasets API call (see :ref:`create-dataset-command`) supports using the same metadata format when creating a Dataset. +Specifying the Content-Type as application/ld+json with the existing /api/dataverses/{id}/datasets API call (see :ref:`create-dataset-command`) supports using the same metadata format when creating a Dataset. With curl, this is done by adding the following header: .. code-block:: bash - -H 'Content-Type: application/json-ld' + -H 'Content-Type: application/ld+json' From ff2419b9d62095405ddc5f67e40f37fcd7b19a4c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 25 Jun 2021 12:34:30 -0400 Subject: [PATCH 240/354] support application/ld+json(new) application/json-ld (old/non-standard) --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 12 +++++------- .../edu/harvard/iq/dataverse/api/Dataverses.java | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 3ecd785257f..0a86d8b67f7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -666,7 +666,7 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, @GET @Path("{id}/versions/{versionId}/metadata") - @Produces("application/json-ld") + @Produces("application/ld+json, application/json-ld") public Response getVersionJsonLDMetadata(@PathParam("id") String id, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) { try { DataverseRequest req = createDataverseRequest(findUserOrDie()); @@ -687,14 +687,14 @@ public Response getVersionJsonLDMetadata(@PathParam("id") String id, @PathParam( @GET @Path("{id}/metadata") - @Produces("application/json-ld") + @Produces("application/ld+json, application/json-ld") public Response getVersionJsonLDMetadata(@PathParam("id") String id, @Context UriInfo uriInfo, @Context HttpHeaders headers) { return getVersionJsonLDMetadata(id, ":draft", uriInfo, headers); } @PUT @Path("{id}/metadata") - @Consumes("application/json-ld") + @Consumes("application/ld+json, application/json-ld") public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { try { @@ -724,16 +724,14 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String @PUT @Path("{id}/metadata/delete") - @Consumes("application/json-ld") + @Consumes("application/ld+json, application/json-ld") public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id) { - logger.info("In delteMetadata"); try { Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); DatasetVersion dsv = ds.getEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); dsv = JSONLDUtil.deleteDatasetVersionMDFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc); - logger.info("Updating ver"); DatasetVersion managedVersion; if (updateDraft) { Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); @@ -1206,7 +1204,7 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S @POST @Path("{id}/actions/:releasemigrated") - @Consumes("application/json-ld") + @Consumes("application/ld+json, application/json-ld") public Response publishMigratedDataset(String jsonldBody, @PathParam("id") String id) { try { AuthenticatedUser user = findAuthenticatedUserOrDie(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 4f4034e7723..675a44d9a5c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -258,7 +258,7 @@ public Response createDataset(String jsonBody, @PathParam("identifier") String p @POST @Path("{identifier}/datasets") - @Consumes("application/json-ld") + @Consumes("application/ld+json, application/json-ld") public Response createDatasetFromJsonLd(String jsonLDBody, @PathParam("identifier") String parentIdtf) { try { User u = findUserOrDie(); @@ -438,7 +438,7 @@ public Response importDatasetDdi(String xml, @PathParam("identifier") String par @POST @Path("{identifier}/datasets/:startmigration") - @Consumes("application/json-ld") + @Consumes("application/ld+json, application/json-ld") public Response recreateDataset(String jsonLDBody, @PathParam("identifier") String parentIdtf) { try { User u = findUserOrDie(); From da9c9e9e24556fafa6bc81d78cddd29b0632094e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 25 Jun 2021 12:34:43 -0400 Subject: [PATCH 241/354] initial test commit --- .../search/tests/data/dataset-finch1.jsonld | 15 ++ .../harvard/iq/dataverse/api/DatasetsIT.java | 156 +++++++++++++++++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 38 ++++- 3 files changed, 206 insertions(+), 3 deletions(-) create mode 100644 scripts/search/tests/data/dataset-finch1.jsonld diff --git a/scripts/search/tests/data/dataset-finch1.jsonld b/scripts/search/tests/data/dataset-finch1.jsonld new file mode 100644 index 00000000000..24d7a1b2c9f --- /dev/null +++ b/scripts/search/tests/data/dataset-finch1.jsonld @@ -0,0 +1,15 @@ +{ + "http://purl.org/dc/terms/title": "Darwin's Finches", + "http://purl.org/dc/terms/creator": { + "https://dataverse.org/schema/citation/author#Name": "Finch, Fiona", + "https://dataverse.org/schema/citation/author#Affiliation": "Birds Inc." + } + "https://dataverse.org/schema/citation/Contact": { + "https://dataverse.org/schema/citation/datasetContact#E-mail": "finch@mailinator.com". + "https://dataverse.org/schema/citation/datasetContact#Name": "Finch, Fiona", + } + "https://dataverse.org/schema/citation/Description": { + "https://dataverse.org/schema/citation/dsDescription#Text": "Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds." + } + "http://purl.org/dc/terms/subject": "Medicine, Health and Life Sciences" +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index a7dc41d9c74..5943650e802 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -36,6 +36,8 @@ import edu.harvard.iq.dataverse.authorization.groups.impl.builtin.AuthenticatedUsers; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.json.JSONLDUtil; + import java.io.File; import java.io.IOException; import java.nio.file.Path; @@ -55,7 +57,7 @@ import static javax.ws.rs.core.Response.Status.NO_CONTENT; import static javax.ws.rs.core.Response.Status.OK; import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; -import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertEquals; import org.hamcrest.CoreMatchers; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; @@ -2163,5 +2165,157 @@ public void testRestrictFileExportDdi() throws IOException { .statusCode(BAD_REQUEST.getStatusCode()) .body("message", equalTo("You do not have permission to download this file.")); } + + + @Test + public void testSemanticMetadataAPIs() { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + + //Create a dataset using native api + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + //Get the metadata with the semantic api + Response response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); + response.then().assertThat().statusCode(OK.getStatusCode()); + + //Compare the metadata with an expected value - the metadatablock entries should be the same but there will be additional fields with values related to the dataset's creation (e.g. new id) + String jsonLDString = JsonPath.from(response.getBody().asString()).getString("data"); + String jsonLD = JSONLDUtil.decontextualizeJsonLD(jsonLDString).toString(); + + String expectedJsonLD = UtilIT.getDatasetJson("scripts/search/tests/data/dataset-finch1.jsonld"); + + //ToDo: Are the static pars as expected + assertEquals(expectedJsonLD, jsonLD); + //Now change the title + response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, + "{\"Title\": \"New Title\", \"@context\":{\"Title\": \"http://purl.org/dc/terms/title\"}}", true); + response.then().assertThat().statusCode(OK.getStatusCode()); + + response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); + response.then().assertThat().statusCode(OK.getStatusCode()); + + //Check that the semantic api returns the new title + jsonLDString = JsonPath.from(response.getBody().asString()).getString("data"); + JsonObject jsonLDObject = JSONLDUtil.decontextualizeJsonLD(jsonLDString); + assertEquals("New Title", jsonLDObject.getString("http://purl.org/dc/terms/title")); + + //Add an additional description (which is multi-valued and compound) + //Also add new terms of use (single value so would fail with replace false if a value existed) + String newDescription = "{\"citation:Description\": {\"dsDescription:Text\": \"New description\", \"https://dataverse.org/schema/core#termsOfUse\": \"New terms\"} \"@context\":{\"citation\": \"https://dataverse.org/schema/citation/\",\"dsDescription\": \"https://dataverse.org/schema/citation/dsDescription#\"}}"; + response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, newDescription, false); + response.then().assertThat().statusCode(OK.getStatusCode()); + + response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); + response.then().assertThat().statusCode(OK.getStatusCode()); + + //Look for a second description + jsonLDString = JsonPath.from(response.getBody().asString()).getString("data"); + jsonLDObject = JSONLDUtil.decontextualizeJsonLD(jsonLDString); + assertEquals("New description", + ((JsonObject) jsonLDObject.getJsonArray("https://dataverse.org/schema/citation/Description").get(1)) + .getString("https://dataverse.org/schema/citation/dsDescription#Text")); + + //Can't add terms of use with replace=false and a value already set (single valued field) + String badTerms = "{\"https://dataverse.org/schema/core#termsOfUse\": \"Bad terms\"}}"; + response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, badTerms, false); + response.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); + + + //Delete the terms of use + response = UtilIT.deleteDatasetJsonLDMetadata(datasetId, apiToken, + "{\"https://dataverse.org/schema/core#termsOfUse\": \"New terms\"}"); + response.then().assertThat().statusCode(OK.getStatusCode()); + + response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); + response.then().assertThat().statusCode(OK.getStatusCode()); + + //Verify that they're gone + jsonLDString = JsonPath.from(response.getBody().asString()).getString("data"); + jsonLDObject = JSONLDUtil.decontextualizeJsonLD(jsonLDString); + assertTrue(!jsonLDObject.containsKey("https://dataverse.org/schema/core#termsOfUse")); + + //Cleanup - delete dataset, dataverse, user... + Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); + deleteDatasetResponse.prettyPrint(); + assertEquals(200, deleteDatasetResponse.getStatusCode()); + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + deleteDataverseResponse.prettyPrint(); + assertEquals(200, deleteDataverseResponse.getStatusCode()); + + Response deleteUserResponse = UtilIT.deleteUser(username); + deleteUserResponse.prettyPrint(); + assertEquals(200, deleteUserResponse.getStatusCode()); + + } + @Test + public void testReCreateDataset() { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + Response makeSuperUser = UtilIT.makeSuperUser(username); + assertEquals(200, makeSuperUser.getStatusCode()); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + //Create a dataset using native API + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + //Get the semantic metadata + Response response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); + response.then().assertThat().statusCode(OK.getStatusCode()); + + String expectedJsonLD = JsonPath.from(response.getBody().asString()).getString("data"); + + //Delete the dataset via native API + Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); + deleteDatasetResponse.prettyPrint(); + assertEquals(200, deleteDatasetResponse.getStatusCode()); + + //Now use the migrate API to recreate the dataset + response = UtilIT.recreateDatasetJsonLD(datasetId, apiToken, dataverseAlias, expectedJsonLD); + response.then().assertThat().statusCode(CREATED.getStatusCode()); + //Id will change but everything else should be as before (DOI and create date) + datasetId = JsonPath.from(response.getBody().asString()).getInt("data/id"); + + response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); + response.then().assertThat().statusCode(OK.getStatusCode()); + String jsonLDString = JsonPath.from(response.getBody().asString()).getString("data"); + JsonObject jsonLD = JSONLDUtil.decontextualizeJsonLD(jsonLDString); + + + //ToDo: Assert that the semantic api response is the same except for the id + assertEquals(expectedJsonLD, jsonLD.toString()); + + //Cleanup by deleting things + deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); + deleteDatasetResponse.prettyPrint(); + assertEquals(200, deleteDatasetResponse.getStatusCode()); + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + deleteDataverseResponse.prettyPrint(); + assertEquals(200, deleteDataverseResponse.getStatusCode()); + + Response deleteUserResponse = UtilIT.deleteUser(username); + deleteUserResponse.prettyPrint(); + assertEquals(200, deleteUserResponse.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index dc471a76a04..e7565b71adc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -31,6 +31,7 @@ import java.nio.file.Path; import java.util.ArrayList; import org.apache.commons.lang3.math.NumberUtils; +import org.apache.cxf.headers.Header; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; @@ -397,7 +398,7 @@ static Response createDatasetViaNativeApi(String dataverseAlias, String pathToJs return createDatasetResponse; } - private static String getDatasetJson(String pathToJsonFile) { + static String getDatasetJson(String pathToJsonFile) { File datasetVersionJson = new File(pathToJsonFile); try { String datasetVersionAsJson = new String(Files.readAllBytes(Paths.get(datasetVersionJson.getAbsolutePath()))); @@ -1504,7 +1505,7 @@ static Response nativeGet(Integer datasetId, String apiToken) { .get("/api/datasets/" + datasetId); return response; } - + static Response privateUrlGet(Integer datasetId, String apiToken) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) @@ -2628,5 +2629,38 @@ static String getBannerMessageIdFromResponse(String getBannerMessagesResponse) { return "0"; } + static Response getDatasetJsonLDMetadata(Integer datasetId, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .accept("application/ld+json") + .get("/api/datasets/" + datasetId + "/metadata"); + return response; + } + + static Response updateDatasetJsonLDMetadata(Integer datasetId, String apiToken, String jsonLDBody, boolean replace) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType("application/ld+json") + .body(jsonLDBody) + .put("/api/datasets/" + datasetId + "/metadata?replace=" + replace); + return response; + } + static Response deleteDatasetJsonLDMetadata(Integer datasetId, String apiToken, String jsonLDBody) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType("application/ld+json") + .body(jsonLDBody) + .put("/api/datasets/" + datasetId + "/metadata/delete"); + return response; + } + + public static Response recreateDatasetJsonLD(Integer datasetId, String apiToken, String dataverseAlias, String jsonLDBody) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType("application/ld+json") + .body(jsonLDBody) + .post("https://demo.dataverse.org/api/dataverses/" + dataverseAlias +"/datasets"); + return response; + } } From d671456ab9159b007707c52341e569a3ff358c64 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jun 2021 13:26:22 -0400 Subject: [PATCH 242/354] fixes per review --- .../java/edu/harvard/iq/dataverse/api/AbstractApiBean.java | 3 ++- src/main/java/propertyFiles/Bundle.properties | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 5fef7af56cf..746a721e4e1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -367,7 +367,8 @@ protected User findUserOrDie() throws WrappedResponse { if (privateUrlUser != null) { if (privateUrlUser.hasAnonymizedAccess()) { String pathInfo = httpRequest.getPathInfo(); - if (!(pathInfo.startsWith("/access/datafile/") && !pathInfo.substring(17).contains("/"))) { + String prefix= "/access/datafile/"; + if (!(pathInfo.startsWith(prefix) && !pathInfo.substring(prefix.length()).contains("/"))) { logger.info("Anonymized access request for " + pathInfo); throw new WrappedResponse(error(Status.UNAUTHORIZED, "API Access not allowed with this Key")); } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index ca2466a98a7..f0194ca60dc 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1741,7 +1741,7 @@ file.dataFilesTab.versions=Versions file.dataFilesTab.versions.headers.dataset=Dataset file.dataFilesTab.versions.headers.summary=Summary file.dataFilesTab.versions.headers.contributors=Contributors -file.dataFilesTab.versions.headers.contributors=Contributor name(s) withheld +file.dataFilesTab.versions.headers.contributors.withheld=Contributor name(s) withheld file.dataFilesTab.versions.headers.published=Published file.dataFilesTab.versions.viewDiffBtn=View Differences file.dataFilesTab.versions.citationMetadata=Citation Metadata: From 069043f71c67854ff2fd3d2fa14fbf79afd49681 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 25 Jun 2021 15:08:45 -0400 Subject: [PATCH 243/354] More forceful language in the "pre-release note" about the dvobject constraint being enforced on existing databases in the next release. (#7451) --- scripts/issues/7451/PRE-RELEASE-INFO.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/issues/7451/PRE-RELEASE-INFO.txt b/scripts/issues/7451/PRE-RELEASE-INFO.txt index b0aca07bfae..c14c25aa93f 100644 --- a/scripts/issues/7451/PRE-RELEASE-INFO.txt +++ b/scripts/issues/7451/PRE-RELEASE-INFO.txt @@ -6,7 +6,12 @@ databases, and was not forced on the databases that had already existed. If your current database was originally created by version 4.20 or newer, you don't need to do anything. -If you have an older database, please run the following script: +If you do have an older database, it MUST BE RE-CHECKED for any +existing duplicates before the next release (5.6, presumably) can be +deployed. Hopefully there are no such inconsistencies in your +database, but if there are any, they will need to be resolved, or the +next version of the application WILL FAIL TO DEPLOY, with an error +message from FlyWay. Please run the following script: https://github.com/IQSS/dataverse/raw/develop/scripts/issues/7451/check_datafiles_7451.sh From 8b74857365a803c604343f7733f797bace646b56 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 25 Jun 2021 15:13:04 -0400 Subject: [PATCH 244/354] renamed the flyway script for the dvobject constraint (since it didn't make it into 5.5) (#7451) --- ...sql => V5.5.0.3__7451-unique-constraint-storageidentifier.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.4.0.1__7451-unique-constraint-storageidentifier.sql => V5.5.0.3__7451-unique-constraint-storageidentifier.sql} (100%) diff --git a/src/main/resources/db/migration/V5.4.0.1__7451-unique-constraint-storageidentifier.sql b/src/main/resources/db/migration/V5.5.0.3__7451-unique-constraint-storageidentifier.sql similarity index 100% rename from src/main/resources/db/migration/V5.4.0.1__7451-unique-constraint-storageidentifier.sql rename to src/main/resources/db/migration/V5.5.0.3__7451-unique-constraint-storageidentifier.sql From b723f37104138f34624b7723c0f5f143057ecc35 Mon Sep 17 00:00:00 2001 From: Thibault Coupin Date: Fri, 4 Jun 2021 20:05:38 +0200 Subject: [PATCH 245/354] Fix #7922 translation for subfield with controlled vocabullary --- src/main/java/edu/harvard/iq/dataverse/DatasetField.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index ef49c68834f..228cedd8663 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -241,7 +241,7 @@ public String getValue() { return datasetFieldValues.get(0).getValue(); } else if (controlledVocabularyValues != null && !controlledVocabularyValues.isEmpty()) { if (controlledVocabularyValues.get(0) != null){ - return controlledVocabularyValues.get(0).getStrValue(); + return controlledVocabularyValues.get(0).getLocaleStrValue(); } } return null; From b1f9016045c114200142d8c1a7466eda478dcf21 Mon Sep 17 00:00:00 2001 From: Robert Verkerk Date: Tue, 29 Jun 2021 13:08:23 +0200 Subject: [PATCH 246/354] Better check HandleAuthHandle for default value of null. If unequal to null then use it. --- .../java/edu/harvard/iq/dataverse/HandlenetServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java index 3881b691ed8..1a8ee8a85e8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java @@ -314,7 +314,7 @@ private String getAuthenticationHandle(DvObject dvObject){ private String getAuthenticationHandle(String handlePrefix) { logger.log(Level.FINE,"getAuthenticationHandle"); - if (systemConfig.getHandleAuthHandle()) { + if (systemConfig.getHandleAuthHandle()!=null) { return systemConfig.getHandleAuthHandle(); } else if (systemConfig.isIndependentHandleService()) { return handlePrefix + "/ADMIN"; From e808fb9119c267feb8455458a3188e4b5f9c6250 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 29 Jun 2021 10:02:41 -0400 Subject: [PATCH 247/354] disable if setting isn't set --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 8 ++++++++ src/main/webapp/dataset.xhtml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 8d310d20627..5097c942658 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -5115,6 +5115,14 @@ public boolean isAnonymizedPrivateUrl() { } } + public boolean isAnonymizedAccessEnabled() { + if (settingsWrapper.getValueForKey(SettingsServiceBean.Key.AnonymizedFieldTypeNames) != null) { + return true; + } else { + return false; + } + } + // todo: we should be able to remove - this is passed in the html pages to other fragments, but they could just access this service bean directly. public FileDownloadServiceBean getFileDownloadService() { return fileDownloadService; diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 61ecac4565f..9b0404078b1 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1011,7 +1011,7 @@
    - From 3ec522307f2265e151c8c11a4dbb095eeef3d02e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 29 Jun 2021 10:38:19 -0400 Subject: [PATCH 248/354] docs, disallow anonymizedaccess param when seting not set --- doc/sphinx-guides/source/api/native-api.rst | 8 ++++++++ doc/sphinx-guides/source/installation/config.rst | 2 ++ src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 3a3f2fe5cec..d10096a3358 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1209,6 +1209,14 @@ The fully expanded example above (without environment variables) looks like this .. code-block:: bash curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/datasets/24/privateUrl + +If Anonymized Access has been enabled on a Dataverse instance (see the :ref:`:.AnonymizedFieldTypeNames` setting), an optional 'anonymizedAccess' query parameter is allowed. +Setting anonymizedAccess=true in your call will create a PrivateURL that only allows an anonymized view of the Dataset (see :ref:`privateurl`). + +.. code-block:: bash + + curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/datasets/24/privateUrl?anonymizedAccess=true + Get the Private URL for a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 03d6ad9235b..50e21227d04 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2242,3 +2242,5 @@ In the DDI metadata exports, the default behavior is to always add the repositor A comma-separated list of field type names that should be 'withheld' when dataset access occurs via a Private Url with Anonymized Access (e.g. to support anonymized review). A suggested minimum includes author, datasetContact, and contributor, but additional fields such as depositor, grantNumber, and publication might also need to be included. + +``curl -X PUT -d 'author, datasetContact, contributor, depositor, grantNumber, publication' http://localhost:8080/api/admin/settings/:AnonymizedFieldTypeNames`` \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 52636de5976..c1359e1c366 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -141,6 +141,7 @@ import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; +import javax.ws.rs.NotAcceptableException; import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; @@ -1257,6 +1258,9 @@ public Response getPrivateUrlData(@PathParam("id") String idSupplied) { @POST @Path("{id}/privateUrl") public Response createPrivateUrl(@PathParam("id") String idSupplied,@DefaultValue("false") @QueryParam ("anonymizedAccess") boolean anonymizedAccess) { + if(anonymizedAccess && settingsSvc.getValueForKey(SettingsServiceBean.Key.AnonymizedFieldTypeNames)==null) { + throw new NotAcceptableException("Anonymized Access not enabled"); + } return response( req -> ok(json(execCommand( new CreatePrivateUrlCommand(req, findDatasetOrDie(idSupplied), anonymizedAccess))))); From 0c91a34c7ef872b1fc00a71ff3bd03e79c035aef Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 29 Jun 2021 11:02:15 -0400 Subject: [PATCH 249/354] bad cut/paste --- src/main/webapp/dataset.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 9b0404078b1..c4d02bff970 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1010,8 +1010,8 @@
    - - From 32213aa7b9da2597e2b5bfcb6868f1afd9bb323a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 29 Jun 2021 11:03:24 -0400 Subject: [PATCH 250/354] add label --- doc/sphinx-guides/source/installation/config.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 50e21227d04..a988e465987 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2237,6 +2237,8 @@ By default, the name of the root Dataverse collection is used as the 'brandname' In the DDI metadata exports, the default behavior is to always add the repository (using its brandname - the root collection name or the value of :ref:`:InstallationName <:InstallationName>`) to the stdyDscr/distStmt/distrbtr element. If this setting is true, this will only be done when a Distributor is not already defined in the Dataset metadata. (Note that, since metadata export files are cached, they will have to be reexported (see :doc:`/admin/metadataexport`) before they incorporate a change in this setting.) +.. _:AnonymizedFieldTypeNames: + :AnonymizedFieldTypeNames +++++++++++++++++++++++++ From a6f06cae871b2f26ad9e6ab3bbfa4e79fde853a2 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 29 Jun 2021 11:19:33 -0400 Subject: [PATCH 251/354] extra . --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index d10096a3358..327b26d1166 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1210,7 +1210,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/datasets/24/privateUrl -If Anonymized Access has been enabled on a Dataverse instance (see the :ref:`:.AnonymizedFieldTypeNames` setting), an optional 'anonymizedAccess' query parameter is allowed. +If Anonymized Access has been enabled on a Dataverse instance (see the :ref:`:AnonymizedFieldTypeNames` setting), an optional 'anonymizedAccess' query parameter is allowed. Setting anonymizedAccess=true in your call will create a PrivateURL that only allows an anonymized view of the Dataset (see :ref:`privateurl`). .. code-block:: bash From d2eddcdb7f13e865aeb177d20f28c099a5884588 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 29 Jun 2021 13:45:22 -0400 Subject: [PATCH 252/354] fix isSet method --- .../iq/dataverse/util/json/JSONLDUtil.java | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 09811410606..c09eef291fd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -5,11 +5,9 @@ import java.sql.Timestamp; import java.time.LocalDate; import java.time.LocalDateTime; -import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; -import java.util.Date; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -49,10 +47,8 @@ import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.TermsOfUseAndAccess.License; -import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.util.bagit.OREMap; -import edu.harvard.iq.dataverse.util.json.JsonLDTerm; public class JSONLDUtil { @@ -720,37 +716,37 @@ public static boolean isSet(TermsOfUseAndAccess terms, String semterm) { case "http://schema.org/license": return !terms.getLicense().equals(TermsOfUseAndAccess.License.NONE); case "https://dataverse.org/schema/core#termsOfUse": - return StringUtils.isBlank(terms.getTermsOfUse()); + return !StringUtils.isBlank(terms.getTermsOfUse()); case "https://dataverse.org/schema/core#confidentialityDeclaration": - return StringUtils.isBlank(terms.getConfidentialityDeclaration()); + return !StringUtils.isBlank(terms.getConfidentialityDeclaration()); case "https://dataverse.org/schema/core#specialPermissions": - return StringUtils.isBlank(terms.getSpecialPermissions()); + return !StringUtils.isBlank(terms.getSpecialPermissions()); case "https://dataverse.org/schema/core#restrictions": - return StringUtils.isBlank(terms.getRestrictions()); + return !StringUtils.isBlank(terms.getRestrictions()); case "https://dataverse.org/schema/core#citationRequirements": - return StringUtils.isBlank(terms.getCitationRequirements()); + return !StringUtils.isBlank(terms.getCitationRequirements()); case "https://dataverse.org/schema/core#depositorRequirements": - return StringUtils.isBlank(terms.getDepositorRequirements()); + return !StringUtils.isBlank(terms.getDepositorRequirements()); case "https://dataverse.org/schema/core#conditions": - return StringUtils.isBlank(terms.getConditions()); + return !StringUtils.isBlank(terms.getConditions()); case "https://dataverse.org/schema/core#disclaimer": - return StringUtils.isBlank(terms.getDisclaimer()); + return !StringUtils.isBlank(terms.getDisclaimer()); case "https://dataverse.org/schema/core#termsOfAccess": - return StringUtils.isBlank(terms.getTermsOfAccess()); + return !StringUtils.isBlank(terms.getTermsOfAccess()); case "https://dataverse.org/schema/core#fileRequestAccess": return !terms.isFileAccessRequest(); case "https://dataverse.org/schema/core#dataAccessPlace": - return StringUtils.isBlank(terms.getDataAccessPlace()); + return !StringUtils.isBlank(terms.getDataAccessPlace()); case "https://dataverse.org/schema/core#originalArchive": - return StringUtils.isBlank(terms.getOriginalArchive()); + return !StringUtils.isBlank(terms.getOriginalArchive()); case "https://dataverse.org/schema/core#availabilityStatus": - return StringUtils.isBlank(terms.getAvailabilityStatus()); + return !StringUtils.isBlank(terms.getAvailabilityStatus()); case "https://dataverse.org/schema/core#contactForAccess": - return StringUtils.isBlank(terms.getContactForAccess()); + return !StringUtils.isBlank(terms.getContactForAccess()); case "https://dataverse.org/schema/core#sizeOfCollection": - return StringUtils.isBlank(terms.getSizeOfCollection()); + return !StringUtils.isBlank(terms.getSizeOfCollection()); case "https://dataverse.org/schema/core#studyCompletion": - return StringUtils.isBlank(terms.getStudyCompletion()); + return !StringUtils.isBlank(terms.getStudyCompletion()); default: logger.warning("isSet called for " + semterm); return false; From 4cd26692ad13fdb788b29ed603993e5a027199d4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 29 Jun 2021 13:45:46 -0400 Subject: [PATCH 253/354] update tests, incr. titanium --- pom.xml | 16 +++++- .../search/tests/data/dataset-finch1.jsonld | 25 ++++++--- .../harvard/iq/dataverse/api/DatasetsIT.java | 51 +++++++++++++++---- .../edu/harvard/iq/dataverse/api/UtilIT.java | 9 ++-- 4 files changed, 76 insertions(+), 25 deletions(-) diff --git a/pom.xml b/pom.xml index 354c12b8b49..f14e7592053 100644 --- a/pom.xml +++ b/pom.xml @@ -161,9 +161,21 @@ org.glassfish javax.json - 1.0.4 + 1.1.4 test + + org.skyscreamer + jsonassert + 1.5.0 + test + + + com.vaadin.external.google + android-json + + + org.apache.httpcomponents httpclient @@ -209,7 +221,7 @@ com.apicatalog titanium-json-ld - 0.8.4 + 0.8.6 diff --git a/scripts/search/tests/data/dataset-finch1.jsonld b/scripts/search/tests/data/dataset-finch1.jsonld index 24d7a1b2c9f..be39c9f14b2 100644 --- a/scripts/search/tests/data/dataset-finch1.jsonld +++ b/scripts/search/tests/data/dataset-finch1.jsonld @@ -1,15 +1,26 @@ + { "http://purl.org/dc/terms/title": "Darwin's Finches", + "http://purl.org/dc/terms/subject": "Medicine, Health and Life Sciences", "http://purl.org/dc/terms/creator": { "https://dataverse.org/schema/citation/author#Name": "Finch, Fiona", "https://dataverse.org/schema/citation/author#Affiliation": "Birds Inc." - } + }, "https://dataverse.org/schema/citation/Contact": { - "https://dataverse.org/schema/citation/datasetContact#E-mail": "finch@mailinator.com". - "https://dataverse.org/schema/citation/datasetContact#Name": "Finch, Fiona", - } + "https://dataverse.org/schema/citation/datasetContact#E-mail": "finch@mailinator.com", + "https://dataverse.org/schema/citation/datasetContact#Name": "Finch, Fiona" + }, "https://dataverse.org/schema/citation/Description": { "https://dataverse.org/schema/citation/dsDescription#Text": "Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds." - } - "http://purl.org/dc/terms/subject": "Medicine, Health and Life Sciences" -} + }, + "@type": [ + "http://www.openarchives.org/ore/terms/Aggregation", + "http://schema.org/Dataset" + ], + "http://schema.org/version": "DRAFT", + "http://schema.org/name": "Darwin's Finches", + "https://dataverse.org/schema/core#fileTermsOfAccess": { + "https://dataverse.org/schema/core#fileRequestAccess": false + }, + "http://schema.org/includedInDataCatalog": "Root" +} \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 5943650e802..e725864e851 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -7,6 +7,7 @@ import java.util.logging.Logger; import org.junit.BeforeClass; import org.junit.Test; +import org.skyscreamer.jsonassert.JSONAssert; import org.junit.Ignore; import com.jayway.restassured.path.json.JsonPath; @@ -27,6 +28,8 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import java.util.UUID; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; + import com.jayway.restassured.parsing.Parser; import static com.jayway.restassured.path.json.JsonPath.with; import com.jayway.restassured.path.xml.XmlPath; @@ -40,6 +43,8 @@ import java.io.File; import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -47,6 +52,9 @@ import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonObjectBuilder; +import javax.json.JsonWriter; +import javax.json.JsonWriterFactory; +import javax.json.stream.JsonGenerator; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; import javax.persistence.Persistence; @@ -2169,7 +2177,6 @@ public void testRestrictFileExportDdi() throws IOException { @Test public void testSemanticMetadataAPIs() { - Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); String username = UtilIT.getUsernameFromResponse(createUser); @@ -2179,7 +2186,6 @@ public void testSemanticMetadataAPIs() { createDataverseResponse.prettyPrint(); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - //Create a dataset using native api Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); createDatasetResponse.prettyPrint(); @@ -2188,16 +2194,32 @@ public void testSemanticMetadataAPIs() { //Get the metadata with the semantic api Response response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); response.then().assertThat().statusCode(OK.getStatusCode()); - //Compare the metadata with an expected value - the metadatablock entries should be the same but there will be additional fields with values related to the dataset's creation (e.g. new id) - String jsonLDString = JsonPath.from(response.getBody().asString()).getString("data"); - String jsonLD = JSONLDUtil.decontextualizeJsonLD(jsonLDString).toString(); + String jsonLDString = getData(response.getBody().asString()); + JsonObject jo=null; + try { + jo = JSONLDUtil.decontextualizeJsonLD(jsonLDString); + } catch (NoSuchMethodError e) { + logger.info(ExceptionUtils.getStackTrace(e)); + } + String jsonLD = jo.toString(); String expectedJsonLD = UtilIT.getDatasetJson("scripts/search/tests/data/dataset-finch1.jsonld"); + jo = Json.createObjectBuilder(jo).remove("@id").remove("http://schema.org/dateModified").build(); + try (StringWriter sw = new StringWriter()) { + Map map = new HashMap<>(); + map.put(JsonGenerator.PRETTY_PRINTING, true); + JsonWriterFactory writerFactory = Json.createWriterFactory(map); + JsonWriter jsonWriter = writerFactory.createWriter(sw); + jsonWriter.writeObject(jo); + jsonWriter.close(); + jsonLD=sw.toString(); + }catch(Exception e) { + e.printStackTrace(); + } //ToDo: Are the static pars as expected - assertEquals(expectedJsonLD, jsonLD); - + JSONAssert.assertEquals(expectedJsonLD, jsonLD, false); //Now change the title response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, "{\"Title\": \"New Title\", \"@context\":{\"Title\": \"http://purl.org/dc/terms/title\"}}", true); @@ -2207,7 +2229,7 @@ public void testSemanticMetadataAPIs() { response.then().assertThat().statusCode(OK.getStatusCode()); //Check that the semantic api returns the new title - jsonLDString = JsonPath.from(response.getBody().asString()).getString("data"); + jsonLDString = getData(response.getBody().asString()); JsonObject jsonLDObject = JSONLDUtil.decontextualizeJsonLD(jsonLDString); assertEquals("New Title", jsonLDObject.getString("http://purl.org/dc/terms/title")); @@ -2221,7 +2243,7 @@ public void testSemanticMetadataAPIs() { response.then().assertThat().statusCode(OK.getStatusCode()); //Look for a second description - jsonLDString = JsonPath.from(response.getBody().asString()).getString("data"); + jsonLDString = getData(response.getBody().asString()); jsonLDObject = JSONLDUtil.decontextualizeJsonLD(jsonLDString); assertEquals("New description", ((JsonObject) jsonLDObject.getJsonArray("https://dataverse.org/schema/citation/Description").get(1)) @@ -2232,7 +2254,7 @@ public void testSemanticMetadataAPIs() { response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, badTerms, false); response.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); - + //Delete the terms of use response = UtilIT.deleteDatasetJsonLDMetadata(datasetId, apiToken, "{\"https://dataverse.org/schema/core#termsOfUse\": \"New terms\"}"); @@ -2242,7 +2264,7 @@ public void testSemanticMetadataAPIs() { response.then().assertThat().statusCode(OK.getStatusCode()); //Verify that they're gone - jsonLDString = JsonPath.from(response.getBody().asString()).getString("data"); + jsonLDString = getData(response.getBody().asString()); jsonLDObject = JSONLDUtil.decontextualizeJsonLD(jsonLDString); assertTrue(!jsonLDObject.containsKey("https://dataverse.org/schema/core#termsOfUse")); @@ -2260,6 +2282,7 @@ public void testSemanticMetadataAPIs() { assertEquals(200, deleteUserResponse.getStatusCode()); } + @Test public void testReCreateDataset() { @@ -2318,4 +2341,10 @@ public void testReCreateDataset() { deleteUserResponse.prettyPrint(); assertEquals(200, deleteUserResponse.getStatusCode()); } + + private String getData(String body) { + try (StringReader rdr = new StringReader(body)) { + return Json.createReader(rdr).readObject().getJsonObject("data").toString(); + } + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index e7565b71adc..2a39069da49 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -11,6 +11,7 @@ import javax.json.JsonObject; import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.logging.Level; @@ -31,7 +32,6 @@ import java.nio.file.Path; import java.util.ArrayList; import org.apache.commons.lang3.math.NumberUtils; -import org.apache.cxf.headers.Header; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; @@ -39,7 +39,6 @@ import static com.jayway.restassured.RestAssured.given; import edu.harvard.iq.dataverse.util.StringUtil; import java.io.StringReader; -import javax.json.JsonArray; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -2641,7 +2640,7 @@ static Response updateDatasetJsonLDMetadata(Integer datasetId, String apiToken, Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) .contentType("application/ld+json") - .body(jsonLDBody) + .body(jsonLDBody.getBytes(StandardCharsets.UTF_8)) .put("/api/datasets/" + datasetId + "/metadata?replace=" + replace); return response; } @@ -2650,7 +2649,7 @@ static Response deleteDatasetJsonLDMetadata(Integer datasetId, String apiToken, Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) .contentType("application/ld+json") - .body(jsonLDBody) + .body(jsonLDBody.getBytes(StandardCharsets.UTF_8)) .put("/api/datasets/" + datasetId + "/metadata/delete"); return response; } @@ -2659,7 +2658,7 @@ public static Response recreateDatasetJsonLD(Integer datasetId, String apiToken, Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) .contentType("application/ld+json") - .body(jsonLDBody) + .body(jsonLDBody.getBytes(StandardCharsets.UTF_8)) .post("https://demo.dataverse.org/api/dataverses/" + dataverseAlias +"/datasets"); return response; } From 21829ec3565c49905e30237ceb375ed1a6019985 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 29 Jun 2021 17:30:07 -0400 Subject: [PATCH 254/354] Bug fixes/cleanup --- .../iq/dataverse/util/json/JSONLDUtil.java | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index c09eef291fd..974cb4f6670 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -25,6 +25,7 @@ import javax.json.JsonValue; import javax.json.JsonWriter; import javax.json.JsonWriterFactory; +import javax.json.JsonValue.ValueType; import javax.json.stream.JsonGenerator; import javax.ws.rs.BadRequestException; @@ -132,6 +133,9 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv, JsonObject jsonld, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { + //Assume draft to start + dsv.setVersionState(VersionState.DRAFT); + populateFieldTypeMap(metadataBlockSvc); // get existing ones? @@ -337,6 +341,7 @@ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv } } else { // Internal/non-metadatablock terms + boolean found=false; if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())) { if(jsonld.getString(key).equals(TermsOfUseAndAccess.CC0_URI)) { setSemTerm(terms, key, TermsOfUseAndAccess.License.NONE); @@ -344,11 +349,13 @@ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv throw new BadRequestException( "Term: " + key + " with value: " + jsonld.getString(key) + " not found."); } + found=true; } else if (datasetTerms.contains(key)) { if(!deleteIfSemTermMatches(terms, key, jsonld.get(key))) { throw new BadRequestException( "Term: " + key + " with value: " + jsonld.getString(key) + " not found."); } + found=true; } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); for (String fileKey : fAccessObject.keySet()) { @@ -357,9 +364,10 @@ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv throw new BadRequestException( "Term: " + key + " with value: " + jsonld.getString(key) + " not found."); } + found=true; } } - } else { + } else if(!found) { throw new BadRequestException( "Term: " + key + " not found."); } @@ -820,58 +828,62 @@ public static void setSemTerm(TermsOfUseAndAccess terms, String semterm, Object private static boolean deleteIfSemTermMatches(TermsOfUseAndAccess terms, String semterm, JsonValue jsonValue) { boolean foundTerm=false; + String val = null; + if(jsonValue.getValueType().equals(ValueType.STRING)) { + val = ((JsonString)jsonValue).getString(); + } switch (semterm) { case "https://dataverse.org/schema/core#termsOfUse": - if(terms.getTermsOfUse().equals(jsonValue.toString())) { - terms.setTermsOfAccess(null); + if(terms.getTermsOfUse().equals(val)) { + terms.setTermsOfUse(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#confidentialityDeclaration": - if(terms.getConfidentialityDeclaration().equals(jsonValue.toString())) { + if(terms.getConfidentialityDeclaration().equals(val)) { terms.setConfidentialityDeclaration(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#specialPermissions": - if(terms.getSpecialPermissions().equals(jsonValue.toString())) { + if(terms.getSpecialPermissions().equals(val)) { terms.setSpecialPermissions(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#restrictions": - if(terms.getRestrictions().equals(jsonValue.toString())) { + if(terms.getRestrictions().equals(val)) { terms.setRestrictions(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#citationRequirements": - if(terms.getCitationRequirements().equals(jsonValue.toString())) { + if(terms.getCitationRequirements().equals(val)) { terms.setCitationRequirements(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#depositorRequirements": - if(terms.getDepositorRequirements().equals(jsonValue.toString())) { + if(terms.getDepositorRequirements().equals(val)) { terms.setDepositorRequirements(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#conditions": - if(terms.getConditions().equals(jsonValue.toString())) { + if(terms.getConditions().equals(val)) { terms.setConditions(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#disclaimer": - if(terms.getDisclaimer().equals(jsonValue.toString())) { + if(terms.getDisclaimer().equals(val)) { terms.setDisclaimer(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#termsOfAccess": - if(terms.getTermsOfAccess().equals(jsonValue.toString())) { + if(terms.getTermsOfAccess().equals(val)) { terms.setTermsOfAccess(null); foundTerm=true; } @@ -883,37 +895,37 @@ private static boolean deleteIfSemTermMatches(TermsOfUseAndAccess terms, String } break; case "https://dataverse.org/schema/core#dataAccessPlace": - if(terms.getDataAccessPlace().equals(jsonValue.toString())) { + if(terms.getDataAccessPlace().equals(val)) { terms.setDataAccessPlace(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#originalArchive": - if(terms.getOriginalArchive().equals(jsonValue.toString())) { + if(terms.getOriginalArchive().equals(val)) { terms.setOriginalArchive(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#availabilityStatus": - if(terms.getAvailabilityStatus().equals(jsonValue.toString())) { + if(terms.getAvailabilityStatus().equals(val)) { terms.setAvailabilityStatus(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#contactForAccess": - if(terms.getContactForAccess().equals(jsonValue.toString())) { + if(terms.getContactForAccess().equals(val)) { terms.setContactForAccess(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#sizeOfCollection": - if(terms.getSizeOfCollection().equals(jsonValue.toString())) { + if(terms.getSizeOfCollection().equals(val)) { terms.setSizeOfCollection(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#studyCompletion": - if(terms.getStudyCompletion().equals(jsonValue.toString())) { + if(terms.getStudyCompletion().equals(val)) { terms.setStudyCompletion(null); foundTerm=true; } From 5f53e158462feb04c1c53bf4d17aed214733bc48 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 29 Jun 2021 17:30:19 -0400 Subject: [PATCH 255/354] finish tests --- .../harvard/iq/dataverse/api/DatasetsIT.java | 103 ++++++++---------- .../edu/harvard/iq/dataverse/api/UtilIT.java | 6 +- 2 files changed, 47 insertions(+), 62 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index e725864e851..f00bcd59119 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -33,9 +33,7 @@ import com.jayway.restassured.parsing.Parser; import static com.jayway.restassured.path.json.JsonPath.with; import com.jayway.restassured.path.xml.XmlPath; -import edu.harvard.iq.dataverse.Dataset; import static edu.harvard.iq.dataverse.api.UtilIT.equalToCI; -import static edu.harvard.iq.dataverse.authorization.AuthenticationResponse.Status.ERROR; import edu.harvard.iq.dataverse.authorization.groups.impl.builtin.AuthenticatedUsers; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -44,7 +42,6 @@ import java.io.File; import java.io.IOException; import java.io.StringReader; -import java.io.StringWriter; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -52,29 +49,15 @@ import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonObjectBuilder; -import javax.json.JsonWriter; -import javax.json.JsonWriterFactory; -import javax.json.stream.JsonGenerator; -import javax.persistence.EntityManager; -import javax.persistence.EntityManagerFactory; -import javax.persistence.Persistence; -import javax.persistence.PersistenceContext; -import static javax.ws.rs.core.Response.Status.BAD_REQUEST; -import static javax.ws.rs.core.Response.Status.CREATED; -import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.NO_CONTENT; -import static javax.ws.rs.core.Response.Status.OK; -import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; import static org.junit.Assert.assertEquals; import org.hamcrest.CoreMatchers; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; -import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import org.junit.Before; import static org.junit.matchers.JUnitMatchers.containsString; public class DatasetsIT { @@ -2186,41 +2169,33 @@ public void testSemanticMetadataAPIs() { createDataverseResponse.prettyPrint(); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - //Create a dataset using native api + // Create a dataset using native api Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); createDatasetResponse.prettyPrint(); Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); - //Get the metadata with the semantic api + // Get the metadata with the semantic api Response response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); response.then().assertThat().statusCode(OK.getStatusCode()); - //Compare the metadata with an expected value - the metadatablock entries should be the same but there will be additional fields with values related to the dataset's creation (e.g. new id) + // Compare the metadata with an expected value - the metadatablock entries + // should be the same but there will be additional fields with values related to + // the dataset's creation (e.g. new id) String jsonLDString = getData(response.getBody().asString()); - JsonObject jo=null; + JsonObject jo = null; try { jo = JSONLDUtil.decontextualizeJsonLD(jsonLDString); } catch (NoSuchMethodError e) { logger.info(ExceptionUtils.getStackTrace(e)); } - String jsonLD = jo.toString(); + String expectedJsonLD = UtilIT.getDatasetJson("scripts/search/tests/data/dataset-finch1.jsonld"); jo = Json.createObjectBuilder(jo).remove("@id").remove("http://schema.org/dateModified").build(); - try (StringWriter sw = new StringWriter()) { - Map map = new HashMap<>(); - map.put(JsonGenerator.PRETTY_PRINTING, true); - JsonWriterFactory writerFactory = Json.createWriterFactory(map); - JsonWriter jsonWriter = writerFactory.createWriter(sw); - jsonWriter.writeObject(jo); - jsonWriter.close(); - jsonLD=sw.toString(); - }catch(Exception e) { - e.printStackTrace(); - } + String jsonLD = jo.toString(); - //ToDo: Are the static pars as expected + // ToDo: Are the static pars as expected JSONAssert.assertEquals(expectedJsonLD, jsonLD, false); - //Now change the title + // Now change the title response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, "{\"Title\": \"New Title\", \"@context\":{\"Title\": \"http://purl.org/dc/terms/title\"}}", true); response.then().assertThat().statusCode(OK.getStatusCode()); @@ -2228,13 +2203,14 @@ public void testSemanticMetadataAPIs() { response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); response.then().assertThat().statusCode(OK.getStatusCode()); - //Check that the semantic api returns the new title + // Check that the semantic api returns the new title jsonLDString = getData(response.getBody().asString()); JsonObject jsonLDObject = JSONLDUtil.decontextualizeJsonLD(jsonLDString); assertEquals("New Title", jsonLDObject.getString("http://purl.org/dc/terms/title")); - //Add an additional description (which is multi-valued and compound) - //Also add new terms of use (single value so would fail with replace false if a value existed) + // Add an additional description (which is multi-valued and compound) + // Also add new terms of use (single value so would fail with replace false if a + // value existed) String newDescription = "{\"citation:Description\": {\"dsDescription:Text\": \"New description\", \"https://dataverse.org/schema/core#termsOfUse\": \"New terms\"} \"@context\":{\"citation\": \"https://dataverse.org/schema/citation/\",\"dsDescription\": \"https://dataverse.org/schema/citation/dsDescription#\"}}"; response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, newDescription, false); response.then().assertThat().statusCode(OK.getStatusCode()); @@ -2242,20 +2218,20 @@ public void testSemanticMetadataAPIs() { response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); response.then().assertThat().statusCode(OK.getStatusCode()); - //Look for a second description + // Look for a second description jsonLDString = getData(response.getBody().asString()); jsonLDObject = JSONLDUtil.decontextualizeJsonLD(jsonLDString); assertEquals("New description", ((JsonObject) jsonLDObject.getJsonArray("https://dataverse.org/schema/citation/Description").get(1)) .getString("https://dataverse.org/schema/citation/dsDescription#Text")); - //Can't add terms of use with replace=false and a value already set (single valued field) + // Can't add terms of use with replace=false and a value already set (single + // valued field) String badTerms = "{\"https://dataverse.org/schema/core#termsOfUse\": \"Bad terms\"}}"; response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, badTerms, false); response.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); - - //Delete the terms of use + // Delete the terms of use response = UtilIT.deleteDatasetJsonLDMetadata(datasetId, apiToken, "{\"https://dataverse.org/schema/core#termsOfUse\": \"New terms\"}"); response.then().assertThat().statusCode(OK.getStatusCode()); @@ -2263,12 +2239,12 @@ public void testSemanticMetadataAPIs() { response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); response.then().assertThat().statusCode(OK.getStatusCode()); - //Verify that they're gone + // Verify that they're gone jsonLDString = getData(response.getBody().asString()); jsonLDObject = JSONLDUtil.decontextualizeJsonLD(jsonLDString); assertTrue(!jsonLDObject.containsKey("https://dataverse.org/schema/core#termsOfUse")); - //Cleanup - delete dataset, dataverse, user... + // Cleanup - delete dataset, dataverse, user... Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); deleteDatasetResponse.prettyPrint(); assertEquals(200, deleteDatasetResponse.getStatusCode()); @@ -2282,7 +2258,7 @@ public void testSemanticMetadataAPIs() { assertEquals(200, deleteUserResponse.getStatusCode()); } - + @Test public void testReCreateDataset() { @@ -2297,38 +2273,47 @@ public void testReCreateDataset() { createDataverseResponse.prettyPrint(); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - //Create a dataset using native API + // Create a dataset using native API Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); createDatasetResponse.prettyPrint(); Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); - //Get the semantic metadata + // Get the semantic metadata Response response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); response.then().assertThat().statusCode(OK.getStatusCode()); - String expectedJsonLD = JsonPath.from(response.getBody().asString()).getString("data"); + String expectedString = JsonPath.from(response.getBody().asString()).getString("data"); - //Delete the dataset via native API + // Delete the dataset via native API Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); deleteDatasetResponse.prettyPrint(); assertEquals(200, deleteDatasetResponse.getStatusCode()); - //Now use the migrate API to recreate the dataset - response = UtilIT.recreateDatasetJsonLD(datasetId, apiToken, dataverseAlias, expectedJsonLD); + // Now use the migrate API to recreate the dataset + // Now use the migrate API to recreate the dataset + response = UtilIT.recreateDatasetJsonLD(apiToken, dataverseAlias, expectedString); + String body = response.getBody().asString(); response.then().assertThat().statusCode(CREATED.getStatusCode()); - //Id will change but everything else should be as before (DOI and create date) - datasetId = JsonPath.from(response.getBody().asString()).getInt("data/id"); + try (StringReader rdr = new StringReader(body)) { + datasetId = Json.createReader(rdr).readObject().getJsonObject("data").getInt("id"); + } + // Get the jsonLD metadata for what we recreated response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); response.then().assertThat().statusCode(OK.getStatusCode()); - String jsonLDString = JsonPath.from(response.getBody().asString()).getString("data"); + + String jsonLDString = getData(response.getBody().asString()); JsonObject jsonLD = JSONLDUtil.decontextualizeJsonLD(jsonLDString); - - //ToDo: Assert that the semantic api response is the same except for the id - assertEquals(expectedJsonLD, jsonLD.toString()); + JsonObject expectedJsonLD = JSONLDUtil.decontextualizeJsonLD(expectedString); + expectedJsonLD = Json.createObjectBuilder(expectedJsonLD).remove("@id").remove("http://schema.org/dateModified") + .build(); + // ToDo: Assert that the semantic api response is the same (everything in the + // expected version is in the new one - deleting the @id and dateModified means + // those won't be compared (with last param = false) + JSONAssert.assertEquals(expectedJsonLD.toString(), jsonLD.toString(), false); - //Cleanup by deleting things + // Now cleanup deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); deleteDatasetResponse.prettyPrint(); assertEquals(200, deleteDatasetResponse.getStatusCode()); @@ -2341,7 +2326,7 @@ public void testReCreateDataset() { deleteUserResponse.prettyPrint(); assertEquals(200, deleteUserResponse.getStatusCode()); } - + private String getData(String body) { try (StringReader rdr = new StringReader(body)) { return Json.createReader(rdr).readObject().getJsonObject("data").toString(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 2a39069da49..5dccd6c0d5f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2654,12 +2654,12 @@ static Response deleteDatasetJsonLDMetadata(Integer datasetId, String apiToken, return response; } - public static Response recreateDatasetJsonLD(Integer datasetId, String apiToken, String dataverseAlias, String jsonLDBody) { + public static Response recreateDatasetJsonLD(String apiToken, String dataverseAlias, String jsonLDBody) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) - .contentType("application/ld+json") + .contentType("application/ld+json; charset=utf-8") .body(jsonLDBody.getBytes(StandardCharsets.UTF_8)) - .post("https://demo.dataverse.org/api/dataverses/" + dataverseAlias +"/datasets"); + .post("/api/dataverses/" + dataverseAlias +"/datasets"); return response; } } From 8eb5fa04da4f61cf082f74e7e11329ab84194463 Mon Sep 17 00:00:00 2001 From: Robert Verkerk Date: Wed, 30 Jun 2021 10:54:48 +0200 Subject: [PATCH 256/354] Change "string" to "String" with uppercase first character --- src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 7dd2248eafe..4fd4358832c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -1045,8 +1045,8 @@ public boolean isIndependentHandleService() { } - public string getHandleAuthHandle() { - string HandleAuthHandle = settingsService.getValueForKey(SettingsServiceBean.Key.HandleAuthHandle, null); + public String getHandleAuthHandle() { + String HandleAuthHandle = settingsService.getValueForKey(SettingsServiceBean.Key.HandleAuthHandle, null); return HandleAuthHandle; } From 1fc42ad5a57ca677a59858c43f2737fcfa5c36a1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 30 Jun 2021 11:43:47 +0200 Subject: [PATCH 257/354] build(ci): atempt to fix coveralls report. #7977 --- .github/workflows/maven_unit_test.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 916f0e7ecbb..464d60c2db6 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -22,7 +22,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - name: Set up JDK 11 + - name: Set up JDK ${{ matrix.jdk }} uses: actions/setup-java@v2 with: java-version: ${{ matrix.jdk }} @@ -38,8 +38,5 @@ jobs: - name: Maven Code Coverage env: CI_NAME: github - BRANCH_NAME_OR_REF: ${{ github.head_ref || github.ref }} - CI_BUILD_NUMBER: ${{ github.run_id }} - CI_BUILD_URL: https://github.com/${{ github.repository }}/commit/${{ github.event.after }}/checks COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }} - run: mvn -V -B jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} \ No newline at end of file + run: mvn -V -B jacoco:report coveralls:report -DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }} \ No newline at end of file From 4328f5bb73ce11e92bc245d773be604727e921ca Mon Sep 17 00:00:00 2001 From: Robert Verkerk Date: Wed, 30 Jun 2021 12:11:34 +0200 Subject: [PATCH 258/354] Change parameter "HandleAuthHandle" so it start with a lower case character when it is being assigned. --- src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 4fd4358832c..0764d07e8fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -1046,8 +1046,8 @@ public boolean isIndependentHandleService() { } public String getHandleAuthHandle() { - String HandleAuthHandle = settingsService.getValueForKey(SettingsServiceBean.Key.HandleAuthHandle, null); - return HandleAuthHandle; + String handleAuthHandle = settingsService.getValueForKey(SettingsServiceBean.Key.HandleAuthHandle, null); + return handleAuthHandle; } public String getMDCLogPath() { From bd65e1abaa159a7e840327a6e7ea431b03c02b31 Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Wed, 30 Jun 2021 11:01:50 -0400 Subject: [PATCH 259/354] Update 5.3-release-notes.md Removed incorrect statement: (If you are using a PostgreSQL server on `localhost:5432`, you can omit `dataverse.db.host` and `dataverse.db.port`.) --- doc/release-notes/5.3-release-notes.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/release-notes/5.3-release-notes.md b/doc/release-notes/5.3-release-notes.md index b78aa742f45..e62da980a3e 100644 --- a/doc/release-notes/5.3-release-notes.md +++ b/doc/release-notes/5.3-release-notes.md @@ -124,7 +124,6 @@ If you are still using pre-5.0 version of Dataverse, and Glassfish version 4, pl 3\. Update your database connection. Please configure your connection details, replacing all the `${DB_...}`. -(If you are using a PostgreSQL server on `localhost:5432`, you can omit `dataverse.db.host` and `dataverse.db.port`.) - `/bin/asadmin create-system-properties "dataverse.db.user=${DB_USER}"` - `/bin/asadmin create-system-properties "dataverse.db.host=${DB_HOST}"` From 82a5b23842a46009c98f5da4ef671e30c9e05c37 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 30 Jun 2021 11:58:49 -0400 Subject: [PATCH 260/354] sync with migration api branch (tests, docs, bug fixes) --- doc/release-notes/6497-semantic-api.md | 7 + .../dataset-semantic-metadata-api.rst | 91 +++++++++++++ doc/sphinx-guides/source/developers/index.rst | 1 + pom.xml | 16 ++- .../search/tests/data/dataset-finch1.jsonld | 26 ++++ .../harvard/iq/dataverse/api/Datasets.java | 8 +- .../iq/dataverse/util/json/JSONLDUtil.java | 116 ++++++++-------- .../harvard/iq/dataverse/api/DatasetsIT.java | 128 ++++++++++++++++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 28 +++- 9 files changed, 337 insertions(+), 84 deletions(-) create mode 100644 doc/release-notes/6497-semantic-api.md create mode 100644 doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst create mode 100644 scripts/search/tests/data/dataset-finch1.jsonld diff --git a/doc/release-notes/6497-semantic-api.md b/doc/release-notes/6497-semantic-api.md new file mode 100644 index 00000000000..58dcad44284 --- /dev/null +++ b/doc/release-notes/6497-semantic-api.md @@ -0,0 +1,7 @@ +# Release Highlights + +### Dataset Semantic API (Experimental) + +Dataset metadata can be retrieved/set/updated using a new, flatter JSONLD format - following the format of an OAI-ORE export (RDA-conformant Bags), allowing for easier transfer of metadata to/from other systems (i.e. without needing to know Dataverse's metadata block and field storage architecture). This new API also allows for the update of terms metadata (#5899). + +This development was supported by the [Research Data Alliance](https://rd-alliance.org), DANS, and Sciences PO and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). diff --git a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst new file mode 100644 index 00000000000..00876d0887a --- /dev/null +++ b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst @@ -0,0 +1,91 @@ +Dataset Semantic Metadata API +============================= + +The OAI_ORE metadata export format represents Dataset metadata using json-ld (see the :doc:`/admin/metadataexport` section). As part of an RDA-supported effort to allow import of Datasets exported as Bags with an included OAI_ORE metadata file, +an experimental API has been created that provides a json-ld alternative the the v1.0 API calls to get/set/delete Dataset metadata in the :doc:`/api/native-api`. + +You may prefer to work with this API if you are building a tool to import from a Bag/OAI-ORE source or already work with json-ld representations of metadata, or if you prefer the flatter json-ld representation to Dataverse software's json representation (which includes structure related to the metadata blocks involved and the type/multiplicity of the metadata fields.) +You may not want to use this API if you need stability and backward compatibility (the 'experimental' designation for this API implies that community feedback is desired and that, in future Dataverse software versions, the API may be modified based on that feedback). + +Note: The examples use the 'application/ld+json' mimetype. For compatibility reasons, the APIs also be used with mimetype "application/json-ld" + +Get Dataset Metadata +-------------------- + +To get the json-ld formatted metadata for a Dataset, specify the Dataset ID (DATASET_ID) or Persistent identifier (DATASET_PID), and, for specific versions, the version number. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export DATASET_ID='12345' + export DATASET_PID='doi:10.5072/FK2A1B2C3' + export VERSION='1.0' + export SERVER_URL=https://demo.dataverse.org + + Example 1: Get metadata for version '1.0' + + curl -H X-Dataverse-key:$API_TOKEN -H 'Accept: application/ld+json' "$SERVER_URL/api/datasets/$DATASET_ID/versions/$VERSION/metadata" + + Example 2: Get metadata for the latest version using the DATASET PID + + curl -H X-Dataverse-key:$API_TOKEN -H 'Accept: application/ld+json' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID" + +You should expect a 200 ("OK") response and JSON-LD mirroring the OAI-ORE representation in the returned 'data' object. + + +Add Dataset Metadata +-------------------- + +To add json-ld formatted metadata for a Dataset, specify the Dataset ID (DATASET_ID) or Persistent identifier (DATASET_PID). Adding '?replace=true' will overwrite an existing metadata value. The default (replace=false) will only add new metadata or add a new value to a multi-valued field. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export DATASET_ID='12345' + export DATASET_PID='doi:10.5072/FK2A1B2C3' + export VERSION='1.0' + export SERVER_URL=https://demo.dataverse.org + + Example: Change the Dataset title + + curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -d '{"Title": "Submit menu test", "@context":{"Title": "http://purl.org/dc/terms/title"}}' "$SERVER_URL/api/datasets/$DATASET_ID/metadata?replace=true" + + Example 2: Add a description using the DATASET PID + + curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -d '{"citation:Description": {"dsDescription:Text": "New description"}, "@context":{"citation": "https://dataverse.org/schema/citation/","dsDescription": "https://dataverse.org/schema/citation/dsDescription#"}}' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID" + +You should expect a 200 ("OK") response indicating whether a draft Dataset version was created or an existing draft was updated. + + +Delete Dataset Metadata +----------------------- + +To delete metadata for a Dataset, send a json-ld representation of the fields to delete and specify the Dataset ID (DATASET_ID) or Persistent identifier (DATASET_PID). + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export DATASET_ID='12345' + export DATASET_PID='doi:10.5072/FK2A1B2C3' + export VERSION='1.0' + export SERVER_URL=https://demo.dataverse.org + + Example: Delete the TermsOfUseAndAccess 'restrictions' value 'No restrictions' for the latest version using the DATASET PID + + curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -d '{"https://dataverse.org/schema/core#restrictions":"No restrictions"}' "$SERVER_URL/api/datasets/:persistentId/metadata/delete?persistentId=$DATASET_PID" + +Note, this example uses the term URI directly rather than adding an '@context' element. You can use either form in any of these API calls. + +You should expect a 200 ("OK") response indicating whether a draft Dataset version was created or an existing draft was updated. + + +Create a Dataset +---------------- + +Specifying the Content-Type as application/ld+json with the existing /api/dataverses/{id}/datasets API call (see :ref:`create-dataset-command`) supports using the same metadata format when creating a Dataset. + +With curl, this is done by adding the following header: + +.. code-block:: bash + + -H 'Content-Type: application/ld+json' diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst index eebfd50ba35..1e76171fc92 100755 --- a/doc/sphinx-guides/source/developers/index.rst +++ b/doc/sphinx-guides/source/developers/index.rst @@ -35,4 +35,5 @@ Developer Guide big-data-support aux-file-support s3-direct-upload-api + dataset-semantic-metadata-api workflows diff --git a/pom.xml b/pom.xml index 354c12b8b49..f14e7592053 100644 --- a/pom.xml +++ b/pom.xml @@ -161,9 +161,21 @@ org.glassfish javax.json - 1.0.4 + 1.1.4 test + + org.skyscreamer + jsonassert + 1.5.0 + test + + + com.vaadin.external.google + android-json + + + org.apache.httpcomponents httpclient @@ -209,7 +221,7 @@ com.apicatalog titanium-json-ld - 0.8.4 + 0.8.6 diff --git a/scripts/search/tests/data/dataset-finch1.jsonld b/scripts/search/tests/data/dataset-finch1.jsonld new file mode 100644 index 00000000000..be39c9f14b2 --- /dev/null +++ b/scripts/search/tests/data/dataset-finch1.jsonld @@ -0,0 +1,26 @@ + +{ + "http://purl.org/dc/terms/title": "Darwin's Finches", + "http://purl.org/dc/terms/subject": "Medicine, Health and Life Sciences", + "http://purl.org/dc/terms/creator": { + "https://dataverse.org/schema/citation/author#Name": "Finch, Fiona", + "https://dataverse.org/schema/citation/author#Affiliation": "Birds Inc." + }, + "https://dataverse.org/schema/citation/Contact": { + "https://dataverse.org/schema/citation/datasetContact#E-mail": "finch@mailinator.com", + "https://dataverse.org/schema/citation/datasetContact#Name": "Finch, Fiona" + }, + "https://dataverse.org/schema/citation/Description": { + "https://dataverse.org/schema/citation/dsDescription#Text": "Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds." + }, + "@type": [ + "http://www.openarchives.org/ore/terms/Aggregation", + "http://schema.org/Dataset" + ], + "http://schema.org/version": "DRAFT", + "http://schema.org/name": "Darwin's Finches", + "https://dataverse.org/schema/core#fileTermsOfAccess": { + "https://dataverse.org/schema/core#fileRequestAccess": false + }, + "http://schema.org/includedInDataCatalog": "Root" +} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 937b6542788..8fcdf9884ca 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -653,7 +653,7 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, @GET @Path("{id}/versions/{versionId}/metadata") - @Produces("application/json-ld") + @Produces("application/ld+json, application/json-ld") public Response getVersionJsonLDMetadata(@PathParam("id") String id, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) { try { DataverseRequest req = createDataverseRequest(findUserOrDie()); @@ -674,14 +674,14 @@ public Response getVersionJsonLDMetadata(@PathParam("id") String id, @PathParam( @GET @Path("{id}/metadata") - @Produces("application/json-ld") + @Produces("application/ld+json, application/json-ld") public Response getVersionJsonLDMetadata(@PathParam("id") String id, @Context UriInfo uriInfo, @Context HttpHeaders headers) { return getVersionJsonLDMetadata(id, ":draft", uriInfo, headers); } @PUT @Path("{id}/metadata") - @Consumes("application/json-ld") + @Consumes("application/ld+json, application/json-ld") public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String id, @DefaultValue("false") @QueryParam("replace") boolean replaceTerms) { try { @@ -711,7 +711,7 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String @PUT @Path("{id}/metadata/delete") - @Consumes("application/json-ld") + @Consumes("application/ld+json, application/json-ld") public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id) { logger.info("In delteMetadata"); try { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 79a34e6999a..4b5f5f74046 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -5,11 +5,9 @@ import java.sql.Timestamp; import java.time.LocalDate; import java.time.LocalDateTime; -import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; -import java.util.Date; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -27,6 +25,7 @@ import javax.json.JsonValue; import javax.json.JsonWriter; import javax.json.JsonWriterFactory; +import javax.json.JsonValue.ValueType; import javax.json.stream.JsonGenerator; import javax.ws.rs.BadRequestException; @@ -49,10 +48,8 @@ import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.TermsOfUseAndAccess.License; -import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.util.bagit.OREMap; -import edu.harvard.iq.dataverse.util.json.JsonLDTerm; public class JSONLDUtil { @@ -132,6 +129,9 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv, JsonObject jsonld, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc, boolean append, boolean migrating) { + //Assume draft to start + dsv.setVersionState(VersionState.DRAFT); + populateFieldTypeMap(metadataBlockSvc); // get existing ones? @@ -144,7 +144,8 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv } fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); } - TermsOfUseAndAccess terms = dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess(); + + TermsOfUseAndAccess terms = (dsv.getTermsOfUseAndAccess()!=null) ? dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess() : new TermsOfUseAndAccess(); for (String key : jsonld.keySet()) { if (!key.equals("@context")) { @@ -172,24 +173,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv addField(dsf, valArray, dsft, datasetFieldSvc, append); - // assemble new terms, add to existing - // multivalue? - // compound? - // merge with existing dv metadata - // dsfl.add(dsf); } else { - // Internal/non-metadatablock terms - // Add metadata related to the Dataset/DatasetVersion - - // ("@id", id) - check is equal to existing globalID? - // Add to 'md on original' ? - // (JsonLDTerm.schemaOrg("version").getLabel(), - // version.getFriendlyVersionNumber()) - // Citation metadata? - // (JsonLDTerm.schemaOrg("datePublished").getLabel(), - // dataset.getPublicationDateFormattedYYYYMMDD()) - // (JsonLDTerm.schemaOrg("name").getLabel()) - // (JsonLDTerm.schemaOrg("dateModified").getLabel()) if (key.equals(JsonLDTerm.schemaOrg("datePublished").getUrl())&& migrating && !append) { dsv.setVersionState(VersionState.RELEASED); @@ -201,6 +185,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv dsv.setMinorVersionNumber(Long.parseLong(friendlyVersion.substring(index + 1))); } } else if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())) { + //Special handling for license if (!append || !isSet(terms, key)) { // Mirror rules from SwordServiceBean if (jsonld.containsKey(JsonLDTerm.termsOfUse.getUrl())) { @@ -216,19 +201,19 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv } } else if (datasetTerms.contains(key)) { + // Other Dataset-level TermsOfUseAndAccess if (!append || !isSet(terms, key)) { - // Other Dataset-level TermsOfUseAndAccess setSemTerm(terms, key, jsonld.getString(key)); } else { throw new BadRequestException( "Can't append to a single-value field that already has a value: " + key); } } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { + // Other DataFile-level TermsOfUseAndAccess JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); for (String fileKey : fAccessObject.keySet()) { if (datafileTerms.contains(fileKey)) { if (!append || !isSet(terms, fileKey)) { - // Other DataFile-level TermsOfUseAndAccess if (fileKey.equals(JsonLDTerm.fileRequestAccess.getUrl())) { setSemTerm(terms, fileKey, fAccessObject.getBoolean(fileKey)); } else { @@ -242,6 +227,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv } } } else { + //Metadata block metadata fields if (dsftMap.containsKey(JsonLDTerm.metadataOnOrig.getUrl())) { DatasetFieldType dsft = dsftMap.get(JsonLDTerm.metadataOnOrig.getUrl()); @@ -282,10 +268,8 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv } } dsv.setTermsOfUseAndAccess(terms); - // move to new dataverse? - // aggBuilder.add(JsonLDTerm.schemaOrg("includedInDataCatalog").getLabel(), - // dataset.getDataverseContext().getDisplayName()); - + // ToDo: support Dataverse location metadata? e.g. move to new dataverse? + // re: JsonLDTerm.schemaOrg("includedInDataCatalog") } } @@ -307,7 +291,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv */ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv, String jsonLDBody, MetadataBlockServiceBean metadataBlockSvc, DatasetFieldServiceBean datasetFieldSvc) { -logger.info("deleteDatasetVersionMD"); + logger.fine("deleteDatasetVersionMD"); JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); //All terms are now URIs //Setup dsftMap - URI to datasetFieldType map @@ -341,7 +325,7 @@ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv // Todo - normalize object vs. array JsonArray valArray = getValues(jsonld.get(key), dsft.isAllowMultiples(), dsft.getName()); -logger.info("Deleting: " + key + " : " + valArray.toString()); + logger.fine("Deleting: " + key + " : " + valArray.toString()); DatasetField dsf2 = getReplacementField(dsf, valArray); if(dsf2 == null) { //Exact match - remove the field @@ -354,6 +338,7 @@ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv } } else { // Internal/non-metadatablock terms + boolean found=false; if (key.equals(JsonLDTerm.schemaOrg("license").getUrl())) { if(jsonld.getString(key).equals(TermsOfUseAndAccess.CC0_URI)) { setSemTerm(terms, key, TermsOfUseAndAccess.License.NONE); @@ -361,11 +346,13 @@ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv throw new BadRequestException( "Term: " + key + " with value: " + jsonld.getString(key) + " not found."); } + found=true; } else if (datasetTerms.contains(key)) { if(!deleteIfSemTermMatches(terms, key, jsonld.get(key))) { throw new BadRequestException( "Term: " + key + " with value: " + jsonld.getString(key) + " not found."); } + found=true; } else if (key.equals(JsonLDTerm.fileTermsOfAccess.getUrl())) { JsonObject fAccessObject = jsonld.getJsonObject(JsonLDTerm.fileTermsOfAccess.getUrl()); for (String fileKey : fAccessObject.keySet()) { @@ -374,9 +361,10 @@ public static DatasetVersion deleteDatasetVersionMDFromJsonLD(DatasetVersion dsv throw new BadRequestException( "Term: " + key + " with value: " + jsonld.getString(key) + " not found."); } + found=true; } } - } else { + } else if(!found) { throw new BadRequestException( "Term: " + key + " not found."); } @@ -733,37 +721,37 @@ public static boolean isSet(TermsOfUseAndAccess terms, String semterm) { case "http://schema.org/license": return !terms.getLicense().equals(TermsOfUseAndAccess.License.NONE); case "https://dataverse.org/schema/core#termsOfUse": - return StringUtils.isBlank(terms.getTermsOfUse()); + return !StringUtils.isBlank(terms.getTermsOfUse()); case "https://dataverse.org/schema/core#confidentialityDeclaration": - return StringUtils.isBlank(terms.getConfidentialityDeclaration()); + return !StringUtils.isBlank(terms.getConfidentialityDeclaration()); case "https://dataverse.org/schema/core#specialPermissions": - return StringUtils.isBlank(terms.getSpecialPermissions()); + return !StringUtils.isBlank(terms.getSpecialPermissions()); case "https://dataverse.org/schema/core#restrictions": - return StringUtils.isBlank(terms.getRestrictions()); + return !StringUtils.isBlank(terms.getRestrictions()); case "https://dataverse.org/schema/core#citationRequirements": - return StringUtils.isBlank(terms.getCitationRequirements()); + return !StringUtils.isBlank(terms.getCitationRequirements()); case "https://dataverse.org/schema/core#depositorRequirements": - return StringUtils.isBlank(terms.getDepositorRequirements()); + return !StringUtils.isBlank(terms.getDepositorRequirements()); case "https://dataverse.org/schema/core#conditions": - return StringUtils.isBlank(terms.getConditions()); + return !StringUtils.isBlank(terms.getConditions()); case "https://dataverse.org/schema/core#disclaimer": - return StringUtils.isBlank(terms.getDisclaimer()); + return !StringUtils.isBlank(terms.getDisclaimer()); case "https://dataverse.org/schema/core#termsOfAccess": - return StringUtils.isBlank(terms.getTermsOfAccess()); + return !StringUtils.isBlank(terms.getTermsOfAccess()); case "https://dataverse.org/schema/core#fileRequestAccess": return !terms.isFileAccessRequest(); case "https://dataverse.org/schema/core#dataAccessPlace": - return StringUtils.isBlank(terms.getDataAccessPlace()); + return !StringUtils.isBlank(terms.getDataAccessPlace()); case "https://dataverse.org/schema/core#originalArchive": - return StringUtils.isBlank(terms.getOriginalArchive()); + return !StringUtils.isBlank(terms.getOriginalArchive()); case "https://dataverse.org/schema/core#availabilityStatus": - return StringUtils.isBlank(terms.getAvailabilityStatus()); + return !StringUtils.isBlank(terms.getAvailabilityStatus()); case "https://dataverse.org/schema/core#contactForAccess": - return StringUtils.isBlank(terms.getContactForAccess()); + return !StringUtils.isBlank(terms.getContactForAccess()); case "https://dataverse.org/schema/core#sizeOfCollection": - return StringUtils.isBlank(terms.getSizeOfCollection()); + return !StringUtils.isBlank(terms.getSizeOfCollection()); case "https://dataverse.org/schema/core#studyCompletion": - return StringUtils.isBlank(terms.getStudyCompletion()); + return !StringUtils.isBlank(terms.getStudyCompletion()); default: logger.warning("isSet called for " + semterm); return false; @@ -837,58 +825,62 @@ public static void setSemTerm(TermsOfUseAndAccess terms, String semterm, Object private static boolean deleteIfSemTermMatches(TermsOfUseAndAccess terms, String semterm, JsonValue jsonValue) { boolean foundTerm=false; + String val = null; + if(jsonValue.getValueType().equals(ValueType.STRING)) { + val = ((JsonString)jsonValue).getString(); + } switch (semterm) { case "https://dataverse.org/schema/core#termsOfUse": - if(terms.getTermsOfUse().equals(jsonValue.toString())) { - terms.setTermsOfAccess(null); + if(terms.getTermsOfUse().equals(val)) { + terms.setTermsOfUse(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#confidentialityDeclaration": - if(terms.getConfidentialityDeclaration().equals(jsonValue.toString())) { + if(terms.getConfidentialityDeclaration().equals(val)) { terms.setConfidentialityDeclaration(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#specialPermissions": - if(terms.getSpecialPermissions().equals(jsonValue.toString())) { + if(terms.getSpecialPermissions().equals(val)) { terms.setSpecialPermissions(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#restrictions": - if(terms.getRestrictions().equals(jsonValue.toString())) { + if(terms.getRestrictions().equals(val)) { terms.setRestrictions(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#citationRequirements": - if(terms.getCitationRequirements().equals(jsonValue.toString())) { + if(terms.getCitationRequirements().equals(val)) { terms.setCitationRequirements(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#depositorRequirements": - if(terms.getDepositorRequirements().equals(jsonValue.toString())) { + if(terms.getDepositorRequirements().equals(val)) { terms.setDepositorRequirements(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#conditions": - if(terms.getConditions().equals(jsonValue.toString())) { + if(terms.getConditions().equals(val)) { terms.setConditions(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#disclaimer": - if(terms.getDisclaimer().equals(jsonValue.toString())) { + if(terms.getDisclaimer().equals(val)) { terms.setDisclaimer(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#termsOfAccess": - if(terms.getTermsOfAccess().equals(jsonValue.toString())) { + if(terms.getTermsOfAccess().equals(val)) { terms.setTermsOfAccess(null); foundTerm=true; } @@ -900,37 +892,37 @@ private static boolean deleteIfSemTermMatches(TermsOfUseAndAccess terms, String } break; case "https://dataverse.org/schema/core#dataAccessPlace": - if(terms.getDataAccessPlace().equals(jsonValue.toString())) { + if(terms.getDataAccessPlace().equals(val)) { terms.setDataAccessPlace(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#originalArchive": - if(terms.getOriginalArchive().equals(jsonValue.toString())) { + if(terms.getOriginalArchive().equals(val)) { terms.setOriginalArchive(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#availabilityStatus": - if(terms.getAvailabilityStatus().equals(jsonValue.toString())) { + if(terms.getAvailabilityStatus().equals(val)) { terms.setAvailabilityStatus(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#contactForAccess": - if(terms.getContactForAccess().equals(jsonValue.toString())) { + if(terms.getContactForAccess().equals(val)) { terms.setContactForAccess(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#sizeOfCollection": - if(terms.getSizeOfCollection().equals(jsonValue.toString())) { + if(terms.getSizeOfCollection().equals(val)) { terms.setSizeOfCollection(null); foundTerm=true; } break; case "https://dataverse.org/schema/core#studyCompletion": - if(terms.getStudyCompletion().equals(jsonValue.toString())) { + if(terms.getStudyCompletion().equals(val)) { terms.setStudyCompletion(null); foundTerm=true; } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index a7dc41d9c74..7074c55d64d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -7,6 +7,7 @@ import java.util.logging.Logger; import org.junit.BeforeClass; import org.junit.Test; +import org.skyscreamer.jsonassert.JSONAssert; import org.junit.Ignore; import com.jayway.restassured.path.json.JsonPath; @@ -27,17 +28,20 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import java.util.UUID; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; + import com.jayway.restassured.parsing.Parser; import static com.jayway.restassured.path.json.JsonPath.with; import com.jayway.restassured.path.xml.XmlPath; -import edu.harvard.iq.dataverse.Dataset; import static edu.harvard.iq.dataverse.api.UtilIT.equalToCI; -import static edu.harvard.iq.dataverse.authorization.AuthenticationResponse.Status.ERROR; import edu.harvard.iq.dataverse.authorization.groups.impl.builtin.AuthenticatedUsers; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.json.JSONLDUtil; + import java.io.File; import java.io.IOException; +import java.io.StringReader; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -45,26 +49,15 @@ import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonObjectBuilder; -import javax.persistence.EntityManager; -import javax.persistence.EntityManagerFactory; -import javax.persistence.Persistence; -import javax.persistence.PersistenceContext; -import static javax.ws.rs.core.Response.Status.BAD_REQUEST; -import static javax.ws.rs.core.Response.Status.CREATED; -import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.NO_CONTENT; -import static javax.ws.rs.core.Response.Status.OK; -import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; -import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertEquals; import org.hamcrest.CoreMatchers; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; -import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import org.junit.Before; import static org.junit.matchers.JUnitMatchers.containsString; public class DatasetsIT { @@ -2164,4 +2157,111 @@ public void testRestrictFileExportDdi() throws IOException { .body("message", equalTo("You do not have permission to download this file.")); } + @Test + public void testSemanticMetadataAPIs() { + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + // Create a dataset using native api + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + // Get the metadata with the semantic api + Response response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); + response.then().assertThat().statusCode(OK.getStatusCode()); + // Compare the metadata with an expected value - the metadatablock entries + // should be the same but there will be additional fields with values related to + // the dataset's creation (e.g. new id) + String jsonLDString = getData(response.getBody().asString()); + JsonObject jo = null; + try { + jo = JSONLDUtil.decontextualizeJsonLD(jsonLDString); + } catch (NoSuchMethodError e) { + logger.info(ExceptionUtils.getStackTrace(e)); + } + + + String expectedJsonLD = UtilIT.getDatasetJson("scripts/search/tests/data/dataset-finch1.jsonld"); + jo = Json.createObjectBuilder(jo).remove("@id").remove("http://schema.org/dateModified").build(); + String jsonLD = jo.toString(); + + // ToDo: Are the static pars as expected + JSONAssert.assertEquals(expectedJsonLD, jsonLD, false); + // Now change the title + response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, + "{\"Title\": \"New Title\", \"@context\":{\"Title\": \"http://purl.org/dc/terms/title\"}}", true); + response.then().assertThat().statusCode(OK.getStatusCode()); + + response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); + response.then().assertThat().statusCode(OK.getStatusCode()); + + // Check that the semantic api returns the new title + jsonLDString = getData(response.getBody().asString()); + JsonObject jsonLDObject = JSONLDUtil.decontextualizeJsonLD(jsonLDString); + assertEquals("New Title", jsonLDObject.getString("http://purl.org/dc/terms/title")); + + // Add an additional description (which is multi-valued and compound) + // Also add new terms of use (single value so would fail with replace false if a + // value existed) + String newDescription = "{\"citation:Description\": {\"dsDescription:Text\": \"New description\", \"https://dataverse.org/schema/core#termsOfUse\": \"New terms\"} \"@context\":{\"citation\": \"https://dataverse.org/schema/citation/\",\"dsDescription\": \"https://dataverse.org/schema/citation/dsDescription#\"}}"; + response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, newDescription, false); + response.then().assertThat().statusCode(OK.getStatusCode()); + + response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); + response.then().assertThat().statusCode(OK.getStatusCode()); + + // Look for a second description + jsonLDString = getData(response.getBody().asString()); + jsonLDObject = JSONLDUtil.decontextualizeJsonLD(jsonLDString); + assertEquals("New description", + ((JsonObject) jsonLDObject.getJsonArray("https://dataverse.org/schema/citation/Description").get(1)) + .getString("https://dataverse.org/schema/citation/dsDescription#Text")); + + // Can't add terms of use with replace=false and a value already set (single + // valued field) + String badTerms = "{\"https://dataverse.org/schema/core#termsOfUse\": \"Bad terms\"}}"; + response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, badTerms, false); + response.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); + + // Delete the terms of use + response = UtilIT.deleteDatasetJsonLDMetadata(datasetId, apiToken, + "{\"https://dataverse.org/schema/core#termsOfUse\": \"New terms\"}"); + response.then().assertThat().statusCode(OK.getStatusCode()); + + response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); + response.then().assertThat().statusCode(OK.getStatusCode()); + + // Verify that they're gone + jsonLDString = getData(response.getBody().asString()); + jsonLDObject = JSONLDUtil.decontextualizeJsonLD(jsonLDString); + assertTrue(!jsonLDObject.containsKey("https://dataverse.org/schema/core#termsOfUse")); + + // Cleanup - delete dataset, dataverse, user... + Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); + deleteDatasetResponse.prettyPrint(); + assertEquals(200, deleteDatasetResponse.getStatusCode()); + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + deleteDataverseResponse.prettyPrint(); + assertEquals(200, deleteDataverseResponse.getStatusCode()); + + Response deleteUserResponse = UtilIT.deleteUser(username); + deleteUserResponse.prettyPrint(); + assertEquals(200, deleteUserResponse.getStatusCode()); + + } + + private String getData(String body) { + try (StringReader rdr = new StringReader(body)) { + return Json.createReader(rdr).readObject().getJsonObject("data").toString(); + } + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index dc471a76a04..f06cd6ef395 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -11,6 +11,7 @@ import javax.json.JsonObject; import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.logging.Level; @@ -38,7 +39,6 @@ import static com.jayway.restassured.RestAssured.given; import edu.harvard.iq.dataverse.util.StringUtil; import java.io.StringReader; -import javax.json.JsonArray; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -397,7 +397,7 @@ static Response createDatasetViaNativeApi(String dataverseAlias, String pathToJs return createDatasetResponse; } - private static String getDatasetJson(String pathToJsonFile) { + static String getDatasetJson(String pathToJsonFile) { File datasetVersionJson = new File(pathToJsonFile); try { String datasetVersionAsJson = new String(Files.readAllBytes(Paths.get(datasetVersionJson.getAbsolutePath()))); @@ -2628,5 +2628,29 @@ static String getBannerMessageIdFromResponse(String getBannerMessagesResponse) { return "0"; } + static Response getDatasetJsonLDMetadata(Integer datasetId, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .accept("application/ld+json") + .get("/api/datasets/" + datasetId + "/metadata"); + return response; + } + static Response updateDatasetJsonLDMetadata(Integer datasetId, String apiToken, String jsonLDBody, boolean replace) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType("application/ld+json") + .body(jsonLDBody.getBytes(StandardCharsets.UTF_8)) + .put("/api/datasets/" + datasetId + "/metadata?replace=" + replace); + return response; + } + + static Response deleteDatasetJsonLDMetadata(Integer datasetId, String apiToken, String jsonLDBody) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType("application/ld+json") + .body(jsonLDBody.getBytes(StandardCharsets.UTF_8)) + .put("/api/datasets/" + datasetId + "/metadata/delete"); + return response; + } } From ccd34ec799667a28cab6327a8b4b1bcea13d25dc Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 30 Jun 2021 12:47:58 -0400 Subject: [PATCH 261/354] rename SQL update script #7451 --- ...sql => V5.5.0.4__7451-unique-constraint-storageidentifier.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.5.0.3__7451-unique-constraint-storageidentifier.sql => V5.5.0.4__7451-unique-constraint-storageidentifier.sql} (100%) diff --git a/src/main/resources/db/migration/V5.5.0.3__7451-unique-constraint-storageidentifier.sql b/src/main/resources/db/migration/V5.5.0.4__7451-unique-constraint-storageidentifier.sql similarity index 100% rename from src/main/resources/db/migration/V5.5.0.3__7451-unique-constraint-storageidentifier.sql rename to src/main/resources/db/migration/V5.5.0.4__7451-unique-constraint-storageidentifier.sql From 62451cfb5179d26bd261d4652f0c6cfdd2647917 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 30 Jun 2021 13:02:26 -0400 Subject: [PATCH 262/354] change studyDsc IDNo to DOI, add citation/holdings with URI --- .../dataverse/export/ddi/DdiExportUtil.java | 20 +++++++++++++++---- .../dataverse/export/ddi/dataset-finch1.xml | 3 ++- .../iq/dataverse/export/ddi/exportfull.xml | 3 ++- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 4fed5913263..97237933826 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -164,6 +164,15 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) DatasetVersionDTO version = datasetDto.getDatasetVersion(); String persistentProtocol = datasetDto.getProtocol(); String persistentAgency = persistentProtocol; + + String persistentAuthority = datasetDto.getAuthority(); + String persistentId = datasetDto.getIdentifier(); + + String pidUri = persistentProtocol + ":" + persistentAuthority + "/" + persistentId; + //Some tests don't send real PIDs - don't try to get their URL form + if(!pidUri.equals("null:null/null")) { + pidUri= new GlobalId(persistentProtocol + ":" + persistentAuthority + "/" + persistentId).toURL().toString(); + } // The "persistentAgency" tag is used for the "agency" attribute of the // ddi section; back in the DVN3 days we used "handle" and "DOI" // for the 2 supported protocols, respectively. For the sake of backward @@ -174,8 +183,6 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) persistentAgency = "DOI"; } - String persistentAuthority = datasetDto.getAuthority(); - String persistentId = datasetDto.getIdentifier(); //docDesc Block writeDocDescElement (xmlw, datasetDto); //stdyDesc Block @@ -189,7 +196,9 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) xmlw.writeStartElement("IDNo"); writeAttribute(xmlw, "agency", persistentAgency); - xmlw.writeCharacters(persistentProtocol + ":" + persistentAuthority + "/" + persistentId); + + + xmlw.writeCharacters(pidUri); xmlw.writeEndElement(); // IDNo writeOtherIdElement(xmlw, version); xmlw.writeEndElement(); // titlStmt @@ -225,7 +234,10 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) xmlw.writeEndElement(); // diststmt writeSeriesElement(xmlw, version); - + xmlw.writeStartElement("holdings"); + writeAttribute(xmlw, "URI", pidUri); + xmlw.writeEndElement(); //holdings + xmlw.writeEndElement(); // citation //End Citation Block diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml index 70a2e139929..ba16a6e4991 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml @@ -17,7 +17,7 @@ Darwin's Finches - doi:10.5072/FK2/PCA2E3 + https://doi.org/10.5072/FK2/PCA2E3 Finch, Fiona @@ -30,6 +30,7 @@ Jimmy Finch Added, Depositor + diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml index 9b803c9f1bb..5dfb763faa1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml @@ -22,7 +22,7 @@ Replication Data for: Title Subtitle Alternative Title - doi:10.5072/FK2/WKUKGV + https://doi.org/10.5072/FK2/WKUKGV OtherIDIdentifier1 OtherIDIdentifier2 @@ -56,6 +56,7 @@ SeriesName SeriesInformation + From 375bf824c2fc45f8446f80cea53a1e16a7f9d4ee Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 30 Jun 2021 13:14:18 -0400 Subject: [PATCH 263/354] prevent page from blowing up if no remind msg in bundle #7975 --- .../iq/dataverse/DatasetServiceBean.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 685017200b5..2ddaa9fe9ff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -732,21 +732,30 @@ public void exportAllDatasets(boolean forceReExport) { //depends on dataset state and user privleges public String getReminderString(Dataset dataset, boolean canPublishDataset) { + String reminderString; + if(!dataset.isReleased() ){ //messages for draft state. if (canPublishDataset){ - return BundleUtil.getStringFromBundle("dataset.message.publish.remind.draft"); + reminderString = BundleUtil.getStringFromBundle("dataset.message.publish.remind.draft"); } else { - return BundleUtil.getStringFromBundle("dataset.message.submit.remind.draft"); + reminderString = BundleUtil.getStringFromBundle("dataset.message.submit.remind.draft"); } } else{ //messages for new version - post-publish if (canPublishDataset){ - return BundleUtil.getStringFromBundle("dataset.message.publish.remind.version"); + reminderString = BundleUtil.getStringFromBundle("dataset.message.publish.remind.version"); } else { - return BundleUtil.getStringFromBundle("dataset.message.submit.remind.version"); + reminderString = BundleUtil.getStringFromBundle("dataset.message.submit.remind.version"); } } + + if (reminderString != null) { + return reminderString; + } else { + logger.warning("Unable to get reminder string from bundle. Returning empty string."); + return ""; + } } public void updateLastExportTimeStamp(Long datasetId) { From 4e2e6a56b6a6aea4009cba0d11c115a567f02847 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 30 Jun 2021 14:29:16 -0400 Subject: [PATCH 264/354] get "create dataset" working again #7986 --- .../java/edu/harvard/iq/dataverse/DatasetPage.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 5260e00a443..93c64b9de41 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1851,6 +1851,11 @@ private String init(boolean initFull) { JsfHelper.addWarningMessage(retrieveDatasetVersionResponse.getDifferentVersionMessage());//BundleUtil.getStringFromBundle("dataset.message.metadataSuccess")); } + // init the citation + // TODO: Need to do this after privateUrl is initialized? + displayCitation = dataset.getCitation(true, workingVersion, isAnonymizedAccess()); + logger.fine("Citation: " + displayCitation); + if(workingVersion.isPublished()) { MakeDataCountEntry entry = new MakeDataCountEntry(FacesContext.getCurrentInstance(), dvRequestService, workingVersion); mdcLogService.logEntry(entry); @@ -1985,11 +1990,6 @@ private String init(boolean initFull) { } } - // init the citation - //Need to do this after privateUrl is initialized ( - displayCitation = dataset.getCitation(true, workingVersion, isAnonymizedAccess()); - logger.fine("Citation: " + displayCitation); - displayLockInfo(dataset); for (FileMetadata fmd : workingVersion.getFileMetadatas()) { From e4b819c311ccb480da4dd31b820d83880ec14fb5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 30 Jun 2021 16:35:57 -0400 Subject: [PATCH 265/354] remove TODO #7986 Per comment from Jim: "This works and doesn't break the anonymized access functionality as I thought it might." --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 93c64b9de41..028d0260dde 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1852,7 +1852,6 @@ private String init(boolean initFull) { } // init the citation - // TODO: Need to do this after privateUrl is initialized? displayCitation = dataset.getCitation(true, workingVersion, isAnonymizedAccess()); logger.fine("Citation: " + displayCitation); From 0344dc14cd3616affa141c8ecd8e79617d4ee03e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 1 Jul 2021 14:38:51 -0400 Subject: [PATCH 266/354] add anonymized access methods --- .../harvard/iq/dataverse/FileMetadata.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FileMetadata.java b/src/main/java/edu/harvard/iq/dataverse/FileMetadata.java index 7b0fb0fd76c..0b2a92fe06a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileMetadata.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileMetadata.java @@ -374,23 +374,19 @@ public String getFileDateToDisplay() { } return ""; } - + public String getFileCitation(){ - return getFileCitation(false); - } - + return getFileCitation(false, false); + } - - - public String getFileCitation(boolean html){ - return new DataCitation(this).toString(html); - } - - public String getDirectFileCitation(boolean html){ - return new DataCitation(this, true).toString(html); + public String getFileCitation(boolean html, boolean anonymized){ + return new DataCitation(this).toString(html, anonymized); } - - + + public String getDirectFileCitation(boolean html, boolean anonymized){ + return new DataCitation(this, true).toString(html, anonymized); + } + public DatasetVersion getDatasetVersion() { return datasetVersion; } From a980be974fd14709c86dbefbeb0391c8b266b2b3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 2 Jul 2021 09:25:14 -0400 Subject: [PATCH 267/354] release notes for related PRs --- doc/release-notes/7786-ddi_changes.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/release-notes/7786-ddi_changes.md diff --git a/doc/release-notes/7786-ddi_changes.md b/doc/release-notes/7786-ddi_changes.md new file mode 100644 index 00000000000..18bf2d2183a --- /dev/null +++ b/doc/release-notes/7786-ddi_changes.md @@ -0,0 +1,9 @@ +### Enhancements to DDI Metadata Exports + +Several changes have been made to the DDI exports to improve support for internationalization and to improve compliance with CESSDA requirements. These changes include: +* Addition of a holdings element with a URI attribute whose value is the URL form of the dataset PID +* Change to use the URL form of the dataset PID in the \ IDNo element +* Addition of xml:lang attributes specifying the dataset metadata language at the document level and for individual elements such as title and description +* Specification of controlled vocabulary terms in duplicate elements in multiple languages (in the installation default langauge and, if different, the dataset metadata language) + +While these changes are intended to improve harvesting and integration with external systems, the could break existing connections that make assumptions about the elements and attributes that have been changed. From e159003877928889aa5f3293b4ae64b99dfbf8d2 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 2 Jul 2021 12:20:55 -0400 Subject: [PATCH 268/354] fix test --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 7074c55d64d..e8f1e71e25e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -2210,7 +2210,7 @@ public void testSemanticMetadataAPIs() { // Add an additional description (which is multi-valued and compound) // Also add new terms of use (single value so would fail with replace false if a // value existed) - String newDescription = "{\"citation:Description\": {\"dsDescription:Text\": \"New description\", \"https://dataverse.org/schema/core#termsOfUse\": \"New terms\"} \"@context\":{\"citation\": \"https://dataverse.org/schema/citation/\",\"dsDescription\": \"https://dataverse.org/schema/citation/dsDescription#\"}}"; + String newDescription = "{\"citation:Description\": {\"dsDescription:Text\": \"New description\"}, \"https://dataverse.org/schema/core#termsOfUse\": \"New terms\", \"@context\":{\"citation\": \"https://dataverse.org/schema/citation/\",\"dsDescription\": \"https://dataverse.org/schema/citation/dsDescription#\"}}"; response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, newDescription, false); response.then().assertThat().statusCode(OK.getStatusCode()); From d857009b3ea3d3b01488390a6edd2b5ebcf76eee Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 2 Jul 2021 12:20:55 -0400 Subject: [PATCH 269/354] fix test --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index f00bcd59119..bb1d7956265 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -2211,7 +2211,7 @@ public void testSemanticMetadataAPIs() { // Add an additional description (which is multi-valued and compound) // Also add new terms of use (single value so would fail with replace false if a // value existed) - String newDescription = "{\"citation:Description\": {\"dsDescription:Text\": \"New description\", \"https://dataverse.org/schema/core#termsOfUse\": \"New terms\"} \"@context\":{\"citation\": \"https://dataverse.org/schema/citation/\",\"dsDescription\": \"https://dataverse.org/schema/citation/dsDescription#\"}}"; + String newDescription = "{\"citation:Description\": {\"dsDescription:Text\": \"New description\"}, \"https://dataverse.org/schema/core#termsOfUse\": \"New terms\", \"@context\":{\"citation\": \"https://dataverse.org/schema/citation/\",\"dsDescription\": \"https://dataverse.org/schema/citation/dsDescription#\"}}"; response = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, newDescription, false); response.then().assertThat().statusCode(OK.getStatusCode()); From cf8b2b52f4f795c9717cef2702e53be377651a92 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 2 Jul 2021 12:24:48 -0400 Subject: [PATCH 270/354] Update doc/release-notes/6497-semantic-api.md Co-authored-by: Philip Durbin --- doc/release-notes/6497-semantic-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6497-semantic-api.md b/doc/release-notes/6497-semantic-api.md index 58dcad44284..8222892d18e 100644 --- a/doc/release-notes/6497-semantic-api.md +++ b/doc/release-notes/6497-semantic-api.md @@ -2,6 +2,6 @@ ### Dataset Semantic API (Experimental) -Dataset metadata can be retrieved/set/updated using a new, flatter JSONLD format - following the format of an OAI-ORE export (RDA-conformant Bags), allowing for easier transfer of metadata to/from other systems (i.e. without needing to know Dataverse's metadata block and field storage architecture). This new API also allows for the update of terms metadata (#5899). +Dataset metadata can be retrieved/set/updated using a new, flatter JSON-LD format - following the format of an OAI-ORE export (RDA-conformant Bags), allowing for easier transfer of metadata to/from other systems (i.e. without needing to know Dataverse's metadata block and field storage architecture). This new API also allows for the update of terms metadata (#5899). This development was supported by the [Research Data Alliance](https://rd-alliance.org), DANS, and Sciences PO and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). From d5ff95563970fb5aac3622bea613fcae70a1d86c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 2 Jul 2021 12:30:10 -0400 Subject: [PATCH 271/354] Update doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst Co-authored-by: Philip Durbin --- .../source/developers/dataset-semantic-metadata-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst index 00876d0887a..8b97e9af0b2 100644 --- a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst @@ -2,7 +2,7 @@ Dataset Semantic Metadata API ============================= The OAI_ORE metadata export format represents Dataset metadata using json-ld (see the :doc:`/admin/metadataexport` section). As part of an RDA-supported effort to allow import of Datasets exported as Bags with an included OAI_ORE metadata file, -an experimental API has been created that provides a json-ld alternative the the v1.0 API calls to get/set/delete Dataset metadata in the :doc:`/api/native-api`. +an experimental API has been created that provides a json-ld alternative to the v1.0 API calls to get/set/delete Dataset metadata in the :doc:`/api/native-api`. You may prefer to work with this API if you are building a tool to import from a Bag/OAI-ORE source or already work with json-ld representations of metadata, or if you prefer the flatter json-ld representation to Dataverse software's json representation (which includes structure related to the metadata blocks involved and the type/multiplicity of the metadata fields.) You may not want to use this API if you need stability and backward compatibility (the 'experimental' designation for this API implies that community feedback is desired and that, in future Dataverse software versions, the API may be modified based on that feedback). From 61627d161911424e3d586e2f4ffea498c02f7a80 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 2 Jul 2021 13:23:48 -0400 Subject: [PATCH 272/354] add create example, remove solr schema copies file --- conf/solr/8.8.1/schema_dv_mdb_copies.xml | 162 ------------------ .../source/_static/api/dataset-create.jsonld | 15 ++ .../dataset-semantic-metadata-api.rst | 12 ++ 3 files changed, 27 insertions(+), 162 deletions(-) delete mode 100644 conf/solr/8.8.1/schema_dv_mdb_copies.xml create mode 100644 doc/sphinx-guides/source/_static/api/dataset-create.jsonld diff --git a/conf/solr/8.8.1/schema_dv_mdb_copies.xml b/conf/solr/8.8.1/schema_dv_mdb_copies.xml deleted file mode 100644 index 39b7bd19968..00000000000 --- a/conf/solr/8.8.1/schema_dv_mdb_copies.xml +++ /dev/null @@ -1,162 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/doc/sphinx-guides/source/_static/api/dataset-create.jsonld b/doc/sphinx-guides/source/_static/api/dataset-create.jsonld new file mode 100644 index 00000000000..16861ff64ad --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/dataset-create.jsonld @@ -0,0 +1,15 @@ +{ + "http://purl.org/dc/terms/title": "Darwin's Finches", + "http://purl.org/dc/terms/subject": "Medicine, Health and Life Sciences", + "http://purl.org/dc/terms/creator": { + "https://dataverse.org/schema/citation/author#Name": "Finch, Fiona", + "https://dataverse.org/schema/citation/author#Affiliation": "Birds Inc." + }, + "https://dataverse.org/schema/citation/Contact": { + "https://dataverse.org/schema/citation/datasetContact#E-mail": "finch@mailinator.com", + "https://dataverse.org/schema/citation/datasetContact#Name": "Finch, Fiona" + }, + "https://dataverse.org/schema/citation/Description": { + "https://dataverse.org/schema/citation/dsDescription#Text": "Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds." + } +} \ No newline at end of file diff --git a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst index 00876d0887a..1025c962d7b 100644 --- a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst @@ -89,3 +89,15 @@ With curl, this is done by adding the following header: .. code-block:: bash -H 'Content-Type: application/ld+json' + + .. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export DATAVERSE_ID=root + export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV + + curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$DATAVERSE_ID/datasets --upload-file dataset-create.jsonld + +An example jsonld file is available at :download:`dataset-create.jsonld <../_static/api/dataset-create.jsonld>` + From ba653ee801bd543628a31ce866772d4a854cf7db Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 2 Jul 2021 13:28:48 -0400 Subject: [PATCH 273/354] update example --- .../source/_static/api/dataset-migrate.jsonld | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld b/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld index 520a2830cee..7ab4ab3690b 100644 --- a/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld +++ b/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld @@ -2,12 +2,16 @@ "citation:Depositor": "Admin, Dataverse", "Title": "Test Dataset", "Subject": "Computer and Information Science", +"Creator": { + "author:Name": "Admin, Dataverse", + "author:Affiliation": "GDCC" +}, "Deposit Date": "2020-10-08", "citation:Distributor": { -"distributor:Name": "Demo Dataverse Repository", -"distributor:Affiliation": "Dataverse Community", -"distributor:Abbreviation": "GDCC", -"distributor:URL": "https://dataverse.org/global-dataverse-community-consortium" + "distributor:Name": "Demo Dataverse Repository", + "distributor:Affiliation": "Dataverse Community", + "distributor:Abbreviation": "GDCC", + "distributor:URL": "https://dataverse.org/global-dataverse-community-consortium" }, "citation:Contact": { "datasetContact:Name": "Admin, Dataverse", @@ -15,7 +19,7 @@ "datasetContact:E-mail": "admin@demo.dataverse.org" }, "citation:Description": { -"dsDescription:Text": "A short description" + "dsDescription:Text": "A short description" }, "@id": "doi:10.33564/FK27U7YBV", "schema:version": "1.0", @@ -24,19 +28,15 @@ "dvcore:fileRequestAccess": false }, "@context": { -"Creator": "http://purl.org/dc/terms/creator", -"Deposit Date": "http://purl.org/dc/terms/dateSubmitted", -"Identifier Scheme": "http://purl.org/spar/datacite/AgentIdentifierScheme", -"Subject": "http://purl.org/dc/terms/subject", -"Title": "http://purl.org/dc/terms/title", -"author": "https://dataverse.org/schema/citation/author#", -"citation": "https://dataverse.org/schema/citation/", -"datasetContact": "https://dataverse.org/schema/citation/datasetContact#", -"dcterms": "http://purl.org/dc/terms/", -"distributor": "https://dataverse.org/schema/citation/distributor#", -"dsDescription": "https://dataverse.org/schema/citation/dsDescription#", -"dvcore": "https://dataverse.org/schema/core#", -"keyword": "https://dataverse.org/schema/citation/keyword#", -"ore": "http://www.openarchives.org/ore/terms/", -"schema": "http://schema.org/" + "Creator": "http://purl.org/dc/terms/creator", + "Deposit Date": "http://purl.org/dc/terms/dateSubmitted", + "Subject": "http://purl.org/dc/terms/subject", + "Title": "http://purl.org/dc/terms/title", + "author": "https://dataverse.org/schema/citation/author#", + "citation": "https://dataverse.org/schema/citation/", + "datasetContact": "https://dataverse.org/schema/citation/datasetContact#", + "distributor": "https://dataverse.org/schema/citation/distributor#", + "dsDescription": "https://dataverse.org/schema/citation/dsDescription#", + "dvcore": "https://dataverse.org/schema/core#", + "schema": "http://schema.org/" }} From 1d54c68fe4b515e56fee490a302cd8464d278d2d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 2 Jul 2021 13:55:19 -0400 Subject: [PATCH 274/354] removed debug logging --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 608d9554f62..645b3cc8355 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -714,14 +714,12 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String @Path("{id}/metadata/delete") @Consumes("application/ld+json, application/json-ld") public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id) { - logger.info("In delteMetadata"); try { Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); DatasetVersion dsv = ds.getEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); dsv = JSONLDUtil.deleteDatasetVersionMDFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc); - logger.info("Updating ver"); DatasetVersion managedVersion; if (updateDraft) { Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); From 4c1d31a59aa4b0be7de74a95efc8d1c08f693a36 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 2 Jul 2021 14:20:29 -0400 Subject: [PATCH 275/354] missing header --- .../source/developers/dataset-semantic-metadata-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst index b81349384dd..da28cc60c53 100644 --- a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst @@ -97,7 +97,7 @@ With curl, this is done by adding the following header: export DATAVERSE_ID=root export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV - curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$DATAVERSE_ID/datasets --upload-file dataset-create.jsonld + curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -X POST $SERVER_URL/api/dataverses/$DATAVERSE_ID/datasets --upload-file dataset-create.jsonld An example jsonld file is available at :download:`dataset-create.jsonld <../_static/api/dataset-create.jsonld>` From caca84406f2bbf99347198ec3325ee54ab9cab16 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 2 Jul 2021 14:24:15 -0400 Subject: [PATCH 276/354] merge issues --- .../edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 9b24b261e56..0a29211c9db 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -148,11 +148,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv } fieldByTypeMap.put(dsf.getDatasetFieldType(), dsf); } -<<<<<<< HEAD - -======= ->>>>>>> refs/heads/IQSS/6497-semantic_api TermsOfUseAndAccess terms = (dsv.getTermsOfUseAndAccess()!=null) ? dsv.getTermsOfUseAndAccess().copyTermsOfUseAndAccess() : new TermsOfUseAndAccess(); for (String key : jsonld.keySet()) { @@ -182,11 +178,7 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv addField(dsf, valArray, dsft, datasetFieldSvc, append); } else { -<<<<<<< HEAD //When migrating, the publication date and version number can be set -======= - ->>>>>>> refs/heads/IQSS/6497-semantic_api if (key.equals(JsonLDTerm.schemaOrg("datePublished").getUrl())&& migrating && !append) { dsv.setVersionState(VersionState.RELEASED); } else if (key.equals(JsonLDTerm.schemaOrg("version").getUrl())&& migrating && !append) { From 0c83179cf21a46765d11d435ca078fcbd7dcac81 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 6 Jul 2021 16:11:50 -0400 Subject: [PATCH 277/354] Added an extra clause for some IQSS-specific harvested identifiers. Extremely unlikely to be encountered anywhere else; but need to be included to be able to QA on a copy of the prod. db. Plus some extr diagnostics. (#7451) --- scripts/issues/7451/check_datafiles_7451.sh | 32 +++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/scripts/issues/7451/check_datafiles_7451.sh b/scripts/issues/7451/check_datafiles_7451.sh index fa1b22e96ed..1e4c95c69f4 100755 --- a/scripts/issues/7451/check_datafiles_7451.sh +++ b/scripts/issues/7451/check_datafiles_7451.sh @@ -69,13 +69,41 @@ else cat /tmp/harvestedidentifiers.tmp | sed 's:\\:\\\\:g' | while read howmany dataset storageidentifier do - PG_QUERY_SI=`printf "${PG_QUERY_FIX_1}" $dataset "$storageidentifier"` + # Harvard prod. db had a few harvested storage identifiers consisting of a single space (" "), + # which would confuse the shell. Extremely unlikely to be found in any other installation. + if [[ "x${storageidentifier}" = "x" ]] + then + storageidentifier=" " + fi + PG_QUERY_SI=`printf "${PG_QUERY_FIX_1}" $dataset "$storageidentifier"` ${PSQL_EXEC} -h ${pg_host} -U ${pg_user} -d ${pg_db} -tA -F ' ' -c "${PG_QUERY_SI}" done echo "... done." - echo + echo + + echo -n "Let's confirm that all these dupes have been fixed... " + ${PSQL_EXEC} -h ${pg_host} -U ${pg_user} -d ${pg_db} -tA -F ' ' -c "${PG_QUERY_1}" | + uniq -c | + awk '{if ($1 > 1) print $0}' | sort -u > /tmp/harvestedidentifiers.tmp + + NUM_CONFIRMED=`cat /tmp/harvestedidentifiers.tmp | wc -l` + + if [ $NUM_CONFIRMED == 0 ] + then + echo "Looks good." + echo + else + echo "Oops!" + echo "Unfortunately, the script failed to fix some of the harvested duplicates." + echo "Please send the contents of the file /tmp/harvestedidentifiers.tmp" + echo "to Dataverse support at support@dataverse.org." + echo "Apologies for the extra trouble..." + echo + exit 1 + fi + fi From b82bcc82b98a66af6097c81e60d883f1078c75d0 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Wed, 7 Jul 2021 14:34:56 -0400 Subject: [PATCH 278/354] #7893 link Rserve documentation to necessary files in Dataverse repo --- doc/sphinx-guides/source/installation/prerequisites.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 4f04024607f..f92d08f4596 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -362,7 +362,14 @@ The Dataverse Software uses `Rserve `_ to communicat to R. Rserve is installed as a library package, as described in the step above. It runs as a daemon process on the server, accepting network connections on a dedicated port. This requires some extra -configuration and we provide a script (:fixedwidthplain:`scripts/r/rserve/rserve-setup.sh`) for setting it up. +configuration and we provide a script, rserve-setup.sh, for setting it up. + +You'll want to obtain local copies of the Rserve setup files found in +https://github.com/IQSS/dataverse/tree/develop/scripts/r/rserve +either by cloning a local copy of the IQSS repository: +:fixedwidthplain:`git clone https://github.com/IQSS/dataverse.git` +or by downloading the files individually. + Run the script as follows (as root):: cd /scripts/r/rserve From 4e644dfe8f5fe3c9cbfc8612eae720d59d7a089d Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Wed, 7 Jul 2021 15:04:29 -0400 Subject: [PATCH 279/354] #7893 remove redundant script mention per feedback from Leonid --- doc/sphinx-guides/source/installation/prerequisites.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index f92d08f4596..ba4d9cc3785 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -362,7 +362,7 @@ The Dataverse Software uses `Rserve `_ to communicat to R. Rserve is installed as a library package, as described in the step above. It runs as a daemon process on the server, accepting network connections on a dedicated port. This requires some extra -configuration and we provide a script, rserve-setup.sh, for setting it up. +configuration and we provide a script for setting it up. You'll want to obtain local copies of the Rserve setup files found in https://github.com/IQSS/dataverse/tree/develop/scripts/r/rserve From d8dabb2b25bdded8f50cfe5766d5852f239f6e4b Mon Sep 17 00:00:00 2001 From: Benjamin Peuch Date: Thu, 8 Jul 2021 16:46:34 +0200 Subject: [PATCH 280/354] Adding -H + API token to curl commands Without ``-H`` and the API token in these curl commands, the native API rejects the user's requests on the ground that they are a 'guest'. --- doc/sphinx-guides/source/api/native-api.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 3a3f2fe5cec..4cc56511422 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -609,13 +609,13 @@ Example: Getting the dataset whose DOI is *10.5072/FK2/J8SJZB*: export SERVER_URL=https://demo.dataverse.org export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB - curl $SERVER_URL/api/datasets/:persistentId/?persistentId=$PERSISTENT_IDENTIFIER + curl -H "X-Dataverse-key:$API_KEY" $SERVER_URL/api/datasets/:persistentId/?persistentId=$PERSISTENT_IDENTIFIER The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl https://demo.dataverse.org/api/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB + curl -H "X-Dataverse-key:$API_KEY" https://demo.dataverse.org/api/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB Getting its draft version: @@ -624,13 +624,13 @@ Getting its draft version: export SERVER_URL=https://demo.dataverse.org export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB - curl http://$SERVER/api/datasets/:persistentId/versions/:draft?persistentId=$PERSISTENT_IDENTIFIER + curl -H "X-Dataverse-key:$API_KEY" http://$SERVER/api/datasets/:persistentId/versions/:draft?persistentId=$PERSISTENT_IDENTIFIER The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl https://demo.dataverse.org/api/datasets/:persistentId/versions/:draft?persistentId=doi:10.5072/FK2/J8SJZB + curl -H "X-Dataverse-key:$API_KEY" https://demo.dataverse.org/api/datasets/:persistentId/versions/:draft?persistentId=doi:10.5072/FK2/J8SJZB |CORS| Show the dataset whose id is passed: From d451f872ae73fc357b2520103622fd8ec9a1ad12 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 8 Jul 2021 11:32:48 -0400 Subject: [PATCH 281/354] update solr version --- doc/release-notes/6497-migrate-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6497-migrate-api.md b/doc/release-notes/6497-migrate-api.md index e5699a0d155..72573cfe905 100644 --- a/doc/release-notes/6497-migrate-api.md +++ b/doc/release-notes/6497-migrate-api.md @@ -10,7 +10,7 @@ This development was supported by the [Research Data Alliance](https://rd-allian Update Solr Schema -- copy schema_dv_mdb_fields.xml and schema_dv_mdb_copies.xml to solr server, for example into /usr/local/solr/solr-7.7.2/server/solr/collection1/conf/ directory +- copy schema_dv_mdb_fields.xml and schema_dv_mdb_copies.xml to solr server, for example into /usr/local/solr/solr-8.8.1/server/solr/collection1/conf/ directory - Restart Solr, or tell Solr to reload its configuration: From a5a745d157036972194ddd790d3320c140219e6e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 8 Jul 2021 12:32:14 -0400 Subject: [PATCH 282/354] remove metadataOnOrig per review --- conf/solr/8.8.1/schema_dv_mdb_fields.xml | 1 - scripts/api/data/metadatablocks/migration.tsv | 5 --- .../iq/dataverse/util/json/JSONLDUtil.java | 40 ------------------- .../iq/dataverse/util/json/JsonLDTerm.java | 2 - 4 files changed, 48 deletions(-) delete mode 100644 scripts/api/data/metadatablocks/migration.tsv diff --git a/conf/solr/8.8.1/schema_dv_mdb_fields.xml b/conf/solr/8.8.1/schema_dv_mdb_fields.xml index 459c71df514..3f844c6183c 100644 --- a/conf/solr/8.8.1/schema_dv_mdb_fields.xml +++ b/conf/solr/8.8.1/schema_dv_mdb_fields.xml @@ -82,7 +82,6 @@ - diff --git a/scripts/api/data/metadatablocks/migration.tsv b/scripts/api/data/metadatablocks/migration.tsv deleted file mode 100644 index 1f9a7a327f5..00000000000 --- a/scripts/api/data/metadatablocks/migration.tsv +++ /dev/null @@ -1,5 +0,0 @@ -#metadataBlock name dataverseAlias displayName blockURI - migration Migrated Metadata https://dataverse.org/schema/migration/ -#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id termURI - metadataOnOrig Metadata on the original source of migrated datasets. textbox 1 FALSE FALSE FALSE FALSE FALSE FALSE migration https://dataverse.org/schema/core#metadataOnOrig -#controlledVocabulary DatasetField Value identifier displayOrder \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 4b5f5f74046..62d58827696 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -226,46 +226,6 @@ public static DatasetVersion updateDatasetVersionMDFromJsonLD(DatasetVersion dsv } } } - } else { - //Metadata block metadata fields - if (dsftMap.containsKey(JsonLDTerm.metadataOnOrig.getUrl())) { - DatasetFieldType dsft = dsftMap.get(JsonLDTerm.metadataOnOrig.getUrl()); - - DatasetField dsf = null; - if (fieldByTypeMap.containsKey(dsft)) { - dsf = fieldByTypeMap.get(dsft); - // If there's an existing field, we use it with append and remove it for !append - // (except if multiple, which is not the default) - if (!append && !dsft.isAllowMultiples()) { - dsfl.remove(dsf); - } - } - if (dsf == null) { - dsf = new DatasetField(); - dsfl.add(dsf); - dsf.setDatasetFieldType(dsft); - } - - List vals = dsf.getDatasetFieldValues(); - - JsonObject currentValue = null; - DatasetFieldValue datasetFieldValue = null; - if (vals.isEmpty()) { - datasetFieldValue = new DatasetFieldValue(); - vals.add(datasetFieldValue); - datasetFieldValue.setDatasetField(dsf); - dsf.setDatasetFieldValues(vals); - - currentValue = Json.createObjectBuilder().build(); - } else { - datasetFieldValue = vals.get(0); - JsonObject currentVal = decontextualizeJsonLD(datasetFieldValue.getValueForEdit()); - - } - currentValue.put(key, jsonld.get(key)); - JsonObject newValue = recontextualizeJsonLD(currentValue, metadataBlockSvc); - datasetFieldValue.setValue(prettyPrint(newValue)); - } } dsv.setTermsOfUseAndAccess(terms); // ToDo: support Dataverse location metadata? e.g. move to new dataverse? diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java index 18e865b2f4d..20aeceda7de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java @@ -45,8 +45,6 @@ public class JsonLDTerm { public static JsonLDTerm totalSize = JsonLDTerm.DVCore("totalSize"); public static JsonLDTerm fileCount = JsonLDTerm.DVCore("fileCount"); public static JsonLDTerm maxFileSize = JsonLDTerm.DVCore("maxFileSize"); - - public static JsonLDTerm metadataOnOrig = JsonLDTerm.DVCore("metadataOnOrig"); public JsonLDTerm(JsonLDNamespace namespace, String term) { this.namespace = namespace; From 0ee85a3f49ca066b806eabd88de34e327561c394 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Thu, 8 Jul 2021 12:52:51 -0400 Subject: [PATCH 283/354] #7893 use fixedwidthplain text instead, clone master instead of develop --- doc/sphinx-guides/source/installation/prerequisites.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index ba4d9cc3785..cf49ae8e70b 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -365,9 +365,9 @@ network connections on a dedicated port. This requires some extra configuration and we provide a script for setting it up. You'll want to obtain local copies of the Rserve setup files found in -https://github.com/IQSS/dataverse/tree/develop/scripts/r/rserve +:fixedwidthplain:`scripts/r/rserve/<../../../../scripts/r/rserve/>` either by cloning a local copy of the IQSS repository: -:fixedwidthplain:`git clone https://github.com/IQSS/dataverse.git` +:fixedwidthplain:`git clone -b master https://github.com/IQSS/dataverse.git` or by downloading the files individually. Run the script as follows (as root):: From cd30129924bf707a3dedd1adc722f03884106b48 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 8 Jul 2021 13:12:09 -0400 Subject: [PATCH 284/354] Update doc/sphinx-guides/source/developers/dataset-migration-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/dataset-migration-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/dataset-migration-api.rst b/doc/sphinx-guides/source/developers/dataset-migration-api.rst index 5ea65df7868..e8b47e6e713 100644 --- a/doc/sphinx-guides/source/developers/dataset-migration-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-migration-api.rst @@ -14,7 +14,7 @@ This API consists of 2 calls: one to create an initial Dataset version, and one These calls can be used in concert with other API calls to add files, update metadata for additional versions, etc. -Start migrating a Dataset into a Dataverse Collection +Start Migrating a Dataset into a Dataverse Collection ----------------------------------------------------- .. note:: This action requires a Dataverse installation account with super-user permissions. From 95eaec291fee60c47477298f3b3df02801cfd883 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 8 Jul 2021 13:12:17 -0400 Subject: [PATCH 285/354] Update doc/sphinx-guides/source/developers/dataset-migration-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/dataset-migration-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/dataset-migration-api.rst b/doc/sphinx-guides/source/developers/dataset-migration-api.rst index e8b47e6e713..b63429751cd 100644 --- a/doc/sphinx-guides/source/developers/dataset-migration-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-migration-api.rst @@ -17,7 +17,7 @@ These calls can be used in concert with other API calls to add files, update met Start Migrating a Dataset into a Dataverse Collection ----------------------------------------------------- -.. note:: This action requires a Dataverse installation account with super-user permissions. +.. note:: This action requires a Dataverse installation account with superuser permissions. To import a dataset with an existing persistent identifier (PID), the provided json-ld metadata should include it. From f4097bc00e2a2d5ca671c329f7e1d6ba0eff19e2 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 8 Jul 2021 13:12:33 -0400 Subject: [PATCH 286/354] Update doc/sphinx-guides/source/developers/dataset-migration-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/dataset-migration-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/dataset-migration-api.rst b/doc/sphinx-guides/source/developers/dataset-migration-api.rst index b63429751cd..31c2163db8f 100644 --- a/doc/sphinx-guides/source/developers/dataset-migration-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-migration-api.rst @@ -36,7 +36,7 @@ An example jsonld file is available at :download:`dataset-migrate.jsonld <../_st Publish a Migrated Dataset -------------------------- -The call above creates a Dataset. Once it is created, other APIs can be used to add files, add additional metadata, etc. When a version is complete, the following call can be used to publish it with its original publication date +The call above creates a Dataset. Once it is created, other APIs can be used to add files, add additional metadata, etc. When a version is complete, the following call can be used to publish it with its original publication date. .. note:: This action requires a Dataverse installation account with super-user permissions. From 64ed2b09ca13ee0d2ca411f838481afc9b025805 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 8 Jul 2021 13:12:40 -0400 Subject: [PATCH 287/354] Update doc/sphinx-guides/source/developers/dataset-migration-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/dataset-migration-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/dataset-migration-api.rst b/doc/sphinx-guides/source/developers/dataset-migration-api.rst index 31c2163db8f..263ffc3aa99 100644 --- a/doc/sphinx-guides/source/developers/dataset-migration-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-migration-api.rst @@ -38,7 +38,7 @@ Publish a Migrated Dataset The call above creates a Dataset. Once it is created, other APIs can be used to add files, add additional metadata, etc. When a version is complete, the following call can be used to publish it with its original publication date. -.. note:: This action requires a Dataverse installation account with super-user permissions. +.. note:: This action requires a Dataverse installation account with superuser permissions. .. code-block:: bash From ff8e4bd6831f60ca87e64997b2e4fb839073f0ad Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 8 Jul 2021 14:54:45 -0400 Subject: [PATCH 288/354] fixes the small formatting issue with the link (#7893) --- doc/sphinx-guides/source/installation/prerequisites.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index cf49ae8e70b..0265e390d14 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -365,7 +365,7 @@ network connections on a dedicated port. This requires some extra configuration and we provide a script for setting it up. You'll want to obtain local copies of the Rserve setup files found in -:fixedwidthplain:`scripts/r/rserve/<../../../../scripts/r/rserve/>` +https://github.com/IQSS/dataverse/tree/master/scripts/r/rserve either by cloning a local copy of the IQSS repository: :fixedwidthplain:`git clone -b master https://github.com/IQSS/dataverse.git` or by downloading the files individually. From 0a057f15abb9e7dde404fe2418d2ab40c9840a9d Mon Sep 17 00:00:00 2001 From: Benjamin Peuch Date: Fri, 9 Jul 2021 09:27:15 +0200 Subject: [PATCH 289/354] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 4cc56511422..8ca1251b749 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -609,7 +609,7 @@ Example: Getting the dataset whose DOI is *10.5072/FK2/J8SJZB*: export SERVER_URL=https://demo.dataverse.org export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB - curl -H "X-Dataverse-key:$API_KEY" $SERVER_URL/api/datasets/:persistentId/?persistentId=$PERSISTENT_IDENTIFIER + curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/datasets/:persistentId/?persistentId=$PERSISTENT_IDENTIFIER The fully expanded example above (without environment variables) looks like this: From a5b64fc4b03983d17d29159d89d0cc83f1307ea6 Mon Sep 17 00:00:00 2001 From: Benjamin Peuch Date: Fri, 9 Jul 2021 09:27:24 +0200 Subject: [PATCH 290/354] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 8ca1251b749..2d8bb4953e9 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -615,7 +615,7 @@ The fully expanded example above (without environment variables) looks like this .. code-block:: bash - curl -H "X-Dataverse-key:$API_KEY" https://demo.dataverse.org/api/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB + curl -H "X-Dataverse-key:$API_TOKEN" https://demo.dataverse.org/api/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB Getting its draft version: From 048fb125191da61a2078758a2d96434312a1b689 Mon Sep 17 00:00:00 2001 From: Benjamin Peuch Date: Fri, 9 Jul 2021 09:27:32 +0200 Subject: [PATCH 291/354] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 2d8bb4953e9..c75a6cfe6b1 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -624,7 +624,7 @@ Getting its draft version: export SERVER_URL=https://demo.dataverse.org export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB - curl -H "X-Dataverse-key:$API_KEY" http://$SERVER/api/datasets/:persistentId/versions/:draft?persistentId=$PERSISTENT_IDENTIFIER + curl -H "X-Dataverse-key:$API_TOKEN" http://$SERVER/api/datasets/:persistentId/versions/:draft?persistentId=$PERSISTENT_IDENTIFIER The fully expanded example above (without environment variables) looks like this: From 4c0b895bb4482ac1371b1fb2278d6b71fe6f7b73 Mon Sep 17 00:00:00 2001 From: Benjamin Peuch Date: Fri, 9 Jul 2021 09:27:36 +0200 Subject: [PATCH 292/354] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index c75a6cfe6b1..38ccbd3e2b1 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -630,7 +630,7 @@ The fully expanded example above (without environment variables) looks like this .. code-block:: bash - curl -H "X-Dataverse-key:$API_KEY" https://demo.dataverse.org/api/datasets/:persistentId/versions/:draft?persistentId=doi:10.5072/FK2/J8SJZB + curl -H "X-Dataverse-key:$API_TOKEN" https://demo.dataverse.org/api/datasets/:persistentId/versions/:draft?persistentId=doi:10.5072/FK2/J8SJZB |CORS| Show the dataset whose id is passed: From 7b53b56cd6e379cd3a5b8dc6fdaed3cfb7deaa91 Mon Sep 17 00:00:00 2001 From: Robert Verkerk Date: Fri, 9 Jul 2021 10:33:00 +0200 Subject: [PATCH 293/354] Update documentation to be more consise about the handle and give a better example. --- doc/sphinx-guides/source/installation/config.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 6e4214ae90c..4073f53868f 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1549,10 +1549,11 @@ By default this setting is absent and the Dataverse Software assumes it to be fa :HandleAuthHandle +++++++++++++++++++++++++ -Specific for Handle PIDs. Set this setting to to be used on a global handle service when the public key is NOT stored in the default handle. -By default this setting is absent and the Dataverse Software assumes it to be not set. +Specific for Handle PIDs. Set this setting to / to be used on a global handle service when the public key is NOT stored in the default handle. +By default this setting is absent and the Dataverse Software assumes it to be not set. If the public key for instance is stored in handle: 21.T12996/USER01. +For this handle the prefix is '21.T12996' and the suffix is 'USER01'. The command to execute is then: -``curl -X PUT -d '' http://localhost:8080/api/admin/settings/:HandleAuthHandle`` +``curl -X PUT -d '21.T12996/USER01' http://localhost:8080/api/admin/settings/:HandleAuthHandle`` .. _:FileValidationOnPublishEnabled: From fb7b368f39881ee3e6a89de66ad53e5f85c40980 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Fri, 9 Jul 2021 13:37:46 -0400 Subject: [PATCH 294/354] #7936 append deaccessionDialog.reasons with periods for proper display --- src/main/java/propertyFiles/Bundle.properties | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index f0194ca60dc..261f6d04800 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1776,12 +1776,12 @@ file.deaccessionDialog.tip=Once you deaccession this dataset it will no longer b file.deaccessionDialog.version=Version file.deaccessionDialog.reason.question1=Which version(s) do you want to deaccession? file.deaccessionDialog.reason.question2=What is the reason for deaccession? -file.deaccessionDialog.reason.selectItem.identifiable=There is identifiable data in one or more files -file.deaccessionDialog.reason.selectItem.beRetracted=The research article has been retracted -file.deaccessionDialog.reason.selectItem.beTransferred=The dataset has been transferred to another repository -file.deaccessionDialog.reason.selectItem.IRB=IRB request -file.deaccessionDialog.reason.selectItem.legalIssue=Legal issue or Data Usage Agreement -file.deaccessionDialog.reason.selectItem.notValid=Not a valid dataset +file.deaccessionDialog.reason.selectItem.identifiable=There is identifiable data in one or more files. +file.deaccessionDialog.reason.selectItem.beRetracted=The research article has been retracted. +file.deaccessionDialog.reason.selectItem.beTransferred=The dataset has been transferred to another repository. +file.deaccessionDialog.reason.selectItem.IRB=IRB request. +file.deaccessionDialog.reason.selectItem.legalIssue=Legal issue or Data Usage Agreement. +file.deaccessionDialog.reason.selectItem.notValid=Not a valid dataset. file.deaccessionDialog.reason.selectItem.other=Other (Please type reason in space provided below) file.deaccessionDialog.enterInfo=Please enter additional information about the reason for deaccession. file.deaccessionDialog.leaveURL=If applicable, please leave a URL where this dataset can be accessed after deaccessioning. From 350835d05b6820e92289495b03c68444665c5fa0 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 12 Jul 2021 11:01:59 -0400 Subject: [PATCH 295/354] #7858 update replace helper error --- .../iq/dataverse/EditDatafilesPage.java | 45 ++++++++----------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index e11ab0d30c8..031bb1dbdec 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -473,7 +473,6 @@ public String initCreateMode(String modeToken, DatasetVersion version, MutableBo public String init() { - System.out.print("edit data files page init..."); // default mode should be EDIT if (mode == null) { mode = FileEditMode.EDIT; @@ -599,12 +598,9 @@ public String init() { } if (mode == FileEditMode.UPLOAD) { - System.out.print("file edit mode " + mode); - if (settingsWrapper.getUploadMethodsCount() == 1){ - System.out.print("settingsWrapper.getUploadMethodsCount() == 1 " ); + if (settingsWrapper.getUploadMethodsCount() == 1){ JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.uploadFiles.label"), BundleUtil.getStringFromBundle("dataset.message.uploadFilesSingle.message", Arrays.asList(systemConfig.getGuidesBaseUrl(), systemConfig.getGuidesVersion()))); } else if (settingsWrapper.getUploadMethodsCount() > 1) { - System.out.print("settingsWrapper.getUploadMethodsCount() > 1 " ); JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.uploadFiles.label"), BundleUtil.getStringFromBundle("dataset.message.uploadFilesMultiple.message", Arrays.asList(systemConfig.getGuidesBaseUrl(), systemConfig.getGuidesVersion()))); } @@ -1296,7 +1292,6 @@ public boolean showFileUploadFragment(){ public boolean showFileUploadComponent(){ if (mode == FileEditMode.UPLOAD || mode == FileEditMode.CREATE) { - System.out.print("showfileuploadfragement == true"); return true; } @@ -1517,7 +1512,6 @@ public void setHasRsyncScript(Boolean hasRsyncScript) { } private void setUpRsync() { - System.out.print("setUpRsync called..."); logger.fine("setUpRsync called..."); if (DataCaptureModuleUtil.rsyncSupportEnabled(settingsWrapper.getValueForKey(SettingsServiceBean.Key.UploadMethods)) && dataset.getFiles().isEmpty()) { //only check for rsync if no files exist @@ -1534,23 +1528,21 @@ private void setUpRsync() { } catch (EJBException ex) { logger.warning("Problem getting rsync script (EJBException): " + EjbUtil.ejbExceptionToString(ex)); FacesContext.getCurrentInstance().addMessage(uploadComponentId, - new FacesMessage(FacesMessage.SEVERITY_ERROR, - "Problem getting rsync script (EJBException): " + EjbUtil.ejbExceptionToString(ex), - "Problem getting rsync script (EJBException):")); + new FacesMessage(FacesMessage.SEVERITY_ERROR, + "Problem getting rsync script (EJBException): " + EjbUtil.ejbExceptionToString(ex), + "Problem getting rsync script (EJBException):")); } catch (RuntimeException ex) { - System.out.print("RuntimeException..."); logger.warning("Problem getting rsync script (RuntimeException): " + ex.getLocalizedMessage()); - FacesContext.getCurrentInstance().addMessage(uploadComponentId, - new FacesMessage(FacesMessage.SEVERITY_ERROR, - "Problem getting rsync script (RuntimeException): " +ex.getMessage(), - "Problem getting rsync script (RuntimeException):")); + FacesContext.getCurrentInstance().addMessage(uploadComponentId, + new FacesMessage(FacesMessage.SEVERITY_ERROR, + "Problem getting rsync script (RuntimeException): " + ex.getMessage(), + "Problem getting rsync script (RuntimeException):")); } catch (CommandException cex) { - System.out.print("commandException..."); logger.warning("Problem getting rsync script (Command Exception): " + cex.getLocalizedMessage()); - FacesContext.getCurrentInstance().addMessage(uploadComponentId, - new FacesMessage(FacesMessage.SEVERITY_ERROR, - "Problem getting rsync script (Command Exception): " +cex.getMessage(), - "Problem getting rsync script (Command Exception):")); + FacesContext.getCurrentInstance().addMessage(uploadComponentId, + new FacesMessage(FacesMessage.SEVERITY_ERROR, + "Problem getting rsync script (Command Exception): " + cex.getMessage(), + "Problem getting rsync script (Command Exception):")); } } } @@ -2009,7 +2001,6 @@ public void handleFileUpload(FileUploadEvent event) throws IOException { * @param event */ public void handleExternalUpload() { - System.out.print("in handle external upload..."); Map paramMap = FacesContext.getCurrentInstance().getExternalContext().getRequestParameterMap(); this.uploadComponentId = paramMap.get("uploadComponentId"); @@ -2056,9 +2047,13 @@ public void handleExternalUpload() { // ----------------------------------------------------------- // Is this a FileReplaceOperation? If so, then diverge! // ----------------------------------------------------------- - if (this.isFileReplaceOperation()){ + if (this.isFileReplaceOperation()){ this.handleReplaceFileUpload(storageLocation, fileName, contentType, checksumValue, checksumType); - this.setFileMetadataSelectedForTagsPopup(fileReplacePageHelper.getNewFileMetadatasBeforeSave().get(0)); + if (!saveEnabled){ + uploadWarningMessage = fileReplacePageHelper.getErrorMessages(); + return; + } + this.setFileMetadataSelectedForTagsPopup(fileReplacePageHelper.getNewFileMetadatasBeforeSave().get(0)); return; } // ----------------------------------------------------------- @@ -2988,11 +2983,7 @@ public boolean rsyncUploadSupported() { // ToDo - rsync was written before multiple store support and currently is hardcoded to use the "s3" store. // When those restrictions are lifted/rsync can be configured per store, the test in the // Dataset Util method should be updated - System.out.print(" IN rsyncUploadSupported() "); - System.out.print("settingsWrapper.isRsyncUpload() " + settingsWrapper.isRsyncUpload()); - System.out.print("DatasetUtil.isAppropriateStorageDriver(dataset) " + DatasetUtil.isAppropriateStorageDriver(dataset)); if(settingsWrapper.isRsyncUpload() && !DatasetUtil.isAppropriateStorageDriver(dataset) ){ - System.out.print("setuprsync failed message adding..."); //dataset.file.upload.setUp.rsync.failed.detail FacesMessage message = new FacesMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("dataset.file.upload.setUp.rsync.failed"), BundleUtil.getStringFromBundle("dataset.file.upload.setUp.rsync.failed.detail")); FacesContext.getCurrentInstance().addMessage(null, message); From ab82a97c3bca3b5f3c19f14f82333ecd0daf51ee Mon Sep 17 00:00:00 2001 From: chenganj Date: Mon, 12 Jul 2021 15:41:04 -0400 Subject: [PATCH 296/354] added release notes --- .../7900-add-multipleFilesMetadata-dataset.md | 4 +++ doc/sphinx-guides/source/api/native-api.rst | 5 +++ .../developers/s3-direct-upload-api.rst | 34 ++++++++++++++++++- .../datasetutility/AddReplaceFileHelper.java | 19 +++++++++-- 4 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 doc/release-notes/7900-add-multipleFilesMetadata-dataset.md diff --git a/doc/release-notes/7900-add-multipleFilesMetadata-dataset.md b/doc/release-notes/7900-add-multipleFilesMetadata-dataset.md new file mode 100644 index 00000000000..13b77259c69 --- /dev/null +++ b/doc/release-notes/7900-add-multipleFilesMetadata-dataset.md @@ -0,0 +1,4 @@ +### Direct Upload API Now Available for Adding Multiple Files Metadata to the Dataset + +Users can now add metadata of multiple files to the dataset once the files exists in the s3 bucket using the direct upload API. +For more information, see the [Direct DataFile Upload/Replace API section](https://guides.dataverse.org/en/5.6/developers/s3-direct-upload-api.html) of the Dataverse Software Guides. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 4cf128331c5..fa40c956699 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2135,9 +2135,14 @@ The jsonData object includes values for: * "md5Hash" - String with MD5 hash value, or * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. A curl example using an ``PERSISTENT_ID`` +* ``SERVER_URL`` - e.g. https://demo.dataverse.org +* ``API_TOKEN`` - API endpoints require an API token that can be passed as the X-Dataverse-key HTTP header. For more details, see the :doc:`auth` section. +* ``PERSISTENT_IDENTIFIER`` - Example: ``doi:10.5072/FK2/7U7YBV`` + .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst index 9f2386facb1..d1a71c313ca 100644 --- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst +++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst @@ -7,7 +7,7 @@ Direct upload involves a series of three activities, each involving interacting * Requesting initiation of a transfer from the server * Use of the pre-signed URL(s) returned in that call to perform an upload/multipart-upload of the file to S3 -* A call to the server to register the file as part of the dataset/replace a file in the dataset or to cancel the transfer +* A call to the server to register the file/files as part of the dataset/replace a file in the dataset or to cancel the transfer This API is only enabled when a Dataset is configured with a data store supporting direct S3 upload. Administrators should be aware that partial transfers, where a client starts uploading the file/parts of the file and does not contact the server to complete/cancel the transfer, will result in data stored in S3 that is not referenced in the Dataverse installation (e.g. should be considered temporary and deleted.) @@ -116,6 +116,38 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above. +To add multiple Uploaded Files to the Dataset +------------------------------------------------- + +Once the files exists in the s3 bucket, a final API call is needed to add all the files to the Dataset. In this API call, additional metadata is added using the "jsonData" parameter. +jsonData normally includes information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must also include values for: + +* "description" - A description of the file +* "directoryLabel" - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset +* "storageIdentifier" - String +* "fileName" - String +* "mimeType" - String +* "fixity/checksum" either: + + * "md5Hash" - String with MD5 hash value, or + * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings + +The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.DataFile.CheckSumType class and currently include MD5, SHA-1, SHA-256, and SHA-512 + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/7U7YBV + export JSON_DATA="[{'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42', 'fileName':'file1.txt', 'mimeType':'text/plain', 'checksum': {'@type': 'SHA-1', '@value': '123456'}}, \ + {'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53', 'fileName':'file2.txt', 'mimeType':'text/plain', 'checksum': {'@type': 'SHA-1', '@value': '123789'}}]" + + curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/addFiles?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA" + +Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. +With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above. + + Replacing an existing file in the Dataset ----------------------------------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 21f2aca1eba..1fcc355ae6b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -5,10 +5,18 @@ */ package edu.harvard.iq.dataverse.datasetutility; -import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataFile.ChecksumType; -import edu.harvard.iq.dataverse.api.Files; +import edu.harvard.iq.dataverse.DataFileServiceBean; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetLock; +import edu.harvard.iq.dataverse.DatasetServiceBean; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.EjbDataverseEngine; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.PermissionServiceBean; import edu.harvard.iq.dataverse.api.Util; +import edu.harvard.iq.dataverse.api.Files; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.Command; @@ -38,7 +46,12 @@ import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.EJBException; -import javax.json.*; +import javax.json.Json; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObject; +import javax.json.JsonArray; +import javax.json.JsonObjectBuilder; +import javax.json.JsonReader; import javax.validation.ConstraintViolation; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; From 235be32f58af5fdd5e17dc6a98df2967398f3a72 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 13 Jul 2021 13:46:23 +0200 Subject: [PATCH 297/354] build(mail): fixed using transitive dependency on com.sun.mail by adding direct dependency. --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index d31497101bf..5062661ee76 100644 --- a/pom.xml +++ b/pom.xml @@ -306,6 +306,11 @@ ${jakartaee-api.version} provided + + com.sun.mail + jakarta.mail + provided + org.glassfish jakarta.faces From 6dc52f2614990a58f3e4a94dcd97979871897a17 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 13 Jul 2021 14:10:59 +0200 Subject: [PATCH 298/354] style(oidc): change star imports to single imports. --- .../oauth2/oidc/OIDCAuthProvider.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java index d9a1baa9e3e..a9c44010950 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java @@ -1,7 +1,15 @@ package edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc; import com.github.scribejava.core.builder.api.DefaultApi20; -import com.nimbusds.oauth2.sdk.*; +import com.nimbusds.oauth2.sdk.AuthorizationCode; +import com.nimbusds.oauth2.sdk.AuthorizationCodeGrant; +import com.nimbusds.oauth2.sdk.AuthorizationGrant; +import com.nimbusds.oauth2.sdk.ErrorObject; +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.ResponseType; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.TokenRequest; +import com.nimbusds.oauth2.sdk.TokenResponse; import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic; import com.nimbusds.oauth2.sdk.auth.Secret; @@ -11,7 +19,12 @@ import com.nimbusds.oauth2.sdk.id.Issuer; import com.nimbusds.oauth2.sdk.id.State; import com.nimbusds.oauth2.sdk.token.BearerAccessToken; -import com.nimbusds.openid.connect.sdk.*; +import com.nimbusds.openid.connect.sdk.AuthenticationRequest; +import com.nimbusds.openid.connect.sdk.Nonce; +import com.nimbusds.openid.connect.sdk.OIDCTokenResponse; +import com.nimbusds.openid.connect.sdk.OIDCTokenResponseParser; +import com.nimbusds.openid.connect.sdk.UserInfoRequest; +import com.nimbusds.openid.connect.sdk.UserInfoResponse; import com.nimbusds.openid.connect.sdk.claims.UserInfo; import com.nimbusds.openid.connect.sdk.op.OIDCProviderConfigurationRequest; import com.nimbusds.openid.connect.sdk.op.OIDCProviderMetadata; @@ -19,7 +32,6 @@ import edu.harvard.iq.dataverse.authorization.exceptions.AuthorizationSetupException; import edu.harvard.iq.dataverse.authorization.providers.oauth2.AbstractOAuth2AuthenticationProvider; import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2Exception; -import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2TokenData; import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2UserRecord; import edu.harvard.iq.dataverse.util.BundleUtil; From 25999e373365191bfdc6fabe5bca9d8ad27de949 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 13 Jul 2021 14:13:29 +0200 Subject: [PATCH 299/354] build(oidc): update Nimbus OIDC SDK to 9.9.1 IQSS/dataverse-security#37 Nimbus SDK <9.5.1 uses json-smart in a vulnerable version. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5062661ee76..98b4328cea1 100644 --- a/pom.xml +++ b/pom.xml @@ -514,7 +514,7 @@ com.nimbusds oauth2-oidc-sdk - 6.18 + 9.9.1 From 6943862a89623746d05027e34ef4882a585a541a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 13 Jul 2021 16:09:04 +0200 Subject: [PATCH 300/354] build(deps): update Google Cloud Storage to latest to mitigate security flaws. IQSS/dataverse-security#36 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d31497101bf..9ba1b3ad302 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,7 @@ 1.2 3.12.0 4.5.13 + 0.157.0 4.13.1 5.7.0 ${junit.jupiter.version} @@ -146,7 +147,7 @@ com.google.cloud google-cloud-bom - 0.115.0-alpha + ${google.cloud.version} pom import @@ -617,7 +618,6 @@ com.google.cloud google-cloud-storage - 1.97.0 From 0138ebb8a8909ed4943c00c2345278258992d2cf Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 13 Jul 2021 12:54:35 -0400 Subject: [PATCH 301/354] add missing create method (in migrate PR) --- .../harvard/iq/dataverse/api/Dataverses.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index ebc52785637..b56a88af75f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -63,6 +63,8 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.StringUtil; import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty; + +import edu.harvard.iq.dataverse.util.json.JSONLDUtil; import edu.harvard.iq.dataverse.util.json.JsonParseException; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.brief; import java.io.StringReader; @@ -229,6 +231,7 @@ public Response addDataverse(String body, @PathParam("identifier") String parent @POST @Path("{identifier}/datasets") + @Consumes("application/json") public Response createDataset(String jsonBody, @PathParam("identifier") String parentIdtf) { try { User u = findUserOrDie(); @@ -266,6 +269,45 @@ public Response createDataset(String jsonBody, @PathParam("identifier") String p return ex.getResponse(); } } + + @POST + @Path("{identifier}/datasets") + @Consumes("application/ld+json, application/json-ld") + public Response createDatasetFromJsonLd(String jsonLDBody, @PathParam("identifier") String parentIdtf) { + try { + User u = findUserOrDie(); + Dataverse owner = findDataverseOrDie(parentIdtf); + Dataset ds = new Dataset(); + + ds.setOwner(owner); + ds = JSONLDUtil.updateDatasetMDFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc, false, false); + + ds.setOwner(owner); + + + + // clean possible dataset/version metadata + DatasetVersion version = ds.getVersions().get(0); + version.setMinorVersionNumber(null); + version.setVersionNumber(null); + version.setVersionState(DatasetVersion.VersionState.DRAFT); + + ds.setAuthority(null); + ds.setIdentifier(null); + ds.setProtocol(null); + ds.setGlobalIdCreateTime(null); + + Dataset managedDs = execCommand(new CreateNewDatasetCommand(ds, createDataverseRequest(u))); + return created("/datasets/" + managedDs.getId(), + Json.createObjectBuilder() + .add("id", managedDs.getId()) + .add("persistentId", managedDs.getGlobalIdString()) + ); + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } @POST @Path("{identifier}/datasets/:import") From 13a7841b3e08e84460fbdf1a8a0405adc53bb4a4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 13 Jul 2021 14:41:24 -0400 Subject: [PATCH 302/354] No "@id" npe fix --- .../edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 62d58827696..37122d5bf1e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -78,7 +78,13 @@ public static Dataset updateDatasetMDFromJsonLD(Dataset ds, String jsonLDBody, DatasetVersion dsv = new DatasetVersion(); JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); - Optional maybePid = GlobalId.parse(jsonld.getString("@id")); + String id = null; + try { + id=jsonld.getString("@id"); + } catch (NullPointerException npe) { + //Do nothing - a null value and other invalid values will be caught in parsing + } + Optional maybePid = GlobalId.parse(id); if (maybePid.isPresent()) { ds.setGlobalId(maybePid.get()); } else { From 86a08e3362e84de51226f26cae258b52dab107af Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 13 Jul 2021 14:52:21 -0400 Subject: [PATCH 303/354] avoid npe in logging --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 37122d5bf1e..e50558a8c17 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -89,8 +89,7 @@ public static Dataset updateDatasetMDFromJsonLD(Dataset ds, String jsonLDBody, ds.setGlobalId(maybePid.get()); } else { // unparsable PID passed. Terminate. - throw new BadRequestException("Cannot parse the @id '" + jsonld.getString("@id") - + "'. Make sure it is in valid form - see Dataverse Native API documentation."); + throw new BadRequestException("Cannot parse the @id. Make sure it is in valid form - see Dataverse Native API documentation."); } dsv = updateDatasetVersionMDFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); From 0c64c68dff6be42299f1d0d27a9da3431167632b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 13 Jul 2021 15:39:11 -0400 Subject: [PATCH 304/354] only require "@id" when migrating --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index e50558a8c17..9c0a8a895f6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -88,8 +88,10 @@ public static Dataset updateDatasetMDFromJsonLD(Dataset ds, String jsonLDBody, if (maybePid.isPresent()) { ds.setGlobalId(maybePid.get()); } else { - // unparsable PID passed. Terminate. - throw new BadRequestException("Cannot parse the @id. Make sure it is in valid form - see Dataverse Native API documentation."); + if(migrating) { + // unparsable PID passed. Terminate. + throw new BadRequestException("Cannot parse the @id. Make sure it is in valid form - see Dataverse Native API documentation."); + } } dsv = updateDatasetVersionMDFromJsonLD(dsv, jsonld, metadataBlockSvc, datasetFieldSvc, append, migrating); From 8e9f2f72223453f85dd81ff425c959582a127b9e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 13 Jul 2021 16:36:15 -0400 Subject: [PATCH 305/354] fix logging in create case --- .../harvard/iq/dataverse/util/json/JSONLDUtil.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 9c0a8a895f6..965251e7124 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -3,6 +3,7 @@ import java.io.StringReader; import java.io.StringWriter; import java.sql.Timestamp; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -15,6 +16,7 @@ import java.util.Optional; import java.util.Map.Entry; import java.util.TreeMap; +import java.util.logging.Level; import java.util.logging.Logger; import javax.json.Json; @@ -107,7 +109,15 @@ public static Dataset updateDatasetMDFromJsonLD(Dataset ds, String jsonLDBody, ds.setModificationTime(Timestamp.valueOf(dateTime)); } try { - logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); + if (logger.isLoggable(Level.FINE)) { + if (ds.getModificationTime() == null) { + // Create (migrating==false case - modification time will be set in the create + // call, but we need a non-null value to reuse the OREMap method for logging + // here + ds.setModificationTime(Timestamp.from(Instant.now())); + } + logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); + } } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); From 73244021d7770735fd97188072a7c59edd8a7e09 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 14 Jul 2021 09:49:34 -0400 Subject: [PATCH 306/354] #7858 fix formatting (blanks) --- src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 031bb1dbdec..8eb88c6e87a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2047,9 +2047,9 @@ public void handleExternalUpload() { // ----------------------------------------------------------- // Is this a FileReplaceOperation? If so, then diverge! // ----------------------------------------------------------- - if (this.isFileReplaceOperation()){ + if (this.isFileReplaceOperation()){ this.handleReplaceFileUpload(storageLocation, fileName, contentType, checksumValue, checksumType); - if (!saveEnabled){ + if (!saveEnabled){ uploadWarningMessage = fileReplacePageHelper.getErrorMessages(); return; } From 8771610cd32a65e45cc589dfd8ce16311cfbd0ec Mon Sep 17 00:00:00 2001 From: Thibault Coupin Date: Thu, 15 Jul 2021 17:42:36 +0200 Subject: [PATCH 307/354] Fix #8002 display issue on permissions management screen --- src/main/webapp/roles-assign.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/roles-assign.xhtml b/src/main/webapp/roles-assign.xhtml index e5c66622e5d..4b31f10dbfc 100644 --- a/src/main/webapp/roles-assign.xhtml +++ b/src/main/webapp/roles-assign.xhtml @@ -72,7 +72,7 @@ #{bundle['permission.'.concat(prm).concat('.label')]}
    -

    +

    From 84e5c950eec45e7329771dc67ae3a853bc7a21b9 Mon Sep 17 00:00:00 2001 From: "don.sizemore" Date: Fri, 16 Jul 2021 14:26:34 -0400 Subject: [PATCH 308/354] #7560 document dataverse.files.s3.profile --- doc/sphinx-guides/source/installation/config.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 74119919c96..42ed02c5897 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -541,6 +541,7 @@ dataverse.files..url-expiration-minutes If direct uploa dataverse.files..min-part-size Multipart direct uploads will occur for files larger than this. Optional. ``1024**3`` dataverse.files..custom-endpoint-url Use custom S3 endpoint. Needs URL either with or without protocol. (none) dataverse.files..custom-endpoint-region Only used when using custom endpoint. Optional. ``dataverse`` +dataverse.files..profile Allows the use of AWS profiles for storage spanning multiple AWS accounts. (none) dataverse.files..proxy-url URL of a proxy protecting the S3 store. Optional. (none) dataverse.files..path-style-access ``true``/``false`` Use path style buckets instead of subdomains. Optional. ``false`` dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` @@ -2257,4 +2258,4 @@ In the DDI metadata exports, the default behavior is to always add the repositor A comma-separated list of field type names that should be 'withheld' when dataset access occurs via a Private Url with Anonymized Access (e.g. to support anonymized review). A suggested minimum includes author, datasetContact, and contributor, but additional fields such as depositor, grantNumber, and publication might also need to be included. -``curl -X PUT -d 'author, datasetContact, contributor, depositor, grantNumber, publication' http://localhost:8080/api/admin/settings/:AnonymizedFieldTypeNames`` \ No newline at end of file +``curl -X PUT -d 'author, datasetContact, contributor, depositor, grantNumber, publication' http://localhost:8080/api/admin/settings/:AnonymizedFieldTypeNames`` From 58cd5b6a608522fb75947cdeb3cebe1612786868 Mon Sep 17 00:00:00 2001 From: "don.sizemore" Date: Fri, 16 Jul 2021 15:10:46 -0400 Subject: [PATCH 309/354] #7560 correct Text in column margin in table Sphinx warning --- .../source/installation/config.rst | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 42ed02c5897..b56179ef810 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -529,25 +529,25 @@ Lastly, go ahead and restart your Payara server. With Dataverse deployed and the S3 Storage Options ################## -=========================================== ================== ========================================================================= ============= -JVM Option Value Description Default value -=========================================== ================== ========================================================================= ============= -dataverse.files.storage-driver-id Enable as the default storage driver. ``file`` -dataverse.files..bucket-name The bucket name. See above. (none) -dataverse.files..download-redirect ``true``/``false`` Enable direct download or proxy through Dataverse. ``false`` -dataverse.files..upload-redirect ``true``/``false`` Enable direct upload of files added to a dataset to the S3 store. ``false`` -dataverse.files..ingestsizelimit Maximum size of directupload files that should be ingested (none) -dataverse.files..url-expiration-minutes If direct uploads/downloads: time until links expire. Optional. 60 -dataverse.files..min-part-size Multipart direct uploads will occur for files larger than this. Optional. ``1024**3`` -dataverse.files..custom-endpoint-url Use custom S3 endpoint. Needs URL either with or without protocol. (none) -dataverse.files..custom-endpoint-region Only used when using custom endpoint. Optional. ``dataverse`` -dataverse.files..profile Allows the use of AWS profiles for storage spanning multiple AWS accounts. (none) -dataverse.files..proxy-url URL of a proxy protecting the S3 store. Optional. (none) -dataverse.files..path-style-access ``true``/``false`` Use path style buckets instead of subdomains. Optional. ``false`` -dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` -dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true`` -dataverse.files..connection-pool-size The maximum number of open connections to the S3 server ``256`` -=========================================== ================== ========================================================================= ============= +=========================================== ================== ========================================================================== ============= +JVM Option Value Description Default value +=========================================== ================== ========================================================================== ============= +dataverse.files.storage-driver-id Enable as the default storage driver. ``file`` +dataverse.files..bucket-name The bucket name. See above. (none) +dataverse.files..download-redirect ``true``/``false`` Enable direct download or proxy through Dataverse. ``false`` +dataverse.files..upload-redirect ``true``/``false`` Enable direct upload of files added to a dataset to the S3 store. ``false`` +dataverse.files..ingestsizelimit Maximum size of directupload files that should be ingested (none) +dataverse.files..url-expiration-minutes If direct uploads/downloads: time until links expire. Optional. 60 +dataverse.files..min-part-size Multipart direct uploads will occur for files larger than this. Optional. ``1024**3`` +dataverse.files..custom-endpoint-url Use custom S3 endpoint. Needs URL either with or without protocol. (none) +dataverse.files..custom-endpoint-region Only used when using custom endpoint. Optional. ``dataverse`` +dataverse.files..profile Allows the use of AWS profiles for storage spanning multiple AWS accounts. (none) +dataverse.files..proxy-url URL of a proxy protecting the S3 store. Optional. (none) +dataverse.files..path-style-access ``true``/``false`` Use path style buckets instead of subdomains. Optional. ``false`` +dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` +dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true`` +dataverse.files..connection-pool-size The maximum number of open connections to the S3 server ``256`` +=========================================== ================== ========================================================================== ============= Reported Working S3-Compatible Storage ###################################### From fa1338bbe8291db62d584e2c74522a7a802a0b7c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 19 Jul 2021 14:54:41 -0400 Subject: [PATCH 310/354] handle resolver forms of PIDs for import/harvest --- .../api/imports/ImportDDIServiceBean.java | 47 ++----------------- .../api/imports/ImportGenericServiceBean.java | 7 ++- 2 files changed, 11 insertions(+), 43 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java index 06be5104c6a..49cd1a380c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java @@ -103,6 +103,8 @@ public class ImportDDIServiceBean { @EJB DatasetFieldServiceBean datasetFieldService; + @EJB ImportGenericServiceBean importGenericService; + // TODO: stop passing the xml source as a string; (it could be huge!) -- L.A. 4.5 // TODO: what L.A. Said. @@ -281,7 +283,7 @@ private void processDocDscr(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws // this will set a StudyId if it has not yet been set; it will get overridden by a metadata // id in the StudyDscr section, if one exists if ( AGENCY_HANDLE.equals( xmlr.getAttributeValue(null, "agency") ) ) { - parseStudyIdHandle( parseText(xmlr), datasetDTO ); + importGenericService.reassignIdentifierAsGlobalId( parseText(xmlr), datasetDTO ); } // EMK TODO: we need to save this somewhere when we add harvesting infrastructure } /*else if ( xmlr.getLocalName().equals("holdings") && StringUtil.isEmpty(datasetDTO..getHarvestHoldings()) ) { @@ -1409,10 +1411,8 @@ private void processTitlStmt(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws FieldDTO field = FieldDTO.createPrimitiveFieldDTO("alternativeTitle", parseText(xmlr)); citation.getFields().add(field); } else if (xmlr.getLocalName().equals("IDNo")) { - if ( AGENCY_HANDLE.equals( xmlr.getAttributeValue(null, "agency") ) ) { - parseStudyIdHandle( parseText(xmlr), datasetDTO ); - } else if ( AGENCY_DOI.equals( xmlr.getAttributeValue(null, "agency") ) ) { - parseStudyIdDOI( parseText(xmlr), datasetDTO ); + if ( AGENCY_HANDLE.equals( xmlr.getAttributeValue(null, "agency") ) || AGENCY_DOI.equals( xmlr.getAttributeValue(null, "agency") ) ) { + importGenericService.reassignIdentifierAsGlobalId(parseText(xmlr), datasetDTO); } else if ( AGENCY_DARA.equals( xmlr.getAttributeValue(null, "agency"))) { /* da|ra - "Registration agency for social and economic data" @@ -1689,43 +1689,6 @@ else if (xmlr.getLocalName().equals("notes")) { return returnValues; } - private void parseStudyIdHandle(String _id, DatasetDTO datasetDTO) { - - int index1 = _id.indexOf(':'); - int index2 = _id.indexOf('/'); - if (index1==-1) { - throw new EJBException("Error parsing (Handle) IdNo: "+_id+". ':' not found in string"); - } else { - datasetDTO.setProtocol(_id.substring(0,index1)); - } - if (index2 == -1) { - throw new EJBException("Error parsing (Handle) IdNo: "+_id+". '/' not found in string"); - - } else { - datasetDTO.setAuthority(_id.substring(index1+1, index2)); - } - datasetDTO.setProtocol("hdl"); - datasetDTO.setIdentifier(_id.substring(index2+1)); - } - - private void parseStudyIdDOI(String _id, DatasetDTO datasetDTO) throws ImportException{ - int index1 = _id.indexOf(':'); - int index2 = _id.indexOf('/'); - if (index1==-1) { - throw new EJBException("Error parsing (DOI) IdNo: "+_id+". ':' not found in string"); - } - - if (index2 == -1) { - throw new ImportException("Error parsing (DOI) IdNo: "+_id+". '/' not found in string"); - - } else { - datasetDTO.setAuthority(_id.substring(index1+1, index2)); - } - datasetDTO.setProtocol("doi"); - - datasetDTO.setIdentifier(_id.substring(index2+1)); - } - private void parseStudyIdDoiICPSRdara(String _id, DatasetDTO datasetDTO) throws ImportException{ /* dara/ICPSR DOIs are formatted without the hdl: prefix; for example - diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index 84195227b33..acce782de3d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -407,7 +407,12 @@ private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) { return null; } - private String reassignIdentifierAsGlobalId(String identifierString, DatasetDTO datasetDTO) { + /* This is a general parser that can take DOI and Handle Ids, in their local or + * URL forms (e.g. doi:... or https://doi.org/...) and parse them into + * protocol/authority/identifier parts that are assigned to the datasetDTO. + * The name reflects the original purpose but it is now used in ImportDDIServiceBean as well. + */ + String reassignIdentifierAsGlobalId(String identifierString, DatasetDTO datasetDTO) { int index1 = identifierString.indexOf(':'); int index2 = identifierString.indexOf('/'); From 20af2f4feeae7b4ae41d598034a57b751f1dc8d5 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 19 Jul 2021 15:57:18 -0400 Subject: [PATCH 311/354] change to public access --- .../iq/dataverse/api/imports/ImportGenericServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index acce782de3d..bd7975835e3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -412,7 +412,7 @@ private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) { * protocol/authority/identifier parts that are assigned to the datasetDTO. * The name reflects the original purpose but it is now used in ImportDDIServiceBean as well. */ - String reassignIdentifierAsGlobalId(String identifierString, DatasetDTO datasetDTO) { + public String reassignIdentifierAsGlobalId(String identifierString, DatasetDTO datasetDTO) { int index1 = identifierString.indexOf(':'); int index2 = identifierString.indexOf('/'); From 9a25e8dc9d2bc409ac450f9a9a8437dba1e84bac Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 20 Jul 2021 10:03:20 -0400 Subject: [PATCH 312/354] revert IDNo pid format change to stay harvest compatible --- doc/release-notes/7786-ddi_changes.md | 3 ++- .../edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 5 +++-- .../edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml | 2 +- .../java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/release-notes/7786-ddi_changes.md b/doc/release-notes/7786-ddi_changes.md index 18bf2d2183a..c6b0d455ddf 100644 --- a/doc/release-notes/7786-ddi_changes.md +++ b/doc/release-notes/7786-ddi_changes.md @@ -1,8 +1,9 @@ +Note: These notes cover several related PRs (#7984, #7958, #7959 respectively for the three bullets below.) If some are not merged before the next release, these notes should be adjusted. + ### Enhancements to DDI Metadata Exports Several changes have been made to the DDI exports to improve support for internationalization and to improve compliance with CESSDA requirements. These changes include: * Addition of a holdings element with a URI attribute whose value is the URL form of the dataset PID -* Change to use the URL form of the dataset PID in the \ IDNo element * Addition of xml:lang attributes specifying the dataset metadata language at the document level and for individual elements such as title and description * Specification of controlled vocabulary terms in duplicate elements in multiple languages (in the installation default langauge and, if different, the dataset metadata language) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 97237933826..693e3cd6b10 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -168,7 +168,8 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) String persistentAuthority = datasetDto.getAuthority(); String persistentId = datasetDto.getIdentifier(); - String pidUri = persistentProtocol + ":" + persistentAuthority + "/" + persistentId; + String pid = persistentProtocol + ":" + persistentAuthority + "/" + persistentId; + String pidUri = pid; //Some tests don't send real PIDs - don't try to get their URL form if(!pidUri.equals("null:null/null")) { pidUri= new GlobalId(persistentProtocol + ":" + persistentAuthority + "/" + persistentId).toURL().toString(); @@ -198,7 +199,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeAttribute(xmlw, "agency", persistentAgency); - xmlw.writeCharacters(pidUri); + xmlw.writeCharacters(pid); xmlw.writeEndElement(); // IDNo writeOtherIdElement(xmlw, version); xmlw.writeEndElement(); // titlStmt diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml index ba16a6e4991..6e4d2638e52 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml @@ -17,7 +17,7 @@ Darwin's Finches - https://doi.org/10.5072/FK2/PCA2E3 + doi:10.5072/FK2/PCA2E3 Finch, Fiona diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml index 5dfb763faa1..0bd74d6f198 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml @@ -22,7 +22,7 @@ Replication Data for: Title Subtitle Alternative Title - https://doi.org/10.5072/FK2/WKUKGV + doi:10.5072/FK2/WKUKGV OtherIDIdentifier1 OtherIDIdentifier2 From b2b34bad386c033a7fd395f6e3b3a2881601c622 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 20 Jul 2021 15:30:25 -0400 Subject: [PATCH 313/354] reword request access to work with file or files #7964 --- src/main/java/propertyFiles/Bundle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 261f6d04800..02b477c008d 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1656,8 +1656,8 @@ file.downloadBtn.format.citation=Data File Citation file.download.filetype.unknown=Original File Format file.more.information.link=Link to more file information for file.requestAccess=Request Access -file.requestAccess.dialog.msg=You need to Log In to request access to this file. -file.requestAccess.dialog.msg.signup=You need to Sign Up or Log In to request access to this file. +file.requestAccess.dialog.msg=You need to Log In to request access. +file.requestAccess.dialog.msg.signup=You need to Sign Up or Log In to request access. file.accessRequested=Access Requested file.ingestInProgress=Ingest in progress... file.dataFilesTab.metadata.header=Metadata From 1a348f5f3660916a538a9d4f3fc1ed062fac83ab Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 14 Apr 2021 19:20:22 +0200 Subject: [PATCH 314/354] Non-integer identifiers using stored procedure This commit replaces `sequentialNumber` `:IdentifierGenerationStyle` with `storedProcGenerated`, and allows generating non-integer identifiers. It also adds a Flyway script to manage this migration. Fixes #7548. --- .../iq/dataverse/DataFileServiceBean.java | 18 +++++----- .../edu/harvard/iq/dataverse/Dataset.java | 27 ++++++--------- .../iq/dataverse/DatasetServiceBean.java | 14 ++++---- .../iq/dataverse/util/SystemConfig.java | 2 +- ...V5.5.0.5__7548-stored-procedure-update.sql | 33 +++++++++++++++++++ .../harvard/iq/dataverse/api/DatasetsIT.java | 6 ++-- 6 files changed, 63 insertions(+), 37 deletions(-) create mode 100644 src/main/resources/db/migration/V5.5.0.5__7548-stored-procedure-update.sql diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index c8c18d2d2fc..83a65110be2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -1428,11 +1428,11 @@ public String generateDataFileIdentifier(DataFile datafile, GlobalIdServiceBean switch (doiIdentifierType) { case "randomString": return generateIdentifierAsRandomString(datafile, idServiceBean, prepend); - case "sequentialNumber": + case "storedProcGenerated": if (doiDataFileFormat.equals(SystemConfig.DataFilePIDFormat.INDEPENDENT.toString())){ - return generateIdentifierAsIndependentSequentialNumber(datafile, idServiceBean, prepend); + return generateIdentifierFromStoredProcedureIndependent(datafile, idServiceBean, prepend); } else { - return generateIdentifierAsDependentSequentialNumber(datafile, idServiceBean, prepend); + return generateIdentifierFromStoredProcedureDependent(datafile, idServiceBean, prepend); } default: /* Should we throw an exception instead?? -- L.A. 4.6.2 */ @@ -1450,24 +1450,24 @@ private String generateIdentifierAsRandomString(DataFile datafile, GlobalIdServi } - private String generateIdentifierAsIndependentSequentialNumber(DataFile datafile, GlobalIdServiceBean idServiceBean, String prepend) { + private String generateIdentifierFromStoredProcedureIndependent(DataFile datafile, GlobalIdServiceBean idServiceBean, String prepend) { String identifier; do { - StoredProcedureQuery query = this.em.createNamedStoredProcedureQuery("Dataset.generateIdentifierAsSequentialNumber"); + StoredProcedureQuery query = this.em.createNamedStoredProcedureQuery("Dataset.generateIdentifierFromStoredProcedure"); query.execute(); - Integer identifierNumeric = (Integer) query.getOutputParameterValue(1); + String identifierFromStoredProcedure = (String) query.getOutputParameterValue(1); // some diagnostics here maybe - is it possible to determine that it's failing // because the stored procedure hasn't been created in the database? - if (identifierNumeric == null) { + if (identifierFromStoredProcedure == null) { return null; } - identifier = prepend + identifierNumeric.toString(); + identifier = prepend + identifierFromStoredProcedure; } while (!isGlobalIdUnique(identifier, datafile, idServiceBean)); return identifier; } - private String generateIdentifierAsDependentSequentialNumber(DataFile datafile, GlobalIdServiceBean idServiceBean, String prepend) { + private String generateIdentifierFromStoredProcedureDependent(DataFile datafile, GlobalIdServiceBean idServiceBean, String prepend) { String identifier; Long retVal; diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index cd40e76a304..69471706904 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -60,29 +60,22 @@ }) /* - Below is the stored procedure for getting a numeric value from a database - sequence. Used when the Dataverse is (optionally) configured to use - incremental numeric values for dataset ids, instead of the default + Below is the database stored procedure for getting a string dataset id. + Used when the Dataverse is (optionally) configured to use + procedurally generated values for dataset ids, instead of the default random strings. - Unfortunately, there's no standard EJB way of handling sequences. So in the - past we would simply use a NativeQuery to call a proprietary Postgres - sequence query. A better way of handling this however is to define any - proprietary SQL functionality outside of the application, in the database, - and call it using the standard JPA @StoredProcedureQuery. - - The identifier sequence and the stored procedure for accessing it are currently - implemented with PostgresQL "CREATE SEQUENCE ..." and "CREATE FUNCTION ..."; - (we explain how to create these in the installation documentation and supply - a script). If necessary, it can be implemented using other SQL flavors - + The use of a stored procedure to create an identifier is explained in the + installation documentation (where an example script is supplied). + The stored procedure can be implemented using other SQL flavors - without having to modify the application code. - -- L.A. 4.6.2 + -- L.A. 4.6.2 (modified by C.S. for version 5.4.1+) */ @NamedStoredProcedureQuery( - name = "Dataset.generateIdentifierAsSequentialNumber", - procedureName = "generateIdentifierAsSequentialNumber", + name = "Dataset.generateIdentifierFromStoredProcedure", + procedureName = "generateIdentifierFromStoredProcedure", parameters = { - @StoredProcedureParameter(mode = ParameterMode.OUT, type = Integer.class) + @StoredProcedureParameter(mode = ParameterMode.OUT, type = String.class) } ) @Entity diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 685017200b5..e7669dc4c48 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -273,8 +273,8 @@ public String generateDatasetIdentifier(Dataset dataset, GlobalIdServiceBean idS switch (identifierType) { case "randomString": return generateIdentifierAsRandomString(dataset, idServiceBean, shoulder); - case "sequentialNumber": - return generateIdentifierAsSequentialNumber(dataset, idServiceBean, shoulder); + case "storedProcGenerated": + return generateIdentifierFromStoredProcedure(dataset, idServiceBean, shoulder); default: /* Should we throw an exception instead?? -- L.A. 4.6.2 */ return generateIdentifierAsRandomString(dataset, idServiceBean, shoulder); @@ -290,19 +290,19 @@ private String generateIdentifierAsRandomString(Dataset dataset, GlobalIdService return identifier; } - private String generateIdentifierAsSequentialNumber(Dataset dataset, GlobalIdServiceBean idServiceBean, String shoulder) { + private String generateIdentifierFromStoredProcedure(Dataset dataset, GlobalIdServiceBean idServiceBean, String shoulder) { String identifier; do { - StoredProcedureQuery query = this.em.createNamedStoredProcedureQuery("Dataset.generateIdentifierAsSequentialNumber"); + StoredProcedureQuery query = this.em.createNamedStoredProcedureQuery("Dataset.generateIdentifierFromStoredProcedure"); query.execute(); - Integer identifierNumeric = (Integer) query.getOutputParameterValue(1); + String identifierFromStoredProcedure = (String) query.getOutputParameterValue(1); // some diagnostics here maybe - is it possible to determine that it's failing // because the stored procedure hasn't been created in the database? - if (identifierNumeric == null) { + if (identifierFromStoredProcedure == null) { return null; } - identifier = shoulder + identifierNumeric.toString(); + identifier = shoulder + identifierFromStoredProcedure; } while (!isIdentifierLocallyUnique(identifier, dataset)); return identifier; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 71accdaa8c8..9fb66101883 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -1017,7 +1017,7 @@ public Integer getUploadMethodCount(){ public boolean isDataFilePIDSequentialDependent(){ String doiIdentifierType = settingsService.getValueForKey(SettingsServiceBean.Key.IdentifierGenerationStyle, "randomString"); String doiDataFileFormat = settingsService.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT"); - if (doiIdentifierType.equals("sequentialNumber") && doiDataFileFormat.equals("DEPENDENT")){ + if (doiIdentifierType.equals("storedProcGenerated") && doiDataFileFormat.equals("DEPENDENT")){ return true; } return false; diff --git a/src/main/resources/db/migration/V5.5.0.5__7548-stored-procedure-update.sql b/src/main/resources/db/migration/V5.5.0.5__7548-stored-procedure-update.sql new file mode 100644 index 00000000000..6377f2ddec2 --- /dev/null +++ b/src/main/resources/db/migration/V5.5.0.5__7548-stored-procedure-update.sql @@ -0,0 +1,33 @@ +-- If the installation is using a stored procedure for generating +-- sequential numeric identifiers, create a wrapper function that +-- works with the new framework (the stored procedure now needs to +-- return a string) and update the database setting +DO $BODY$ +BEGIN + UPDATE setting SET content='storedProcGenerated' + WHERE name=':IdentifierGenerationStyle' + AND content='sequentialNumber'; + BEGIN + PERFORM generateIdentifierAsSequentialNumber(); + EXCEPTION + -- If the above function does not exist, we can stop executing this script + WHEN undefined_function THEN + RETURN; + END; + BEGIN + PERFORM generateIdentifierFromStoredProcedure(); + EXCEPTION + -- We only create this function if it doesn't already exist, + -- to avoid overwriting user modifications + WHEN undefined_function THEN + CREATE FUNCTION generateIdentifierFromStoredProcedure() + RETURNS varchar AS $$ + DECLARE + identifier varchar; + BEGIN + identifier := generateIdentifierAsSequentialNumber()::varchar; + RETURN identifier; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + END; +END $BODY$; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index a7dc41d9c74..956cce5eecc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -733,7 +733,7 @@ public void testExcludeEmail() { } @Test - public void testSequentialNumberAsIdentifierGenerationStyle() { + public void testStoredProcGeneratedAsIdentifierGenerationStyle() { Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); @@ -744,8 +744,8 @@ public void testSequentialNumberAsIdentifierGenerationStyle() { createDataverseResponse.prettyPrint(); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - Response setSequentialNumberAsIdentifierGenerationStyle = UtilIT.setSetting(SettingsServiceBean.Key.IdentifierGenerationStyle, "sequentialNumber"); - setSequentialNumberAsIdentifierGenerationStyle.then().assertThat() + Response setStoredProcGeneratedAsIdentifierGenerationStyle = UtilIT.setSetting(SettingsServiceBean.Key.IdentifierGenerationStyle, "storedProcGenerated"); + setStoredProcGeneratedAsIdentifierGenerationStyle.then().assertThat() .statusCode(OK.getStatusCode()); Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); From f08b2e47a96e3990a65bdb4d0463019139760e3e Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 21 Jul 2021 10:30:46 +0200 Subject: [PATCH 315/354] Doc for string identifiers using stored procedure Update doc for :IdentifierGenerationStyle and :DataFilePIDFormat. Update the existing example (sequential numerical values) and added a new one (base36 timestamp). Fixes #7548. --- .../source/_static/util/createsequence.sql | 24 ++-- .../util/identifier_from_timestamp.sql | 46 +++++++ .../source/installation/config.rst | 113 +++++++++++++----- 3 files changed, 138 insertions(+), 45 deletions(-) create mode 100644 doc/sphinx-guides/source/_static/util/identifier_from_timestamp.sql diff --git a/doc/sphinx-guides/source/_static/util/createsequence.sql b/doc/sphinx-guides/source/_static/util/createsequence.sql index 2677832abd8..7ac1968de2c 100644 --- a/doc/sphinx-guides/source/_static/util/createsequence.sql +++ b/doc/sphinx-guides/source/_static/util/createsequence.sql @@ -1,14 +1,14 @@ --- A script for creating a numeric identifier sequence, and an external --- stored procedure, for accessing the sequence from inside the application, --- in a non-hacky, JPA way. +-- A script for creating a numeric identifier sequence, and an external +-- stored procedure, for accessing the sequence from inside the application, +-- in a non-hacky, JPA way. -- NOTE: -- 1. The database user name "dvnapp" is hard-coded here - it may -- need to be changed to match your database user name; - + -- 2. In the code below, the sequence starts with 1, but it can be adjusted by --- changing the MINVALUE as needed. +-- changing the MINVALUE as needed. CREATE SEQUENCE datasetidentifier_seq INCREMENT 1 @@ -22,12 +22,12 @@ ALTER TABLE datasetidentifier_seq OWNER TO "dvnapp"; -- And now create a PostgreSQL FUNCTION, for JPA to -- access as a NamedStoredProcedure: -CREATE OR REPLACE FUNCTION generateIdentifierAsSequentialNumber( - OUT identifier int) - RETURNS int AS -$BODY$ +CREATE OR REPLACE FUNCTION generateIdentifierFromStoredProcedure() +RETURNS varchar AS $$ +DECLARE + identifier varchar; BEGIN - select nextval('datasetidentifier_seq') into identifier; + identifier := nextval('datasetidentifier_seq')::varchar; + RETURN identifier; END; -$BODY$ - LANGUAGE plpgsql; +$$ LANGUAGE plpgsql IMMUTABLE; diff --git a/doc/sphinx-guides/source/_static/util/identifier_from_timestamp.sql b/doc/sphinx-guides/source/_static/util/identifier_from_timestamp.sql new file mode 100644 index 00000000000..a755b5ecd4a --- /dev/null +++ b/doc/sphinx-guides/source/_static/util/identifier_from_timestamp.sql @@ -0,0 +1,46 @@ +-- A script for creating, through a database stored procedure, sequential +-- 8 character identifiers from a base36 representation of current timestamp. + +CREATE OR REPLACE FUNCTION base36_encode( + IN digits bigint, IN min_width int = 0) +RETURNS varchar AS $$ +DECLARE + chars char[]; + ret varchar; + val bigint; +BEGIN + chars := ARRAY[ + '0','1','2','3','4','5','6','7','8','9', + 'a','b','c','d','e','f','g','h','i','j', + 'k','l','m','n','o','p','q','r','s','t', + 'u','v','w','x','y','z']; + val := digits; + ret := ''; + IF val < 0 THEN + val := val * -1; + END IF; + WHILE val != 0 LOOP + ret := chars[(val % 36)+1] || ret; + val := val / 36; + END LOOP; + + IF min_width > 0 AND char_length(ret) < min_width THEN + ret := lpad(ret, min_width, '0'); + END IF; + + RETURN ret; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +CREATE OR REPLACE FUNCTION generateIdentifierFromStoredProcedure() +RETURNS varchar AS $$ +DECLARE + curr_time_msec bigint; + identifier varchar; +BEGIN + curr_time_msec := extract(epoch from now())*1000; + identifier := base36_encode(curr_time_msec); + RETURN identifier; +END; +$$ LANGUAGE plpgsql IMMUTABLE; diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 39f27f749dc..fae538b7ff8 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1476,49 +1476,96 @@ Out of the box, the DOI shoulder is set to "FK2/" but this is for testing only! :IdentifierGenerationStyle ++++++++++++++++++++++++++ -By default, the Dataverse Software generates a random 6 character string, pre-pended by the Shoulder if set, to use as the identifier -for a Dataset. Set this to ``sequentialNumber`` to use sequential numeric values -instead (again pre-pended by the Shoulder if set). (the assumed default setting is ``randomString``). -In addition to this setting, a database sequence must be created in the database. -We provide the script below (downloadable :download:`here `). -You may need to make some changes to suit your system setup, see the comments for more information: +By default, the Dataverse Software generates a random 6 character string, +pre-pended by the Shoulder if set, to use as the identifier for a Dataset. +Set this to ``storedProcGenerated`` to generate instead a custom *unique* +identifier (again pre-pended by the Shoulder if set) through a database +stored procedure or function (the assumed default setting is ``randomString``). +In addition to this setting, a stored procedure or function must be created in +the database. + +As a first example, the script below (downloadable +:download:`here `) produces +sequential numerical values. You may need to make some changes to suit your +system setup, see the comments for more information: .. literalinclude:: ../_static/util/createsequence.sql + :language: plpgsql -Note that the SQL above is Postgres-specific. If necessary, it can be reimplemented -in any other SQL flavor - the standard JPA code in the application simply expects -the database to have a saved function ("stored procedure") named ``generateIdentifierAsSequentialNumber`` -with the single return argument ``identifier``. +As a second example, the script below (downloadable +:download:`here `) produces +sequential 8 character identifiers from a base36 representation of current +timestamp. -Please note that ``:IdentifierGenerationStyle`` also plays a role for the "identifier" for files. See the section on ``:DataFilePIDFormat`` below for more details. +.. literalinclude:: ../_static/util/identifier_from_timestamp.sql + :language: plpgsql + +Note that the SQL in these examples scripts is Postgres-specific. +If necessary, it can be reimplemented in any other SQL flavor - the standard +JPA code in the application simply expects the database to have a saved +function ("stored procedure") named ``generateIdentifierFromStoredProcedure()`` +returning a single ``varchar`` argument. + +Please note that ``:IdentifierGenerationStyle`` also plays a role for the +"identifier" for files. See the section on :ref:`:DataFilePIDFormat` below for +more details. .. _:DataFilePIDFormat: :DataFilePIDFormat ++++++++++++++++++ -This setting controls the way that the "identifier" component of a file's persistent identifier (PID) relates to the PID of its "parent" dataset. - -By default the identifier for a file is dependent on its parent dataset. For example, if the identifier of a dataset is "TJCLKP", the identifier for a file within that dataset will consist of the parent dataset's identifier followed by a slash ("/"), followed by a random 6 character string, yielding "TJCLKP/MLGWJO". Identifiers in this format are what you should expect if you leave ``:DataFilePIDFormat`` undefined or set it to ``DEPENDENT`` and have not changed the ``:IdentifierGenerationStyle`` setting from its default. - -Alternatively, the identifier for File PIDs can be configured to be independent of Dataset PIDs using the setting "``INDEPENDENT``". In this case, file PIDs will not contain the PIDs of their parent datasets, and their PIDs will be generated the exact same way that datasets' PIDs are, based on the ``:IdentifierGenerationStyle`` setting described above (random 6 character strings or sequential numbers, pre-pended by any shoulder). - -The chart below shows examples from each possible combination of parameters from the two settings. ``:IdentifierGenerationStyle`` can be either ``randomString`` (the default) or ``sequentialNumber`` and ``:DataFilePIDFormat`` can be either ``DEPENDENT`` (the default) or ``INDEPENDENT``. In the examples below the "identifier" for the dataset is "TJCLKP" for "randomString" and "100001" for "sequentialNumber". - -+-----------------+---------------+------------------+ -| | randomString | sequentialNumber | -| | | | -+=================+===============+==================+ -| **DEPENDENT** | TJCLKP/MLGWJO | 100001/1 | -+-----------------+---------------+------------------+ -| **INDEPENDENT** | MLGWJO | 100002 | -+-----------------+---------------+------------------+ - -As seen above, in cases where ``:IdentifierGenerationStyle`` is set to *sequentialNumber* and ``:DataFilePIDFormat`` is set to *DEPENDENT*, each file within a dataset will be assigned a number *within* that dataset starting with "1". - -Otherwise, if ``:DataFilePIDFormat`` is set to *INDEPENDENT*, then each file will be assigned a PID with the next number in the overall sequence, regardless of what dataset it is in. If the file is created after a dataset with the PID 100001, then the file will be assigned the PID 100002. This option is functional, but it is not a recommended use case. - -Note that in either case, when using the ``sequentialNumber`` option, datasets and files share the same database sequence that was created as part of the setup described in ``:IdentifierGenerationStyle`` above. +This setting controls the way that the "identifier" component of a file's +persistent identifier (PID) relates to the PID of its "parent" dataset. + +By default the identifier for a file is dependent on its parent dataset. +For example, if the identifier of a dataset is "TJCLKP", the identifier for +a file within that dataset will consist of the parent dataset's identifier +followed by a slash ("/"), followed by a random 6 character string, +yielding "TJCLKP/MLGWJO". Identifiers in this format are what you should +expect if you leave ``:DataFilePIDFormat`` undefined or set it to +``DEPENDENT`` and have not changed the ``:IdentifierGenerationStyle`` +setting from its default. + +Alternatively, the identifier for File PIDs can be configured to be +independent of Dataset PIDs using the setting ``INDEPENDENT``. +In this case, file PIDs will not contain the PIDs of their parent datasets, +and their PIDs will be generated the exact same way that datasets' PIDs are, +based on the ``:IdentifierGenerationStyle`` setting described above +(random 6 character strings or custom unique identifiers through a stored +procedure, pre-pended by any shoulder). + +The chart below shows examples from each possible combination of parameters +from the two settings. ``:IdentifierGenerationStyle`` can be either +``randomString`` (the default) or ``storedProcGenerated`` and +``:DataFilePIDFormat`` can be either ``DEPENDENT`` (the default) or +``INDEPENDENT``. In the examples below the "identifier" for the dataset is +"TJCLKP" for ``randomString`` and "100001" for ``storedProcGenerated`` (when +using sequential numerical values, as described in +:ref:`:IdentifierGenerationStyle` above), or "krby26qt" for +``storedProcGenerated`` (when using base36 timestamps, as described in +:ref:`:IdentifierGenerationStyle` above). + ++-----------------+---------------+----------------------+---------------------+ +| | randomString | storedProcGenerated | storedProcGenerated | +| | | | | +| | | (sequential numbers) | (base36 timestamps) | ++=================+===============+======================+=====================+ +| **DEPENDENT** | TJCLKP/MLGWJO | 100001/1 | krby26qt/1 | ++-----------------+---------------+----------------------+---------------------+ +| **INDEPENDENT** | MLGWJO | 100002 | krby27pz | ++-----------------+---------------+----------------------+---------------------+ + +As seen above, in cases where ``:IdentifierGenerationStyle`` is set to +``storedProcGenerated`` and ``:DataFilePIDFormat`` is set to ``DEPENDENT``, +each file within a dataset will be assigned a number *within* that dataset +starting with "1". + +Otherwise, if ``:DataFilePIDFormat`` is set to ``INDEPENDENT``, each file +within the dataset is assigned with a new PID which is the next available +identifier provided from the database stored procedure. In our example: +"100002" when using sequential numbers or "krby27pz" when using base36 +timestamps. .. _:FilePIDsEnabled: From 6a7066e4027fce6ac66ddd77c917390407580aad Mon Sep 17 00:00:00 2001 From: landreev Date: Tue, 20 Jul 2021 17:45:55 -0400 Subject: [PATCH 316/354] =?UTF-8?q?Added=20a=20comment=20to=20the=20stored?= =?UTF-8?q?ProcGenerated=20test=C2=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #7548 --- .../edu/harvard/iq/dataverse/api/DatasetsIT.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 956cce5eecc..03a43a90dec 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -734,7 +734,18 @@ public void testExcludeEmail() { @Test public void testStoredProcGeneratedAsIdentifierGenerationStyle() { - + // Please note that this test only works if the stored procedure + // named generateIdentifierFromStoredProcedure() has been created in the + // database (see the documentation for the "IdentifierGenerationStyle" option + // in the Configuration section of the Installation guide). + // Furthermore, the test below expects the identifier generated by the stored + // procedure to be a numeric string. The "sequential numerical values" procedure, + // documented as the first example in the guide above, will satisfy the test. + // If your installation is using a stored procedure that generates the identifiers + // in any other format, the test below will fail. + // (The test is mainly used by the Dataverse Project's automated API test system + // to verify the integrity of this framework for generating the identifiers on the + // database side). Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); String username = UtilIT.getUsernameFromResponse(createUser); From 7b1e24d994660a1b765385ee9e3bf1b0eec44afa Mon Sep 17 00:00:00 2001 From: landreev Date: Tue, 20 Jul 2021 18:21:44 -0400 Subject: [PATCH 317/354] added the release note #7548 --- doc/release-notes/7548-stored-procedure-update.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 doc/release-notes/7548-stored-procedure-update.md diff --git a/doc/release-notes/7548-stored-procedure-update.md b/doc/release-notes/7548-stored-procedure-update.md new file mode 100644 index 00000000000..26d2a8ab784 --- /dev/null +++ b/doc/release-notes/7548-stored-procedure-update.md @@ -0,0 +1,15 @@ +### Upgrade Notes + +**If your installation relies on the database-side stored procedure for generating sequential numeric identifiers:** + +*(Note: You can skip the following paragraph if your installation uses the default-style, randomly-generated six alphanumeric +character-long identifiers for your datasets!)* + +The underlying database framework has been modified in this release, to make it easier for installations +to create custom procedures for generating identifier strings that suit their needs. Your current configuration will +be automatically updated by the database upgrade (Flyway) script incorporated in the release. No manual configuration +changes should be necessary. However, after the upgrade, we recommend that you confirm that your installation can still +create new datasets, and that they are still assigned sequential numeric identifiers. In the unlikely chance that this +is no longer working, please re-create the stored procedure following the steps described in the documentation for the +`:IdentifierGenerationStyle` setting in the *Configuration* section of the Installation Guide for this release (v5.6). +(Running the script supplied there will NOT overwrite the position on the sequence you are currently using!) From bf360f978d8334975aa81620126d19745d222e3a Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 21 Jul 2021 10:54:38 -0400 Subject: [PATCH 318/354] #7858 updates from CR --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 7 +++---- .../java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 8 +++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 5f6fb449be7..4f73e64d5a4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1904,11 +1904,10 @@ private String init(boolean initFull) { setHasRsyncScript(false); } } catch (RuntimeException ex) { - logger.warning("Problem getting rsync script: " + ex.getLocalizedMessage()); - FacesContext.getCurrentInstance().addMessage("Dataset Page Runtime", new FacesMessage("message") - ); + logger.warning("Problem getting rsync script(RuntimeException): " + ex.getLocalizedMessage()); + FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_ERROR, "Problem getting rsync script:", ex.getLocalizedMessage())); } catch (CommandException cex) { - logger.warning("Problem getting rsync script (Command Exception): -catching in page.." + cex.getLocalizedMessage()); + logger.warning("Problem getting rsync script (Command Exception): " + cex.getLocalizedMessage()); FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_ERROR, "Problem getting rsync script:", cex.getLocalizedMessage())); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 8eb88c6e87a..d2620d9a240 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2049,11 +2049,9 @@ public void handleExternalUpload() { // ----------------------------------------------------------- if (this.isFileReplaceOperation()){ this.handleReplaceFileUpload(storageLocation, fileName, contentType, checksumValue, checksumType); - if (!saveEnabled){ - uploadWarningMessage = fileReplacePageHelper.getErrorMessages(); - return; - } - this.setFileMetadataSelectedForTagsPopup(fileReplacePageHelper.getNewFileMetadatasBeforeSave().get(0)); + if (fileReplacePageHelper.getNewFileMetadatasBeforeSave() != null){ + this.setFileMetadataSelectedForTagsPopup(fileReplacePageHelper.getNewFileMetadatasBeforeSave().get(0)); + } return; } // ----------------------------------------------------------- From b0b94252a25bf92e2b9826d5c1d6f8af2951b27b Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 21 Jul 2021 11:38:22 -0400 Subject: [PATCH 319/354] Moves the default zip download value (100MB) to where we normally keep all the defaults. (#7932) --- src/main/java/edu/harvard/iq/dataverse/api/Access.java | 9 ++------- .../harvard/iq/dataverse/dataaccess/DataFileZipper.java | 1 - .../java/edu/harvard/iq/dataverse/util/SystemConfig.java | 6 +++--- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index bcb6f3d6c61..9fd63a5fe04 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -748,13 +748,8 @@ public Response datafiles(@PathParam("fileIds") String fileIds, @QueryParam("gbr } private Response downloadDatafiles(String rawFileIds, boolean donotwriteGBResponse, String apiTokenParam, UriInfo uriInfo, HttpHeaders headers, HttpServletResponse response) throws WebApplicationException /* throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { - long setLimit = systemConfig.getZipDownloadLimit(); - if (!(setLimit > 0L)) { - setLimit = DataFileZipper.DEFAULT_ZIPFILE_LIMIT; - } - - final long zipDownloadSizeLimit = setLimit; //to use via anon inner class - + final long zipDownloadSizeLimit = systemConfig.getZipDownloadLimit(); + logger.fine("setting zip download size limit to " + zipDownloadSizeLimit + " bytes."); if (rawFileIds == null || rawFileIds.equals("")) { diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java index 15ff7b5ac99..68553f7f5c8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java @@ -39,7 +39,6 @@ * @author Leonid Andreev */ public class DataFileZipper { - public static long DEFAULT_ZIPFILE_LIMIT = 100 * 1024 * 1024; // 100MB private static final Logger logger = Logger.getLogger(DataFileZipper.class.getCanonicalName()); private static final String MANIFEST_FILE_NAME = "MANIFEST.TXT"; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 0764d07e8fa..48607976171 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -93,6 +93,7 @@ public class SystemConfig { * zip file upload. */ private static final int defaultZipUploadFilesLimit = 1000; + private static final long defaultZipDownloadLimit = 104857600L; // 100MB private static final int defaultMultipleUploadFilesLimit = 1000; private static final int defaultLoginSessionTimeout = 480; // = 8 hours @@ -423,13 +424,12 @@ static int getIntLimitFromStringOrDefault(String limitSetting, Integer defaultVa /** * Download-as-zip size limit. - * returns 0 if not specified; - * (the file zipper will then use the default value) + * returns defaultZipDownloadLimit if not specified; * set to -1 to disable zip downloads. */ public long getZipDownloadLimit() { String zipLimitOption = settingsService.getValueForKey(SettingsServiceBean.Key.ZipDownloadLimit); - return getLongLimitFromStringOrDefault(zipLimitOption, 0L); + return getLongLimitFromStringOrDefault(zipLimitOption, defaultZipDownloadLimit); } public int getZipUploadFilesLimit() { From b1acb2b824a05864618b0bfb48c3bf77aa50a824 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 21 Jul 2021 11:44:19 -0400 Subject: [PATCH 320/354] add dev guide links to list of APIs #6497 --- doc/sphinx-guides/source/api/intro.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst index 101c6c2bfaa..933932cd7b9 100755 --- a/doc/sphinx-guides/source/api/intro.rst +++ b/doc/sphinx-guides/source/api/intro.rst @@ -204,6 +204,15 @@ Please note that some APIs are only documented in other guides that are more sui - :doc:`/installation/config` +- Developer Guide + + - :doc:`/developers/aux-file-support` + - :doc:`/developers/big-data-support` + - :doc:`/developers/dataset-migration-api` + - :doc:`/developers/dataset-semantic-metadata-api` + - :doc:`/developers/s3-direct-upload-api` + - :doc:`/developers/workflows` + Client Libraries ~~~~~~~~~~~~~~~~ From f038ae3b1003767ec20a1159a43b1db5cb9bdc68 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 21 Jul 2021 12:24:37 -0400 Subject: [PATCH 321/354] updating docs for zipdownload limit to reflect new behavior in 5.5 --- doc/sphinx-guides/source/installation/config.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 74119919c96..25c2b1a04b5 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1689,10 +1689,12 @@ Notes: :ZipDownloadLimit +++++++++++++++++ -For performance reasons, your Dataverse installation will only create zip files on the fly up to 100 MB but the limit can be increased. Here's an example of raising the limit to 1 GB: +For performance reasons, your Dataverse installation will only allow creation of zip files up to 100 MB, but the limit can be increased. Here's an example of raising the limit to 1 GB: ``curl -X PUT -d 1000000000 http://localhost:8080/api/admin/settings/:ZipDownloadLimit`` +In the UI, users trying to download a zip file larger than the Dataverse installation's :ZipDownloadLimit will receive messaging that the zip file is too large, and the user will be presented with alternate access options. + :TabularIngestSizeLimit +++++++++++++++++++++++ From 81fa59324873a3e756ec1d7aca4bb703f5b3d1ad Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 21 Jul 2021 17:58:53 -0400 Subject: [PATCH 322/354] sanitize schema.org md --- .../java/edu/harvard/iq/dataverse/DatasetVersion.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 864dca45664..78c1687a7b7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1707,11 +1707,11 @@ public String getJsonLd() { JsonArrayBuilder authors = Json.createArrayBuilder(); for (DatasetAuthor datasetAuthor : this.getDatasetAuthors()) { JsonObjectBuilder author = Json.createObjectBuilder(); - String name = datasetAuthor.getName().getValue(); + String name = datasetAuthor.getName().getDisplayValue(); DatasetField authorAffiliation = datasetAuthor.getAffiliation(); String affiliation = null; if (authorAffiliation != null) { - affiliation = datasetAuthor.getAffiliation().getValue(); + affiliation = datasetAuthor.getAffiliation().getDisplayValue(); } // We are aware of "givenName" and "familyName" but instead of a person it might be an organization such as "Gallup Organization". //author.add("@type", "Person"); @@ -1949,6 +1949,10 @@ public String getJsonLd() { job.add("distribution", fileArray); } jsonLd = job.build().toString(); + + //Most fields above should be stripped/sanitized but, since this is output in the dataset page as header metadata, do a final sanitize step to make sure + jsonLd = MarkupChecker.stripAllTags(jsonLd); + return jsonLd; } From e84671129e2191c57e1ae9274388cfe8ca8551f2 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 22 Jul 2021 13:39:05 -0400 Subject: [PATCH 323/354] out with the old... --- doc/release-notes/5.6-release-notes.md | 71 ++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 doc/release-notes/5.6-release-notes.md diff --git a/doc/release-notes/5.6-release-notes.md b/doc/release-notes/5.6-release-notes.md new file mode 100644 index 00000000000..663bcc29f2c --- /dev/null +++ b/doc/release-notes/5.6-release-notes.md @@ -0,0 +1,71 @@ +# Dataverse Software 5.5 + +This release brings new features, enhancements, and bug fixes to the Dataverse Software. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project. + +## Release Highlights + +### XXX + +XXX + +## Major Use Cases + +Newly-supported major use cases in this release include: + +- XX (Issue #XXX, PR #XXX) + +## Notes for Dataverse Installation Administrators + +### XXX + +XXX + +## Notes for Tool Developers and Integrators + +### XXX + +## Complete List of Changes + +For the complete list of code changes in this release, see the [5.6 Milestone](https://github.com/IQSS/dataverse/milestone/97?closed=1) in Github. + +For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org. + +## Installation + +If this is a new installation, please see our [Installation Guide](https://guides.dataverse.org/en/5.6/installation/). + +## Upgrade Instructions + +0\. These instructions assume that you've already successfully upgraded from Dataverse Software 4.x to Dataverse Software 5 following the instructions in the [Dataverse Software 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). After upgrading from the 4.x series to 5.0, you should progress through the other 5.x releases before attempting the upgrade to 5.6. + +1\. Undeploy the previous version. + +- `$PAYARA/bin/asadmin list-applications` +- `$PAYARA/bin/asadmin undeploy dataverse<-version>` + +2\. Stop Payara and remove the generated directory + +- `service payara stop` +- `rm -rf $PAYARA/glassfish/domains/domain1/generated` + +3\. Start Payara + +- `service payara start` + +4\. Deploy this version. + +- `$PAYARA/bin/asadmin deploy dataverse-5.6.war` + +5\. Restart payara + +- `service payara stop` +- `service payara start` + +## Additional Release Steps + +1\. Follow the steps to update your Solr configuration, found in the "Notes for Dataverse Installation Administrators" section above. Note that there are different instructions for Dataverse installations running with custom metadata blocks and those without. + +2\. Update Geospatial Metadata Block (if used) + +- `wget https://github.com/IQSS/dataverse/releases/download/v5.5/geospatial.tsv` +- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @geospatial.tsv -H "Content-type: text/tab-separated-values"` From 26de08901cdab6cc56814cc7a27857ec579dabf0 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 22 Jul 2021 13:59:40 -0400 Subject: [PATCH 324/354] in with the new... --- doc/release-notes/5.6-release-notes.md | 52 ++++++++++++++++--- doc/release-notes/6497-semantic-api.md | 7 --- .../7548-stored-procedure-update.md | 15 ------ doc/release-notes/7700-upgrade-payara.md | 19 ------- doc/release-notes/7786-ddi_changes.md | 4 +- .../7900-add-multipleFilesMetadata-dataset.md | 4 -- 6 files changed, 46 insertions(+), 55 deletions(-) delete mode 100644 doc/release-notes/6497-semantic-api.md delete mode 100644 doc/release-notes/7548-stored-procedure-update.md delete mode 100644 doc/release-notes/7700-upgrade-payara.md delete mode 100644 doc/release-notes/7900-add-multipleFilesMetadata-dataset.md diff --git a/doc/release-notes/5.6-release-notes.md b/doc/release-notes/5.6-release-notes.md index 663bcc29f2c..afb8768932b 100644 --- a/doc/release-notes/5.6-release-notes.md +++ b/doc/release-notes/5.6-release-notes.md @@ -4,9 +4,16 @@ This release brings new features, enhancements, and bug fixes to the Dataverse S ## Release Highlights -### XXX +### Dataset Semantic API (Experimental) + +Dataset metadata can be retrieved/set/updated using a new, flatter JSON-LD format - following the format of an OAI-ORE export (RDA-conformant Bags), allowing for easier transfer of metadata to/from other systems (i.e. without needing to know Dataverse's metadata block and field storage architecture). This new API also allows for the update of terms metadata (#5899). + +This development was supported by the [Research Data Alliance](https://rd-alliance.org), DANS, and Sciences PO and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). -XXX +### Direct Upload API Now Available for Adding Multiple Files Metadata to the Dataset + +Users can now add metadata of multiple files to the dataset once the files exists in the s3 bucket using the direct upload API. +For more information, see the [Direct DataFile Upload/Replace API section](https://guides.dataverse.org/en/5.6/developers/s3-direct-upload-api.html) of the Dataverse Software Guides. ## Major Use Cases @@ -16,9 +23,31 @@ Newly-supported major use cases in this release include: ## Notes for Dataverse Installation Administrators -### XXX +### Payara 5.2021.4 (or Higher) Required + +Some changes in this release require an upgrade to Payara 5.2021.4 or higher. + +Instructions on how to update can be found in the +[Payara documentation](https://docs.payara.fish/community/docs/5.2021.4/documentation/user-guides/upgrade-payara.html) + +It would likely be safer to upgrade Payara first, while still running Dataverse 5.6, and then proceed with the steps +below. Upgrading from an earlier version of Payara should be a straightforward process: + +1. Undeploy Dataverse +2. Stop Payara +3. Move the current Payara directory out of the way +4. Unzip the new Payara version in its place +5. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved `domain1` +6. Start Payara, deploy Dataverse 5.6. + +We still recommend that you read the detailed upgrade instructions above and if you run into any issues with this +upgrade, it will help to be able to separate them from any problems with the upgrade of Dataverse proper. + +### Enhancement to DDI Metadata Exports + +Several changes have been made to the DDI exports to improve support for internationalization and to improve compliance with CESSDA requirements. These changes include: -XXX +* Addition of a holdings element with a URI attribute whose value is the URL form of the dataset PID ## Notes for Tool Developers and Integrators @@ -63,9 +92,16 @@ If this is a new installation, please see our [Installation Guide](https://guide ## Additional Release Steps -1\. Follow the steps to update your Solr configuration, found in the "Notes for Dataverse Installation Administrators" section above. Note that there are different instructions for Dataverse installations running with custom metadata blocks and those without. +**If your installation relies on the database-side stored procedure for generating sequential numeric identifiers:** -2\. Update Geospatial Metadata Block (if used) +*(Note: You can skip the following paragraph if your installation uses the default-style, randomly-generated six alphanumeric +character-long identifiers for your datasets!)* -- `wget https://github.com/IQSS/dataverse/releases/download/v5.5/geospatial.tsv` -- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @geospatial.tsv -H "Content-type: text/tab-separated-values"` +The underlying database framework has been modified in this release, to make it easier for installations +to create custom procedures for generating identifier strings that suit their needs. Your current configuration will +be automatically updated by the database upgrade (Flyway) script incorporated in the release. No manual configuration +changes should be necessary. However, after the upgrade, we recommend that you confirm that your installation can still +create new datasets, and that they are still assigned sequential numeric identifiers. In the unlikely chance that this +is no longer working, please re-create the stored procedure following the steps described in the documentation for the +`:IdentifierGenerationStyle` setting in the *Configuration* section of the Installation Guide for this release (v5.6). +(Running the script supplied there will NOT overwrite the position on the sequence you are currently using!) diff --git a/doc/release-notes/6497-semantic-api.md b/doc/release-notes/6497-semantic-api.md deleted file mode 100644 index 8222892d18e..00000000000 --- a/doc/release-notes/6497-semantic-api.md +++ /dev/null @@ -1,7 +0,0 @@ -# Release Highlights - -### Dataset Semantic API (Experimental) - -Dataset metadata can be retrieved/set/updated using a new, flatter JSON-LD format - following the format of an OAI-ORE export (RDA-conformant Bags), allowing for easier transfer of metadata to/from other systems (i.e. without needing to know Dataverse's metadata block and field storage architecture). This new API also allows for the update of terms metadata (#5899). - -This development was supported by the [Research Data Alliance](https://rd-alliance.org), DANS, and Sciences PO and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). diff --git a/doc/release-notes/7548-stored-procedure-update.md b/doc/release-notes/7548-stored-procedure-update.md deleted file mode 100644 index 26d2a8ab784..00000000000 --- a/doc/release-notes/7548-stored-procedure-update.md +++ /dev/null @@ -1,15 +0,0 @@ -### Upgrade Notes - -**If your installation relies on the database-side stored procedure for generating sequential numeric identifiers:** - -*(Note: You can skip the following paragraph if your installation uses the default-style, randomly-generated six alphanumeric -character-long identifiers for your datasets!)* - -The underlying database framework has been modified in this release, to make it easier for installations -to create custom procedures for generating identifier strings that suit their needs. Your current configuration will -be automatically updated by the database upgrade (Flyway) script incorporated in the release. No manual configuration -changes should be necessary. However, after the upgrade, we recommend that you confirm that your installation can still -create new datasets, and that they are still assigned sequential numeric identifiers. In the unlikely chance that this -is no longer working, please re-create the stored procedure following the steps described in the documentation for the -`:IdentifierGenerationStyle` setting in the *Configuration* section of the Installation Guide for this release (v5.6). -(Running the script supplied there will NOT overwrite the position on the sequence you are currently using!) diff --git a/doc/release-notes/7700-upgrade-payara.md b/doc/release-notes/7700-upgrade-payara.md deleted file mode 100644 index de7c58bf963..00000000000 --- a/doc/release-notes/7700-upgrade-payara.md +++ /dev/null @@ -1,19 +0,0 @@ -### Payara 5.2021.4 (or Higher) Required - -Some changes in this release require an upgrade to Payara 5.2021.4 or higher. - -Instructions on how to update can be found in the -[Payara documentation](https://docs.payara.fish/community/docs/5.2021.4/documentation/user-guides/upgrade-payara.html) - -It would likely be safer to upgrade Payara first, while still running Dataverse 5.6, and then proceed with the steps -below. Upgrading from an earlier version of Payara should be a straightforward process: - -1. Undeploy Dataverse -2. Stop Payara -3. Move the current Payara directory out of the way -4. Unzip the new Payara version in its place -5. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved `domain1` -6. Start Payara, deploy Dataverse 5.6. - -We still recommend that you read the detailed upgrade instructions above and if you run into any issues with this -upgrade, it will help to be able to separate them from any problems with the upgrade of Dataverse proper. diff --git a/doc/release-notes/7786-ddi_changes.md b/doc/release-notes/7786-ddi_changes.md index c6b0d455ddf..6cbc1cfb17c 100644 --- a/doc/release-notes/7786-ddi_changes.md +++ b/doc/release-notes/7786-ddi_changes.md @@ -1,9 +1,9 @@ -Note: These notes cover several related PRs (#7984, #7958, #7959 respectively for the three bullets below.) If some are not merged before the next release, these notes should be adjusted. +Note: These notes cover several related PRs (#7958, #7959 respectively for the three bullets below.) If some are not merged before the next release, these notes should be adjusted. ### Enhancements to DDI Metadata Exports Several changes have been made to the DDI exports to improve support for internationalization and to improve compliance with CESSDA requirements. These changes include: -* Addition of a holdings element with a URI attribute whose value is the URL form of the dataset PID + * Addition of xml:lang attributes specifying the dataset metadata language at the document level and for individual elements such as title and description * Specification of controlled vocabulary terms in duplicate elements in multiple languages (in the installation default langauge and, if different, the dataset metadata language) diff --git a/doc/release-notes/7900-add-multipleFilesMetadata-dataset.md b/doc/release-notes/7900-add-multipleFilesMetadata-dataset.md deleted file mode 100644 index 13b77259c69..00000000000 --- a/doc/release-notes/7900-add-multipleFilesMetadata-dataset.md +++ /dev/null @@ -1,4 +0,0 @@ -### Direct Upload API Now Available for Adding Multiple Files Metadata to the Dataset - -Users can now add metadata of multiple files to the dataset once the files exists in the s3 bucket using the direct upload API. -For more information, see the [Direct DataFile Upload/Replace API section](https://guides.dataverse.org/en/5.6/developers/s3-direct-upload-api.html) of the Dataverse Software Guides. From 80a92c5bacef1a0d84be2b429f105fa6780699c9 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 22 Jul 2021 15:24:15 -0400 Subject: [PATCH 325/354] some stubs for stuff that I feel raises to level of release highlights --- doc/release-notes/5.6-release-notes.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/5.6-release-notes.md b/doc/release-notes/5.6-release-notes.md index afb8768932b..9b94f724134 100644 --- a/doc/release-notes/5.6-release-notes.md +++ b/doc/release-notes/5.6-release-notes.md @@ -1,9 +1,15 @@ -# Dataverse Software 5.5 +# Dataverse Software 5.6 This release brings new features, enhancements, and bug fixes to the Dataverse Software. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project. ## Release Highlights +### Anonymized Accesses in Support of Double Blind Review + +### Guestbooks API + + + ### Dataset Semantic API (Experimental) Dataset metadata can be retrieved/set/updated using a new, flatter JSON-LD format - following the format of an OAI-ORE export (RDA-conformant Bags), allowing for easier transfer of metadata to/from other systems (i.e. without needing to know Dataverse's metadata block and field storage architecture). This new API also allows for the update of terms metadata (#5899). @@ -15,6 +21,7 @@ This development was supported by the [Research Data Alliance](https://rd-allian Users can now add metadata of multiple files to the dataset once the files exists in the s3 bucket using the direct upload API. For more information, see the [Direct DataFile Upload/Replace API section](https://guides.dataverse.org/en/5.6/developers/s3-direct-upload-api.html) of the Dataverse Software Guides. + ## Major Use Cases Newly-supported major use cases in this release include: From 9ef174db163bc7659699f5901d15da82a53607cd Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 22 Jul 2021 17:49:43 -0400 Subject: [PATCH 326/354] additional updates, will get back to it tomorrow --- doc/release-notes/5.6-release-notes.md | 29 +++++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/doc/release-notes/5.6-release-notes.md b/doc/release-notes/5.6-release-notes.md index 9b94f724134..36921c620d3 100644 --- a/doc/release-notes/5.6-release-notes.md +++ b/doc/release-notes/5.6-release-notes.md @@ -6,30 +6,39 @@ This release brings new features, enhancements, and bug fixes to the Dataverse S ### Anonymized Accesses in Support of Double Blind Review -### Guestbooks API +Dataverse installations can select whether or not to allow users to create anonymized private URLs and can control which specific fields are anonymized. +If this is enabled at the installation level, users can create private URLs that do not expose identifying information about dataset depositors, allowing for double blind reviews of datasets. +### Guestbook Responses API + +A new API to retrieve Guestbook responses has been added. This makes it easier to retrieve the records for large guestbooks and also makes it easier to integrate with external systems. ### Dataset Semantic API (Experimental) -Dataset metadata can be retrieved/set/updated using a new, flatter JSON-LD format - following the format of an OAI-ORE export (RDA-conformant Bags), allowing for easier transfer of metadata to/from other systems (i.e. without needing to know Dataverse's metadata block and field storage architecture). This new API also allows for the update of terms metadata (#5899). +Dataset metadata can be retrieved, set, and updated using a new, flatter JSON-LD format - following the format of an OAI-ORE export (RDA-conformant Bags), allowing for easier transfer of metadata to/from other systems (i.e. without needing to know Dataverse's metadata block and field storage architecture). This new API also allows for the update of terms metadata (#5899). This development was supported by the [Research Data Alliance](https://rd-alliance.org), DANS, and Sciences PO and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). ### Direct Upload API Now Available for Adding Multiple Files Metadata to the Dataset -Users can now add metadata of multiple files to the dataset once the files exists in the s3 bucket using the direct upload API. -For more information, see the [Direct DataFile Upload/Replace API section](https://guides.dataverse.org/en/5.6/developers/s3-direct-upload-api.html) of the Dataverse Software Guides. - +Users can now add metadata of multiple files to the dataset once the files exists in the s3 bucket using the direct upload API. For more information, see the [Direct DataFile Upload/Replace API section](https://guides.dataverse.org/en/5.6/developers/s3-direct-upload-api.html) of the Dataverse Software Guides. ## Major Use Cases Newly-supported major use cases in this release include: -- XX (Issue #XXX, PR #XXX) +- Double blind (Issue #XXX, PR #XXX) +- Guestbook API (Issue #XXX, PR #XXX) +- Semantic API (Issue #XXX, PR #XXX) +- Extended Traces API (Issue #XXX, PR #XXX) +- Terms API updates (Issue #XXX, PR #XXX) ## Notes for Dataverse Installation Administrators +### DB Constraint + + ### Payara 5.2021.4 (or Higher) Required Some changes in this release require an upgrade to Payara 5.2021.4 or higher. @@ -38,7 +47,7 @@ Instructions on how to update can be found in the [Payara documentation](https://docs.payara.fish/community/docs/5.2021.4/documentation/user-guides/upgrade-payara.html) It would likely be safer to upgrade Payara first, while still running Dataverse 5.6, and then proceed with the steps -below. Upgrading from an earlier version of Payara should be a straightforward process: +below. Upgrading from an earlier version of Payara should be a straightforward process: 1. Undeploy Dataverse 2. Stop Payara @@ -46,7 +55,7 @@ below. Upgrading from an earlier version of Payara should be a straightforward p 4. Unzip the new Payara version in its place 5. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved `domain1` 6. Start Payara, deploy Dataverse 5.6. - + We still recommend that you read the detailed upgrade instructions above and if you run into any issues with this upgrade, it will help to be able to separate them from any problems with the upgrade of Dataverse proper. @@ -56,6 +65,10 @@ Several changes have been made to the DDI exports to improve support for interna * Addition of a holdings element with a URI attribute whose value is the URL form of the dataset PID +## New JVM Options and DB Settings + +:AnonymizedFieldTypeNames + ## Notes for Tool Developers and Integrators ### XXX From 52fbc935b2e0f6f0ab0c3e0f961b3735cf169cb5 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 23 Jul 2021 12:56:31 -0400 Subject: [PATCH 327/354] lost test update --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 246eb0b2503..1bd92707231 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -2292,8 +2292,8 @@ public void testReCreateDataset() { Response response = UtilIT.getDatasetJsonLDMetadata(datasetId, apiToken); response.then().assertThat().statusCode(OK.getStatusCode()); - String expectedString = JsonPath.from(response.getBody().asString()).getString("data"); - + String expectedString = getData(response.getBody().asString()); + // Delete the dataset via native API Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); deleteDatasetResponse.prettyPrint(); From 55463f9d8f23c6344aae3d07d1672a944f9477c3 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 23 Jul 2021 19:11:36 -0400 Subject: [PATCH 328/354] updates before PR --- doc/release-notes/5.6-release-notes.md | 69 ++++++++++---------------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/doc/release-notes/5.6-release-notes.md b/doc/release-notes/5.6-release-notes.md index 36921c620d3..920c37c26d3 100644 --- a/doc/release-notes/5.6-release-notes.md +++ b/doc/release-notes/5.6-release-notes.md @@ -4,11 +4,9 @@ This release brings new features, enhancements, and bug fixes to the Dataverse S ## Release Highlights -### Anonymized Accesses in Support of Double Blind Review +### Anonymized Access in Support of Double Blind Review -Dataverse installations can select whether or not to allow users to create anonymized private URLs and can control which specific fields are anonymized. - -If this is enabled at the installation level, users can create private URLs that do not expose identifying information about dataset depositors, allowing for double blind reviews of datasets. +Dataverse installations can select whether or not to allow users to create anonymized private URLs and can control which specific identifying fields are anonymized. If this is enabled, users can create private URLs that do not expose identifying information about dataset depositors, allowing for double blind reviews of datasets in the Dataverse installation. ### Guestbook Responses API @@ -28,50 +26,37 @@ Users can now add metadata of multiple files to the dataset once the files exist Newly-supported major use cases in this release include: -- Double blind (Issue #XXX, PR #XXX) -- Guestbook API (Issue #XXX, PR #XXX) -- Semantic API (Issue #XXX, PR #XXX) -- Extended Traces API (Issue #XXX, PR #XXX) -- Terms API updates (Issue #XXX, PR #XXX) +- Users can create Private URLs that anonymize dataset metadata, allowing for double blind peer review. (Issue #1724, PR #7908) +- Users can download Guestbook records using a new API. (Issue #7767, PR #7931) +- Users can update terms metadata using the new semantic API. (Issue #5899, PR #7414) +- Users can retrieve, set, and update metadata using a new, flatter JSON-LD format. (Issue #6497, PR #7414) +- Administrators can use the Traces API to retreive information about specific types of user activity (Issue #7952, PR #7953) ## Notes for Dataverse Installation Administrators -### DB Constraint +### New Database Constraint +A new DB Constraint has been added in this release. Full instructions on how to identify whether or not your database needs any cleanup before the upgrade can be found in the [Dataverse software GitHub repository](https://github.com/IQSS/dataverse/blob/develop/scripts/issues/7451/PRE-RELEASE-INFO.txt). This information was also emailed out to Dataverse installation contacts. ### Payara 5.2021.4 (or Higher) Required Some changes in this release require an upgrade to Payara 5.2021.4 or higher. -Instructions on how to update can be found in the -[Payara documentation](https://docs.payara.fish/community/docs/5.2021.4/documentation/user-guides/upgrade-payara.html) - -It would likely be safer to upgrade Payara first, while still running Dataverse 5.6, and then proceed with the steps -below. Upgrading from an earlier version of Payara should be a straightforward process: - -1. Undeploy Dataverse -2. Stop Payara -3. Move the current Payara directory out of the way -4. Unzip the new Payara version in its place -5. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved `domain1` -6. Start Payara, deploy Dataverse 5.6. - -We still recommend that you read the detailed upgrade instructions above and if you run into any issues with this -upgrade, it will help to be able to separate them from any problems with the upgrade of Dataverse proper. +Instructions on how to update can be found in the [Payara documentation](https://docs.payara.fish/community/docs/5.2021.4/documentation/user-guides/upgrade-payara.html) We've included the necessary steps below, but we recommend that you review the Payara upgrade instructions as it could be helpful during any troubleshooting. ### Enhancement to DDI Metadata Exports -Several changes have been made to the DDI exports to improve support for internationalization and to improve compliance with CESSDA requirements. These changes include: - -* Addition of a holdings element with a URI attribute whose value is the URL form of the dataset PID +To increase support for internationalization and to improve compliance with CESSDA requirements, DDI exports now have a holdings element with a URI attribute whose value is the URL form of the dataset PID. ## New JVM Options and DB Settings -:AnonymizedFieldTypeNames +:AnonymizedFieldTypeNames can be used to identify specific fields that will be anonymized when a user creates an anonymized Private URL. ## Notes for Tool Developers and Integrators -### XXX +### Semantic API + +The new Semantic API is especially helpful in data migrations and getting metadata into a Dataverse installation. Learn more in the [Developers Guide](https://guides.dataverse.org/en/5.6/developers/). ## Complete List of Changes @@ -97,15 +82,21 @@ If this is a new installation, please see our [Installation Guide](https://guide - `service payara stop` - `rm -rf $PAYARA/glassfish/domains/domain1/generated` -3\. Start Payara +3\. Move the current Payara directory out of the way + +4\. Unzip the new Payara version in its place + +5\. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved `domain1` + +6\. Start Payara - `service payara start` -4\. Deploy this version. +7\. Deploy this version. - `$PAYARA/bin/asadmin deploy dataverse-5.6.war` -5\. Restart payara +8\. Restart payara - `service payara stop` - `service payara start` @@ -114,14 +105,8 @@ If this is a new installation, please see our [Installation Guide](https://guide **If your installation relies on the database-side stored procedure for generating sequential numeric identifiers:** -*(Note: You can skip the following paragraph if your installation uses the default-style, randomly-generated six alphanumeric -character-long identifiers for your datasets!)* +Note that you can skip this step if your installation uses the default-style, randomly-generated six alphanumeric character-long identifiers for your datasets! This is the case with most Dataverse installations. + +The underlying database framework has been modified in this release, to make it easier for installations to create custom procedures for generating identifier strings that suit their needs. Your current configuration will be automatically updated by the database upgrade (Flyway) script incorporated in the release. No manual configuration changes should be necessary. However, after the upgrade, we recommend that you confirm that your installation can still create new datasets, and that they are still assigned sequential numeric identifiers. In the unlikely chance that this is no longer working, please re-create the stored procedure following the steps described in the documentation for the `:IdentifierGenerationStyle` setting in the *Configuration* section of the Installation Guide for this release (v5.6). -The underlying database framework has been modified in this release, to make it easier for installations -to create custom procedures for generating identifier strings that suit their needs. Your current configuration will -be automatically updated by the database upgrade (Flyway) script incorporated in the release. No manual configuration -changes should be necessary. However, after the upgrade, we recommend that you confirm that your installation can still -create new datasets, and that they are still assigned sequential numeric identifiers. In the unlikely chance that this -is no longer working, please re-create the stored procedure following the steps described in the documentation for the -`:IdentifierGenerationStyle` setting in the *Configuration* section of the Installation Guide for this release (v5.6). (Running the script supplied there will NOT overwrite the position on the sequence you are currently using!) From d8fd8fe4ed66522aa78e3c7f4705e326a4cb4278 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 26 Jul 2021 09:10:57 -0400 Subject: [PATCH 329/354] #118 fix show deleted --- .../dataverse/ManageFilePermissionsPage.java | 26 +++++++++++++++---- .../webapp/permissions-manage-files.xhtml | 4 +-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index 6d0e2f77c50..ef64b114034 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -115,6 +115,17 @@ public TreeMap> getFileAccessRequestMap() { } + private boolean backingShowDeleted = true; + + public String showDeletedCheckboxChange() { + + if (backingShowDeleted != showDeleted) { + initMaps(); + backingShowDeleted = showDeleted; + } + return ""; + } + public String init() { if (dataset.getId() != null) { dataset = datasetService.find(dataset.getId()); @@ -136,17 +147,22 @@ private void initMaps() { // initialize files and usergroup list roleAssigneeMap.clear(); fileMap.clear(); - fileAccessRequestMap.clear(); + fileAccessRequestMap.clear(); for (DataFile file : dataset.getFiles()) { - boolean fileIsDeleted = !((dataset.getLatestVersion().isDraft() && file.getFileMetadata().getDatasetVersion().isDraft()) - || (dataset.getLatestVersion().isReleased() && file.getFileMetadata().getDatasetVersion().equals(dataset.getLatestVersion()))); // only include if the file is restricted (or its draft version is restricted) //Added a null check in case there are files that have no metadata records SEK //for 6587 make sure that a file is in the current version befor adding to the fileMap SEK 2/11/2020 - if (file.getFileMetadata() != null && (file.isRestricted() || file.getFileMetadata().isRestricted()) - && (!fileIsDeleted || isShowDeleted())) { + if (file.getFileMetadata() != null && (file.isRestricted() || file.getFileMetadata().isRestricted())) { + //only test if file is deleted if it's restricted + boolean fileIsDeleted = !((dataset.getLatestVersion().isDraft() && file.getFileMetadata().getDatasetVersion().isDraft()) + || (dataset.getLatestVersion().isReleased() && file.getFileMetadata().getDatasetVersion().equals(dataset.getLatestVersion()))); + + if (!isShowDeleted() && fileIsDeleted) { + //if don't show deleted and is deleted get out. + break; + } // we get the direct role assignments assigned to the file List ras = roleService.directRoleAssignments(file); List raList = new ArrayList<>(ras.size()); diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 054f97aec80..49e2ff86768 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -50,7 +50,7 @@

    - +
    @@ -156,7 +156,7 @@
    - +
    From b6a056697026b4e007f7628d59f74a85c714cd8c Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 26 Jul 2021 10:19:16 -0400 Subject: [PATCH 330/354] #118 fix file mapping wrt deleted files --- .../edu/harvard/iq/dataverse/ManageFilePermissionsPage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index ef64b114034..36a0b7cc692 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -160,8 +160,8 @@ private void initMaps() { || (dataset.getLatestVersion().isReleased() && file.getFileMetadata().getDatasetVersion().equals(dataset.getLatestVersion()))); if (!isShowDeleted() && fileIsDeleted) { - //if don't show deleted and is deleted get out. - break; + //if don't show deleted and is deleted go to next file... + continue; } // we get the direct role assignments assigned to the file List ras = roleService.directRoleAssignments(file); From 1bf69dd2b3f7fab45d38a407bf74094d77d9464c Mon Sep 17 00:00:00 2001 From: Shiro Kuriwaki Date: Mon, 26 Jul 2021 11:16:32 -0400 Subject: [PATCH 331/354] Update maintainer for R client Also added a note on main strength and where pyDataverse is probably better. --- doc/sphinx-guides/source/api/client-libraries.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst index af8b2f19015..a47eb2d58aa 100755 --- a/doc/sphinx-guides/source/api/client-libraries.rst +++ b/doc/sphinx-guides/source/api/client-libraries.rst @@ -27,9 +27,10 @@ It was created and is maintained by `The Agile Monkeys `_. +https://github.com/IQSS/dataverse-client-r is the official R package for Dataverse Software APIs. The latest release can be installed from `CRAN `_. +The R client can search and download datasets. It is useful when automatically (instead of manually) downloading data files as part of a script. For bulk edit and upload operations, we currently recommend pyDataverse. -The package is currently maintained by `Will Beasley `_. It was created by `Thomas Leeper `_ whose Dataverse collection can be found at https://dataverse.harvard.edu/dataverse/leeper +The package is currently maintained by `Shiro Kuriwaki `_. It was originally created by `Thomas Leeper `_ and then formerly maintained by `Will Beasley `_. Java ---- From 15865803309e5c0c5f4de3be61a43e5a5661e5e5 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 26 Jul 2021 14:04:05 -0400 Subject: [PATCH 332/354] #118 simplify checkbox change method --- .../edu/harvard/iq/dataverse/ManageFilePermissionsPage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index 36a0b7cc692..c728062a5a8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -117,13 +117,13 @@ public TreeMap> getFileAccessRequestMap() { private boolean backingShowDeleted = true; - public String showDeletedCheckboxChange() { + public void showDeletedCheckboxChange() { if (backingShowDeleted != showDeleted) { initMaps(); backingShowDeleted = showDeleted; } - return ""; + } public String init() { From 74ba6b69e59c9e66678fb3ba9f70ee309eeaaaab Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 27 Jul 2021 10:18:46 -0400 Subject: [PATCH 333/354] updates from CR --- doc/release-notes/5.6-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/5.6-release-notes.md b/doc/release-notes/5.6-release-notes.md index 920c37c26d3..26fcb373689 100644 --- a/doc/release-notes/5.6-release-notes.md +++ b/doc/release-notes/5.6-release-notes.md @@ -20,7 +20,7 @@ This development was supported by the [Research Data Alliance](https://rd-allian ### Direct Upload API Now Available for Adding Multiple Files Metadata to the Dataset -Users can now add metadata of multiple files to the dataset once the files exists in the s3 bucket using the direct upload API. For more information, see the [Direct DataFile Upload/Replace API section](https://guides.dataverse.org/en/5.6/developers/s3-direct-upload-api.html) of the Dataverse Software Guides. +Users can now add metadata of multiple files to the dataset once the files exists in the s3 bucket using the direct upload API. This makes direct uploads more efficient and reduces server load by only updating the dataset once instead of once per file. For more information, see the [Direct DataFile Upload/Replace API section](https://guides.dataverse.org/en/5.6/developers/s3-direct-upload-api.html) of the Dataverse Software Guides. ## Major Use Cases From 669adb3022c091b69811e8f903edb095cc1342c6 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 27 Jul 2021 10:19:46 -0400 Subject: [PATCH 334/354] updates from CR --- doc/release-notes/5.6-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/5.6-release-notes.md b/doc/release-notes/5.6-release-notes.md index 26fcb373689..768dae42bda 100644 --- a/doc/release-notes/5.6-release-notes.md +++ b/doc/release-notes/5.6-release-notes.md @@ -50,7 +50,7 @@ To increase support for internationalization and to improve compliance with CESS ## New JVM Options and DB Settings -:AnonymizedFieldTypeNames can be used to identify specific fields that will be anonymized when a user creates an anonymized Private URL. +:AnonymizedFieldTypeNames can be used to enable creation of anonymized Private URLs and to specify which fields will be anonymized. ## Notes for Tool Developers and Integrators From 8496b7c3db1270234ce3a5936e40af27aa0fa302 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 27 Jul 2021 12:32:06 -0400 Subject: [PATCH 335/354] put back the few extra lines to the upgrade instructions (about not running the commands as root, etc.; these were originally added in 5.4). #8012. --- doc/release-notes/5.6-release-notes.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/5.6-release-notes.md b/doc/release-notes/5.6-release-notes.md index 768dae42bda..c05e13e38f1 100644 --- a/doc/release-notes/5.6-release-notes.md +++ b/doc/release-notes/5.6-release-notes.md @@ -72,6 +72,14 @@ If this is a new installation, please see our [Installation Guide](https://guide 0\. These instructions assume that you've already successfully upgraded from Dataverse Software 4.x to Dataverse Software 5 following the instructions in the [Dataverse Software 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). After upgrading from the 4.x series to 5.0, you should progress through the other 5.x releases before attempting the upgrade to 5.6. +If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user. + +In the following commands we assume that Payara 5 is installed in `/usr/local/payara5`. If not, adjust as needed. + +`export PAYARA=/usr/local/payara5` + +(or `setenv PAYARA /usr/local/payara5` if you are using a `csh`-like shell) + 1\. Undeploy the previous version. - `$PAYARA/bin/asadmin list-applications` @@ -94,7 +102,7 @@ If this is a new installation, please see our [Installation Guide](https://guide 7\. Deploy this version. -- `$PAYARA/bin/asadmin deploy dataverse-5.6.war` +- `$PAYARA/bin/asadmin deploy dataverse-5.6.war`5.6-release-notes.md 8\. Restart payara From 8f749e7e0fadfcc35590934fbd49bc95e6c50b74 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 23 Jul 2021 15:52:40 -0400 Subject: [PATCH 336/354] respond to QA comments --- doc/release-notes/6497-migrate-api.md | 14 ++------------ .../source/_static/api/dataset-migrate.jsonld | 1 + .../source/developers/dataset-migration-api.rst | 16 +++++++--------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/doc/release-notes/6497-migrate-api.md b/doc/release-notes/6497-migrate-api.md index 72573cfe905..7b351de0a38 100644 --- a/doc/release-notes/6497-migrate-api.md +++ b/doc/release-notes/6497-migrate-api.md @@ -2,16 +2,6 @@ ### Dataset Migration API (Experimental) -Datasets can now imported/updated following the format of an OAI-ORE export (RDA-conformant Bags), allowing for not only easier migration from one Dataverse installation to another, but also for better support of import from other systems. This experimental endpoint also allows keeping the existing persistent identifier (where the authority and shoulder match those for which the software is configured) and publication dates. This endpoint also allows for the update of terms metadata (#5899). +Datasets can now imported following the format of an OAI-ORE export (RDA-conformant Bags), allowing for easier migration from one Dataverse installation to another, and migration from other systems. This experimental, super-user only, endpoint also allows keeping the existing persistent identifier (where the authority and shoulder match those for which the software is configured) and publication dates. -This development was supported by the [Research Data Alliance](https://rd-alliance.org) and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). - -### Additional Upgrade Steps - -Update Solr Schema - -- copy schema_dv_mdb_fields.xml and schema_dv_mdb_copies.xml to solr server, for example into /usr/local/solr/solr-8.8.1/server/solr/collection1/conf/ directory - -- Restart Solr, or tell Solr to reload its configuration: - - `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"` +This development was supported by DANS and the [Research Data Alliance](https://rd-alliance.org) and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). diff --git a/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld b/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld index 7ab4ab3690b..07cf66ace53 100644 --- a/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld +++ b/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld @@ -24,6 +24,7 @@ "@id": "doi:10.33564/FK27U7YBV", "schema:version": "1.0", "schema:license": "https://creativecommons.org/publicdomain/zero/1.0/", +"schema:datePublished": "2021-07-21", "dvcore:fileTermsOfAccess": { "dvcore:fileRequestAccess": false }, diff --git a/doc/sphinx-guides/source/developers/dataset-migration-api.rst b/doc/sphinx-guides/source/developers/dataset-migration-api.rst index 263ffc3aa99..d46db5e3c9e 100644 --- a/doc/sphinx-guides/source/developers/dataset-migration-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-migration-api.rst @@ -6,12 +6,13 @@ The Dataverse software includes several ways to add Datasets originally created This experimental migration API offers an additional option with some potential advantages: * metadata can be specified using the json-ld format used in the OAI-ORE metadata export -* existing PIDs can be maintained (currently limited to the case where the PID can be managed by the Dataverse software, e.g. where the authority and shoulder match those the software is configured for) +* existing publication dates and PIDs are maintained (currently limited to the case where the PID can be managed by the Dataverse software, e.g. where the authority and shoulder match those the software is configured for) * adding files can be done via the standard APIs, including using direct-upload to S3 -* the dataset can be published keeping the original publication date -This API consists of 2 calls: one to create an initial Dataset version, and one to publish the version with a specified publication date. -These calls can be used in concert with other API calls to add files, update metadata for additional versions, etc. +This API consists of 2 calls: one to create an initial Dataset version, and one to 'republish' the dataset through Dataverse with a specified publication date. +Both calls require super-admin privileges. + +These calls can be used in concert with other API calls to add files, update metadata, etc. before the 'republish' step is done. Start Migrating a Dataset into a Dataverse Collection @@ -26,12 +27,10 @@ To import a dataset with an existing persistent identifier (PID), the provided j export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export DATAVERSE_ID=root - export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV - + curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$DATAVERSE_ID/datasets/:startmigration --upload-file dataset-migrate.jsonld -An example jsonld file is available at :download:`dataset-migrate.jsonld <../_static/api/dataset-migrate.jsonld>` - +An example jsonld file is available at :download:`dataset-migrate.jsonld <../_static/api/dataset-migrate.jsonld>` . Note that you would need to replace the PID in the sample file with one supported in your Dataverse instance. (Also note that `Issue #8028 `_ currently breaks testing this API with DataCite test DOIs.) Publish a Migrated Dataset -------------------------- @@ -43,7 +42,6 @@ The call above creates a Dataset. Once it is created, other APIs can be used to .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV export SERVER_URL=https://demo.dataverse.org curl -H 'Content-Type: application/jsonld' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated" From a8263c36d0aee38ca0298bb91b2a059a61936e53 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 27 Jul 2021 14:13:13 -0400 Subject: [PATCH 337/354] rearranged the payara upgrade notes a bit (#8012) --- doc/release-notes/5.6-release-notes.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/doc/release-notes/5.6-release-notes.md b/doc/release-notes/5.6-release-notes.md index c05e13e38f1..3928ebfa39c 100644 --- a/doc/release-notes/5.6-release-notes.md +++ b/doc/release-notes/5.6-release-notes.md @@ -40,7 +40,7 @@ A new DB Constraint has been added in this release. Full instructions on how to ### Payara 5.2021.4 (or Higher) Required -Some changes in this release require an upgrade to Payara 5.2021.4 or higher. +Some changes in this release require an upgrade to Payara 5.2021.4 or higher. (See the upgrade section). Instructions on how to update can be found in the [Payara documentation](https://docs.payara.fish/community/docs/5.2021.4/documentation/user-guides/upgrade-payara.html) We've included the necessary steps below, but we recommend that you review the Payara upgrade instructions as it could be helpful during any troubleshooting. @@ -72,6 +72,9 @@ If this is a new installation, please see our [Installation Guide](https://guide 0\. These instructions assume that you've already successfully upgraded from Dataverse Software 4.x to Dataverse Software 5 following the instructions in the [Dataverse Software 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). After upgrading from the 4.x series to 5.0, you should progress through the other 5.x releases before attempting the upgrade to 5.6. +The steps below include a required upgrade to Payara 5.2021.4 or higher. (It is a simple matter of reusing your existing domain directory with the new distribution). But we also recommend that you review the Payara upgrade instructions as it could be helpful during any troubleshooting: [Payara documentation](https://docs.payara.fish/community/docs/5.2021.4/documentation/user-guides/upgrade-payara.html) + + If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user. In the following commands we assume that Payara 5 is installed in `/usr/local/payara5`. If not, adjust as needed. @@ -80,19 +83,22 @@ In the following commands we assume that Payara 5 is installed in `/usr/local/pa (or `setenv PAYARA /usr/local/payara5` if you are using a `csh`-like shell) -1\. Undeploy the previous version. +1\. Undeploy the previous version - `$PAYARA/bin/asadmin list-applications` - `$PAYARA/bin/asadmin undeploy dataverse<-version>` -2\. Stop Payara and remove the generated directory + +2\. Stop Payara - `service payara stop` - `rm -rf $PAYARA/glassfish/domains/domain1/generated` 3\. Move the current Payara directory out of the way -4\. Unzip the new Payara version in its place +- `mv $PAYARA $PAYARA.MOVED` + +4\. Download the new Payara version (5.2021.4+), and unzip it in its place 5\. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved `domain1` @@ -102,7 +108,7 @@ In the following commands we assume that Payara 5 is installed in `/usr/local/pa 7\. Deploy this version. -- `$PAYARA/bin/asadmin deploy dataverse-5.6.war`5.6-release-notes.md +- `$PAYARA/bin/asadmin deploy dataverse-5.6.war` 8\. Restart payara From c833eb98362792296239444bdb0510be10135a38 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Jul 2021 15:23:28 -0400 Subject: [PATCH 338/354] handle version 1.0 info already set --- .../edu/harvard/iq/dataverse/api/Datasets.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 6265f2637dc..348323b49d5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1233,23 +1233,23 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin // First Release ds.getLatestVersion().setVersionNumber(Long.valueOf(1)); ds.getLatestVersion().setMinorVersionNumber(Long.valueOf(0)); - //Also set publication date if this is the first - if(dateTime != null) { - ds.setPublicationDate(Timestamp.valueOf(dateTime)); - } - // Release User is only set in FinalizeDatasetPublicationCommand if the pub date - // is null, so set it here. - ds.setReleaseUser((AuthenticatedUser) user); - } else if (ds.getLatestVersion().isMinorUpdate()) { ds.getLatestVersion().setVersionNumber(Long.valueOf(ds.getVersionNumber())); ds.getLatestVersion().setMinorVersionNumber(Long.valueOf(ds.getMinorVersionNumber() + 1)); - } else { // major, non-first release ds.getLatestVersion().setVersionNumber(Long.valueOf(ds.getVersionNumber() + 1)); ds.getLatestVersion().setMinorVersionNumber(Long.valueOf(0)); } + if(ds.getLatestVersion().getVersionNumber()==1 && ds.getLatestVersion().getMinorVersionNumber()==0) { + //Also set publication date if this is the first + if(dateTime != null) { + ds.setPublicationDate(Timestamp.valueOf(dateTime)); + } + // Release User is only set in FinalizeDatasetPublicationCommand if the pub date + // is null, so set it here. + ds.setReleaseUser((AuthenticatedUser) user); + } } } catch (Exception e) { From d549f72af2ba398bbe28729b6762e5f6f6468e60 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Jul 2021 15:47:32 -0400 Subject: [PATCH 339/354] enable DOI updates --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 348323b49d5..3385be07795 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1271,11 +1271,11 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin if (prePubWf.isPresent()) { // Start the workflow, the workflow will call FinalizeDatasetPublication later wfService.start(prePubWf.get(), - new WorkflowContext(createDataverseRequest(user), ds, TriggerType.PrePublishDataset, true), + new WorkflowContext(createDataverseRequest(user), ds, TriggerType.PrePublishDataset, false), false); } else { FinalizeDatasetPublicationCommand cmd = new FinalizeDatasetPublicationCommand(ds, - createDataverseRequest(user), true); + createDataverseRequest(user), false); ds = commandEngine.submit(cmd); } } catch (CommandException ex) { From 93481bd2039e7bd551752dd975c768490f29be52 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Jul 2021 16:10:37 -0400 Subject: [PATCH 340/354] cut/pasted to the wrong place --- .../edu/harvard/iq/dataverse/api/Datasets.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 3385be07795..97f082a8db2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1241,16 +1241,15 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin ds.getLatestVersion().setVersionNumber(Long.valueOf(ds.getVersionNumber() + 1)); ds.getLatestVersion().setMinorVersionNumber(Long.valueOf(0)); } - if(ds.getLatestVersion().getVersionNumber()==1 && ds.getLatestVersion().getMinorVersionNumber()==0) { - //Also set publication date if this is the first - if(dateTime != null) { - ds.setPublicationDate(Timestamp.valueOf(dateTime)); - } - // Release User is only set in FinalizeDatasetPublicationCommand if the pub date - // is null, so set it here. - ds.setReleaseUser((AuthenticatedUser) user); + } + if(ds.getLatestVersion().getVersionNumber()==1 && ds.getLatestVersion().getMinorVersionNumber()==0) { + //Also set publication date if this is the first + if(dateTime != null) { + ds.setPublicationDate(Timestamp.valueOf(dateTime)); } - + // Release User is only set in FinalizeDatasetPublicationCommand if the pub date + // is null, so set it here. + ds.setReleaseUser((AuthenticatedUser) user); } } catch (Exception e) { logger.fine(e.getMessage()); From f4c6112c48387c1e3534fab99c61e182044a0e16 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 28 Jul 2021 10:11:38 -0400 Subject: [PATCH 341/354] updates from code review, adding in notes from recently merged PR --- doc/release-notes/5.6-release-notes.md | 8 ++++++-- doc/release-notes/6497-migrate-api.md | 7 ------- 2 files changed, 6 insertions(+), 9 deletions(-) delete mode 100644 doc/release-notes/6497-migrate-api.md diff --git a/doc/release-notes/5.6-release-notes.md b/doc/release-notes/5.6-release-notes.md index 3928ebfa39c..2801e1525b8 100644 --- a/doc/release-notes/5.6-release-notes.md +++ b/doc/release-notes/5.6-release-notes.md @@ -18,6 +18,12 @@ Dataset metadata can be retrieved, set, and updated using a new, flatter JSON-LD This development was supported by the [Research Data Alliance](https://rd-alliance.org), DANS, and Sciences PO and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). +### Dataset Migration API (Experimental) + +Datasets can now imported following the format of an OAI-ORE export (RDA-conformant Bags), allowing for easier migration from one Dataverse installation to another, and migration from other systems. This experimental, superuser only, endpoint also allows keeping the existing persistent identifier (where the authority and shoulder match those for which the software is configured) and publication dates. + +This development was supported by DANS and the [Research Data Alliance](https://rd-alliance.org) and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). + ### Direct Upload API Now Available for Adding Multiple Files Metadata to the Dataset Users can now add metadata of multiple files to the dataset once the files exists in the s3 bucket using the direct upload API. This makes direct uploads more efficient and reduces server load by only updating the dataset once instead of once per file. For more information, see the [Direct DataFile Upload/Replace API section](https://guides.dataverse.org/en/5.6/developers/s3-direct-upload-api.html) of the Dataverse Software Guides. @@ -74,7 +80,6 @@ If this is a new installation, please see our [Installation Guide](https://guide The steps below include a required upgrade to Payara 5.2021.4 or higher. (It is a simple matter of reusing your existing domain directory with the new distribution). But we also recommend that you review the Payara upgrade instructions as it could be helpful during any troubleshooting: [Payara documentation](https://docs.payara.fish/community/docs/5.2021.4/documentation/user-guides/upgrade-payara.html) - If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user. In the following commands we assume that Payara 5 is installed in `/usr/local/payara5`. If not, adjust as needed. @@ -88,7 +93,6 @@ In the following commands we assume that Payara 5 is installed in `/usr/local/pa - `$PAYARA/bin/asadmin list-applications` - `$PAYARA/bin/asadmin undeploy dataverse<-version>` - 2\. Stop Payara - `service payara stop` diff --git a/doc/release-notes/6497-migrate-api.md b/doc/release-notes/6497-migrate-api.md deleted file mode 100644 index 7b351de0a38..00000000000 --- a/doc/release-notes/6497-migrate-api.md +++ /dev/null @@ -1,7 +0,0 @@ -# Release Highlights - -### Dataset Migration API (Experimental) - -Datasets can now imported following the format of an OAI-ORE export (RDA-conformant Bags), allowing for easier migration from one Dataverse installation to another, and migration from other systems. This experimental, super-user only, endpoint also allows keeping the existing persistent identifier (where the authority and shoulder match those for which the software is configured) and publication dates. - -This development was supported by DANS and the [Research Data Alliance](https://rd-alliance.org) and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025). From ae56c4adea3d612281858bc8026560482723a349 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 28 Jul 2021 17:02:24 +0200 Subject: [PATCH 342/354] chore(payara): update everything to Payara 5.2021.5 #8030 --- conf/docker-aio/0prep_deps.sh | 4 ++-- conf/docker-aio/c8.dockerfile | 2 +- doc/release-notes/7700-upgrade-payara.md | 6 +++--- doc/sphinx-guides/source/developers/dev-environment.rst | 4 ++-- doc/sphinx-guides/source/installation/prerequisites.rst | 6 +++--- downloads/download.sh | 2 +- pom.xml | 2 +- scripts/vagrant/setup.sh | 2 +- .../edu/harvard/iq/dataverse/util/DataSourceProducer.java | 2 +- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/conf/docker-aio/0prep_deps.sh b/conf/docker-aio/0prep_deps.sh index c26492b2d25..f1832fd2ec3 100755 --- a/conf/docker-aio/0prep_deps.sh +++ b/conf/docker-aio/0prep_deps.sh @@ -4,10 +4,10 @@ if [ ! -d dv/deps ]; then fi wdir=`pwd` -if [ ! -e dv/deps/payara-5.2021.4.zip ]; then +if [ ! -e dv/deps/payara-5.2021.5.zip ]; then echo "payara dependency prep" # no more fiddly patching :) - wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.4/payara-5.2021.4.zip -O dv/deps/payara-5.2021.4.zip + wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip -O dv/deps/payara-5.2021.5.zip fi if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then diff --git a/conf/docker-aio/c8.dockerfile b/conf/docker-aio/c8.dockerfile index 82d3fdac023..515f69b8c55 100644 --- a/conf/docker-aio/c8.dockerfile +++ b/conf/docker-aio/c8.dockerfile @@ -24,7 +24,7 @@ COPY disableipv6.conf /etc/sysctl.d/ RUN rm /etc/httpd/conf/* COPY httpd.conf /etc/httpd/conf RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz -RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.4.zip ; ln -s /opt/payara5 /opt/glassfish4 +RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.5.zip ; ln -s /opt/payara5 /opt/glassfish4 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm` # this dies under Java 11, do we keep it? diff --git a/doc/release-notes/7700-upgrade-payara.md b/doc/release-notes/7700-upgrade-payara.md index de7c58bf963..40e9552bfe3 100644 --- a/doc/release-notes/7700-upgrade-payara.md +++ b/doc/release-notes/7700-upgrade-payara.md @@ -1,9 +1,9 @@ -### Payara 5.2021.4 (or Higher) Required +### Payara 5.2021.5 (or Higher) Required -Some changes in this release require an upgrade to Payara 5.2021.4 or higher. +Some changes in this release require an upgrade to Payara 5.2021.5 or higher. Instructions on how to update can be found in the -[Payara documentation](https://docs.payara.fish/community/docs/5.2021.4/documentation/user-guides/upgrade-payara.html) +[Payara documentation](https://docs.payara.fish/community/docs/5.2021.5/documentation/user-guides/upgrade-payara.html) It would likely be safer to upgrade Payara first, while still running Dataverse 5.6, and then proceed with the steps below. Upgrading from an earlier version of Payara should be a straightforward process: diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 264324fb604..61ab98bf292 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -85,9 +85,9 @@ To install Payara, run the following commands: ``cd /usr/local`` -``sudo curl -O -L https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.4/payara-5.2021.4.zip`` +``sudo curl -O -L https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip`` -``sudo unzip payara-5.2021.4.zip`` +``sudo unzip payara-5.2021.5.zip`` ``sudo chown -R $USER /usr/local/payara5`` diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 0265e390d14..e3dc04ac70b 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -44,7 +44,7 @@ On RHEL/derivative you can make Java 11 the default with the ``alternatives`` co Payara ------ -Payara 5.2021.4 is recommended. Newer versions might work fine, regular updates are recommended. +Payara 5.2021.5 is recommended. Newer versions might work fine, regular updates are recommended. Installing Payara ================= @@ -55,8 +55,8 @@ Installing Payara - Download and install Payara (installed in ``/usr/local/payara5`` in the example commands below):: - # wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.4/payara-5.2021.4.zip - # unzip payara-5.2021.4.zip + # wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip + # unzip payara-5.2021.5.zip # mv payara5 /usr/local If you intend to install and run Payara under a service account (and we hope you do), chown -R the Payara hierarchy to root to protect it but give the service account access to the below directories: diff --git a/downloads/download.sh b/downloads/download.sh index c9cd2942295..33476c24b76 100755 --- a/downloads/download.sh +++ b/downloads/download.sh @@ -1,5 +1,5 @@ #!/bin/sh -curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.4/payara-5.2021.4.zip +curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip curl -L -O https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar diff --git a/pom.xml b/pom.xml index 92b0ff3d951..f76a2708752 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ false 8.0.0 - 5.2021.4 + 5.2021.5 42.2.19 1.11.762 1.2 diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh index 292c5b56f63..24bac307709 100644 --- a/scripts/vagrant/setup.sh +++ b/scripts/vagrant/setup.sh @@ -52,7 +52,7 @@ SOLR_USER=solr echo "Ensuring Unix user '$SOLR_USER' exists" useradd $SOLR_USER || : DOWNLOAD_DIR='/dataverse/downloads' -PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2021.4.zip" +PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2021.5.zip" SOLR_TGZ="$DOWNLOAD_DIR/solr-8.8.1.tgz" if [ ! -f $PAYARA_ZIP ] || [ ! -f $SOLR_TGZ ]; then echo "Couldn't find $PAYARA_ZIP or $SOLR_TGZ! Running download script...." diff --git a/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java b/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java index b393ca4a605..e9ac5bfd230 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java @@ -38,7 +38,7 @@ //}) // // ... but at this time we don't think we need any. The full list -// of properties can be found at https://docs.payara.fish/community/docs/5.2021.4/documentation/payara-server/jdbc/advanced-connection-pool-properties.html#full-list-of-properties +// of properties can be found at https://docs.payara.fish/community/docs/5.2021.5/documentation/payara-server/jdbc/advanced-connection-pool-properties.html#full-list-of-properties // // All these properties cannot be configured via MPCONFIG as Payara doesn't support this (yet). To be enhanced. // See also https://github.com/payara/Payara/issues/5024 From 259e26d748e50f62a32f7b232978954e9b219fd4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 28 Jul 2021 12:07:39 -0400 Subject: [PATCH 343/354] add flag required to also update PID at provider during migrate --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 97f082a8db2..f382320d8a2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1206,7 +1206,7 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S @POST @Path("{id}/actions/:releasemigrated") @Consumes("application/ld+json, application/json-ld") - public Response publishMigratedDataset(String jsonldBody, @PathParam("id") String id) { + public Response publishMigratedDataset(String jsonldBody, @PathParam("id") String id, @DefaultValue("false") @QueryParam ("updatepidatprovider") boolean contactPIDProvider) { try { AuthenticatedUser user = findAuthenticatedUserOrDie(); if (!user.isSuperuser()) { @@ -1270,11 +1270,11 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin if (prePubWf.isPresent()) { // Start the workflow, the workflow will call FinalizeDatasetPublication later wfService.start(prePubWf.get(), - new WorkflowContext(createDataverseRequest(user), ds, TriggerType.PrePublishDataset, false), + new WorkflowContext(createDataverseRequest(user), ds, TriggerType.PrePublishDataset, !contactPIDProvider), false); } else { FinalizeDatasetPublicationCommand cmd = new FinalizeDatasetPublicationCommand(ds, - createDataverseRequest(user), false); + createDataverseRequest(user), !contactPIDProvider); ds = commandEngine.submit(cmd); } } catch (CommandException ex) { From 0bfda2ef7c0a7927320a38f70669b50985c908ae Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 28 Jul 2021 12:44:44 -0400 Subject: [PATCH 344/354] update docs --- doc/sphinx-guides/source/admin/dataverses-datasets.rst | 4 +++- .../source/developers/dataset-migration-api.rst | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index a55c90d2eb3..a18204588c2 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -41,7 +41,7 @@ Recursively assigns the users and groups having a role(s),that are in the set co curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/addRoleAssignmentsToChildren Configure a Dataverse Collection to store all new files in a specific file store -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ To direct new files (uploaded when datasets are created or edited) for all datasets in a given Dataverse collection, the store can be specified via the API as shown below, or by editing the 'General Information' for a Dataverse collection on the Dataverse collection page. Only accessible to superusers. :: @@ -110,6 +110,8 @@ Mints a new identifier for a dataset previously registered with a handle. Only a curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://$SERVER/api/admin/$dataset-id/reregisterHDLToPID +.. _send-metadata-to-pid-provider: + Send Dataset metadata to PID provider ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/sphinx-guides/source/developers/dataset-migration-api.rst b/doc/sphinx-guides/source/developers/dataset-migration-api.rst index d46db5e3c9e..b01cdadae79 100644 --- a/doc/sphinx-guides/source/developers/dataset-migration-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-migration-api.rst @@ -7,6 +7,7 @@ This experimental migration API offers an additional option with some potential * metadata can be specified using the json-ld format used in the OAI-ORE metadata export * existing publication dates and PIDs are maintained (currently limited to the case where the PID can be managed by the Dataverse software, e.g. where the authority and shoulder match those the software is configured for) +* updating the PID at the provider can be done immediately or later (with other existing APIs) * adding files can be done via the standard APIs, including using direct-upload to S3 This API consists of 2 calls: one to create an initial Dataset version, and one to 'republish' the dataset through Dataverse with a specified publication date. @@ -47,3 +48,9 @@ The call above creates a Dataset. Once it is created, other APIs can be used to curl -H 'Content-Type: application/jsonld' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated" datePublished is the only metadata supported in this call. + +An optional query parameter: updatepidatprovider (default is false) can be set to true to automatically update the metadata and targetUrl of the PID at the provider. With this set true, the result of this call will be that the PID redirects to this dataset rather than the dataset in the source repository. + + curl -H 'Content-Type: application/jsonld' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated?updatepidatprovider=true" + + If the parameter is not added and set to true, other existing APIs can be used to update the PID at the provider later, e.g. :ref:`send-metadata-to-pid-provider` \ No newline at end of file From e38305d1896065f599b5adbb6dfa561fa74ca11f Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 29 Jul 2021 11:41:24 -0400 Subject: [PATCH 345/354] #7790 delete physical files on ds version delete --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 4f73e64d5a4..be960082bd6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -2777,14 +2777,22 @@ public String editFileMetadata(){ public String deleteDatasetVersion() { DeleteDatasetVersionCommand cmd; + + Map deleteStorageLocations = datafileService.getPhysicalFilesToDelete(dataset.getLatestVersion()); + boolean deleteCommandSuccess = false; try { cmd = new DeleteDatasetVersionCommand(dvRequestService.getDataverseRequest(), dataset); commandEngine.submit(cmd); JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("datasetVersion.message.deleteSuccess")); + deleteCommandSuccess = true; } catch (CommandException ex) { JH.addMessage(FacesMessage.SEVERITY_FATAL, BundleUtil.getStringFromBundle("dataset.message.deleteFailure")); logger.severe(ex.getMessage()); } + + if (deleteCommandSuccess && !deleteStorageLocations.isEmpty()) { + datafileService.finalizeFileDeletes(deleteStorageLocations); + } return returnToDatasetOnly(); } From ab5eb62435c35c1e4b4aab63d190f754ed1d7383 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sun, 1 Aug 2021 15:10:30 -0400 Subject: [PATCH 346/354] fix for mistmatched tags and skipped tabular files --- .../edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 693e3cd6b10..9248069c8c1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -1425,7 +1425,8 @@ public static void createDataDscr(XMLStreamWriter xmlw, DatasetVersion datasetVe * These days we return early to avoid this exposure. */ if (dataFile.isRestricted()) { - return; + //Skip this file but don't exit the loop so that tabular info from non-restricted files still get written + continue; } if (dataFile != null && dataFile.isTabularData()) { From 2f6e5c7fac3be40db003fe768fde1db2ca8e88f1 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Mon, 2 Aug 2021 17:08:29 -0400 Subject: [PATCH 347/354] Update 5.6-release-notes.md --- doc/release-notes/5.6-release-notes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/5.6-release-notes.md b/doc/release-notes/5.6-release-notes.md index 2801e1525b8..e8a82e4f3b1 100644 --- a/doc/release-notes/5.6-release-notes.md +++ b/doc/release-notes/5.6-release-notes.md @@ -78,7 +78,7 @@ If this is a new installation, please see our [Installation Guide](https://guide 0\. These instructions assume that you've already successfully upgraded from Dataverse Software 4.x to Dataverse Software 5 following the instructions in the [Dataverse Software 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). After upgrading from the 4.x series to 5.0, you should progress through the other 5.x releases before attempting the upgrade to 5.6. -The steps below include a required upgrade to Payara 5.2021.4 or higher. (It is a simple matter of reusing your existing domain directory with the new distribution). But we also recommend that you review the Payara upgrade instructions as it could be helpful during any troubleshooting: [Payara documentation](https://docs.payara.fish/community/docs/5.2021.4/documentation/user-guides/upgrade-payara.html) +The steps below include a required upgrade to Payara 5.2021.5 or higher. (It is a simple matter of reusing your existing domain directory with the new distribution). But we also recommend that you review the Payara upgrade instructions as it could be helpful during any troubleshooting: [Payara documentation](https://docs.payara.fish/community/docs/5.2021.5/documentation/user-guides/upgrade-payara.html) If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user. @@ -102,7 +102,7 @@ In the following commands we assume that Payara 5 is installed in `/usr/local/pa - `mv $PAYARA $PAYARA.MOVED` -4\. Download the new Payara version (5.2021.4+), and unzip it in its place +4\. Download the new Payara version (5.2021.5+), and unzip it in its place 5\. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved `domain1` From b50def3737e829438dad3dbcc4d55ac6a8fbd6ef Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 2 Aug 2021 17:20:10 -0400 Subject: [PATCH 348/354] format fixes --- doc/sphinx-guides/source/developers/dataset-migration-api.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/dataset-migration-api.rst b/doc/sphinx-guides/source/developers/dataset-migration-api.rst index b01cdadae79..df31f09fc4b 100644 --- a/doc/sphinx-guides/source/developers/dataset-migration-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-migration-api.rst @@ -51,6 +51,8 @@ datePublished is the only metadata supported in this call. An optional query parameter: updatepidatprovider (default is false) can be set to true to automatically update the metadata and targetUrl of the PID at the provider. With this set true, the result of this call will be that the PID redirects to this dataset rather than the dataset in the source repository. +.. code-block:: bash + curl -H 'Content-Type: application/jsonld' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated?updatepidatprovider=true" - If the parameter is not added and set to true, other existing APIs can be used to update the PID at the provider later, e.g. :ref:`send-metadata-to-pid-provider` \ No newline at end of file +If the parameter is not added and set to true, other existing APIs can be used to update the PID at the provider later, e.g. :ref:`send-metadata-to-pid-provider` \ No newline at end of file From de648cef5d71a6e724cdc8fcf5d1ec65b30ec076 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 3 Aug 2021 10:18:47 -0400 Subject: [PATCH 349/354] remove fine log stmt that has a NPE --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 90847ad127a..1e868fa0fc7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -104,12 +104,6 @@ public static Dataset updateDatasetMDFromJsonLD(Dataset ds, String jsonLDBody, LocalDateTime dateTime = getDateTimeFrom(dateString); ds.setModificationTime(Timestamp.valueOf(dateTime)); } - try { - logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString()); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } } return ds; } From 3baaa5b436470113146ad9329d8cf2a9272345ea Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 3 Aug 2021 10:40:24 -0400 Subject: [PATCH 350/354] fix content-type in docs --- doc/sphinx-guides/source/developers/dataset-migration-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dataset-migration-api.rst b/doc/sphinx-guides/source/developers/dataset-migration-api.rst index df31f09fc4b..1dc8f7866e0 100644 --- a/doc/sphinx-guides/source/developers/dataset-migration-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-migration-api.rst @@ -45,7 +45,7 @@ The call above creates a Dataset. Once it is created, other APIs can be used to export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org - curl -H 'Content-Type: application/jsonld' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated" + curl -H 'Content-Type: application/ld+json' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated" datePublished is the only metadata supported in this call. @@ -53,6 +53,6 @@ An optional query parameter: updatepidatprovider (default is false) can be set t .. code-block:: bash - curl -H 'Content-Type: application/jsonld' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated?updatepidatprovider=true" + curl -H 'Content-Type: application/ld+json' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated?updatepidatprovider=true" If the parameter is not added and set to true, other existing APIs can be used to update the PID at the provider later, e.g. :ref:`send-metadata-to-pid-provider` \ No newline at end of file From eca842f9bee9e9e49ccf5f052494a4567f134bad Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Tue, 3 Aug 2021 11:22:25 -0400 Subject: [PATCH 351/354] Update DdiExportUtil.java --- .../edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 9248069c8c1..9061c890f01 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -1422,10 +1422,9 @@ public static void createDataDscr(XMLStreamWriter xmlw, DatasetVersion datasetVe * included for restricted files but that meant that summary * statistics were exposed. (To get at these statistics, API users * should instead use the "Data Variable Metadata Access" endpoint.) - * These days we return early to avoid this exposure. + * These days we skip restricted files to avoid this exposure. */ if (dataFile.isRestricted()) { - //Skip this file but don't exit the loop so that tabular info from non-restricted files still get written continue; } From 200453e54917f8bf5d1f420806891aa0a903e4a3 Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Wed, 4 Aug 2021 14:25:04 -0400 Subject: [PATCH 352/354] Update conf.py Update to 5.6 --- doc/sphinx-guides/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 30c53f9febf..a68a623d24e 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -65,9 +65,9 @@ # built documents. # # The short X.Y version. -version = '5.5' +version = '5.6' # The full version, including alpha/beta/rc tags. -release = '5.5' +release = '5.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 0d855fa69e711a7e5dc3a410321347281e9d4874 Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Wed, 4 Aug 2021 14:25:46 -0400 Subject: [PATCH 353/354] Update versions.rst Update to 5.6 --- doc/sphinx-guides/source/versions.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index 9c640a99aa7..99f088db41c 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -6,8 +6,9 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. -- 5.5 +- 5.6 +- `5.5 `__ - `5.4.1 `__ - `5.4 `__ - `5.3 `__ From d6c6ef4cac46956c40851903f16c4afac2616f75 Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Wed, 4 Aug 2021 14:27:21 -0400 Subject: [PATCH 354/354] Update pom.xml Update to 5.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f76a2708752..bb53d1efa64 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ --> edu.harvard.iq dataverse - 5.5 + 5.6 war dataverse