From 9b2fb8d7338e7aec6c91e53efd6e40c896ec08b9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 24 Jan 2025 11:20:37 -0500 Subject: [PATCH] stop loading Codemeta in Docker, add setDisplayOnCreate API #10519 In Docker, we were testing with the Codemeta block. This made for a nice real-world story of creating a "software" dataset type and using it with the Codemeta block. The Codemeta block also has the advantage of having some fields that are set to displayOnCreate, which helped us make assertions that the code is working properly. It's the only non-citation block with fields set to displayOnCreate. However, Jenkins doesn't have the Codemeta block, meaning that tests are failing. Also, we aren't ready to promote the Codemeta block to be shipped with Dataverse because a new version is out: https://github.com/IQSS/dataverse/issues/10859 So, we are switching from Codemeta to the Astrophysics block. We create an "instrument" dataset type. Like all non-citation blocks (except for Codemeta), there are no fields that are set to displayOnCreate=true. Therefore, we added an API for this. --- doc/release-notes/10519-dataset-types.md | 4 +- doc/sphinx-guides/source/api/native-api.rst | 21 ++++++ .../scripts/bootstrap/dev/init.sh | 9 --- .../dataverse/api/DatasetFieldServiceApi.java | 16 +++++ .../iq/dataverse/api/DatasetTypesIT.java | 68 +++++++++++-------- .../iq/dataverse/api/MetadataBlocksIT.java | 5 +- .../edu/harvard/iq/dataverse/api/UtilIT.java | 7 ++ 7 files changed, 86 insertions(+), 44 deletions(-) diff --git a/doc/release-notes/10519-dataset-types.md b/doc/release-notes/10519-dataset-types.md index 1e1ce08ced3..99cf79a796f 100644 --- a/doc/release-notes/10519-dataset-types.md +++ b/doc/release-notes/10519-dataset-types.md @@ -7,6 +7,6 @@ This will have the following effects for the APIs used by the new Dataverse UI ( - The list of fields shown when creating a dataset will include fields marked as "displayoncreate" (in the tsv/database) for metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API. - The metadata blocks shown when editing a dataset will include metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API. -The CodeMeta metadata block is now available in the Dockerized development environment. +Mostly in order to write automated tests for the above, a [displayOnCreate](https://dataverse-guide--11001.org.readthedocs.build/en/11001/api/native-api.html#set-displayoncreate-for-a-dataset-field) API endpoint has been added. -For more information, see the guides and #10519. +For more information, see the guides ([overview](https://dataverse-guide--11001.org.readthedocs.build/en/11001/user/dataset-management.html#dataset-types), [new APIs](https://dataverse-guide--11001.org.readthedocs.build/en/11001/api/native-api.html#link-dataset-type-with-metadata-blocks)), #10519 and #11001. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 3042afbf2fa..f28187b965c 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5090,6 +5090,27 @@ The fully expanded example above (without environment variables) looks like this curl "https://demo.dataverse.org/api/datasetfields/facetables" +.. _setDisplayOnCreate: + +Set displayOnCreate for a Dataset Field +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set displayOnCreate for a dataset field. See also :doc:`/admin/metadatacustomization` in the Admin Guide. + +.. code-block:: bash + + export SERVER_URL=http://localhost:8080 + export FIELD=subtitle + export BOOLEAN=true + + curl -X POST "$SERVER_URL/api/admin/datasetfield/setDisplayOnCreate?datasetFieldType=$FIELD&setDisplayOnCreate=$BOOLEAN" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -X POST "http://localhost:8080/api/admin/datasetfield/setDisplayOnCreate?datasetFieldType=studyAssayCellType&setDisplayOnCreate=true" + .. _Notifications: Notifications diff --git a/modules/container-configbaker/scripts/bootstrap/dev/init.sh b/modules/container-configbaker/scripts/bootstrap/dev/init.sh index f70cc099e2d..f8770436652 100644 --- a/modules/container-configbaker/scripts/bootstrap/dev/init.sh +++ b/modules/container-configbaker/scripts/bootstrap/dev/init.sh @@ -17,15 +17,6 @@ export API_TOKEN # ${ENV_OUT} comes from bootstrap.sh and will expose the saved information back to the host if enabled. echo "API_TOKEN=${API_TOKEN}" >> "${ENV_OUT}" -echo "Loading CodeMeta metadata block (needed for API tests)..." -curl "${DATAVERSE_URL}/api/admin/datasetfield/load" -X POST --data-binary @/scripts/bootstrap/base/data/metadatablocks/codemeta.tsv -H "Content-type: text/tab-separated-values" - -echo "Fetching Solr schema from Dataverse and running update-fields.sh..." -curl "${DATAVERSE_URL}/api/admin/index/solr/schema" | /scripts/update-fields.sh /var/solr/data/collection1/conf/schema.xml - -echo "Reloading Solr..." -curl "http://solr:8983/solr/admin/cores?action=RELOAD&core=collection1" - echo "Publishing root dataverse..." curl -H "X-Dataverse-key:$API_TOKEN" -X POST "${DATAVERSE_URL}/api/dataverses/:root/actions/:publish" diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java index 907295ad848..cbb0f4ffcfd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java @@ -42,6 +42,7 @@ import java.util.logging.Logger; import jakarta.persistence.NoResultException; import jakarta.persistence.TypedQuery; +import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.Response.Status; import java.io.BufferedInputStream; @@ -545,4 +546,19 @@ public static String getDataverseLangDirectory() { return dataverseLangDirectory; } + /** + * Set setDisplayOnCreate for a DatasetFieldType. + */ + @POST + @Path("/setDisplayOnCreate") + public Response setDisplayOnCreate(@QueryParam("datasetFieldType") String datasetFieldTypeIn, @QueryParam("setDisplayOnCreate") boolean setDisplayOnCreateIn) { + DatasetFieldType dft = datasetFieldService.findByName(datasetFieldTypeIn); + if (dft == null) { + return error(Status.NOT_FOUND, "Cound not find a DatasetFieldType by looking up " + datasetFieldTypeIn); + } + dft.setDisplayOnCreate(setDisplayOnCreateIn); + DatasetFieldType saved = datasetFieldService.save(dft); + return ok("DisplayOnCreate for DatasetFieldType " + saved.getName() + " is now " + saved.isDisplayOnCreate()); + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java index c225ea3085a..7c73498dead 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java @@ -20,28 +20,32 @@ public class DatasetTypesIT { + final static String INSTRUMENT = "instrument"; + @BeforeAll public static void setUpClass() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); - Response getSoftwareType = UtilIT.getDatasetType(DatasetType.DATASET_TYPE_SOFTWARE); - getSoftwareType.prettyPrint(); - - String typeFound = JsonPath.from(getSoftwareType.getBody().asString()).getString("data.name"); - System.out.println("type found: " + typeFound); - if (DatasetType.DATASET_TYPE_SOFTWARE.equals(typeFound)) { - return; - } - - System.out.println("The \"software\" type wasn't found. Create it."); Response createUser = UtilIT.createRandomUser(); createUser.then().assertThat().statusCode(OK.getStatusCode()); String username = UtilIT.getUsernameFromResponse(createUser); String apiToken = UtilIT.getApiTokenFromResponse(createUser); UtilIT.setSuperuserStatus(username, true).then().assertThat().statusCode(OK.getStatusCode()); - String jsonIn = Json.createObjectBuilder().add("name", DatasetType.DATASET_TYPE_SOFTWARE).build().toString(); + ensureDatasetTypeIsPresent(DatasetType.DATASET_TYPE_SOFTWARE, apiToken); + ensureDatasetTypeIsPresent(INSTRUMENT, apiToken); + } + private static void ensureDatasetTypeIsPresent(String datasetType, String apiToken) { + Response getDatasetType = UtilIT.getDatasetType(datasetType); + getDatasetType.prettyPrint(); + String typeFound = JsonPath.from(getDatasetType.getBody().asString()).getString("data.name"); + System.out.println("type found: " + typeFound); + if (datasetType.equals(typeFound)) { + return; + } + System.out.println("The " + datasetType + "type wasn't found. Create it."); + String jsonIn = Json.createObjectBuilder().add("name", datasetType).build().toString(); Response typeAdded = UtilIT.addDatasetType(jsonIn, apiToken); typeAdded.prettyPrint(); typeAdded.then().assertThat().statusCode(OK.getStatusCode()); @@ -402,7 +406,7 @@ public void testUpdateDatasetTypeLinksWithMetadataBlocks() { } @Test - public void testLinkSoftwareToCodemeta() { + public void testLinkInstrumentToAstro() { Response createUser = UtilIT.createRandomUser(); createUser.then().assertThat().statusCode(OK.getStatusCode()); String username = UtilIT.getUsernameFromResponse(createUser); @@ -410,15 +414,15 @@ public void testLinkSoftwareToCodemeta() { UtilIT.setSuperuserStatus(username, true).then().assertThat().statusCode(OK.getStatusCode()); String metadataBlockLink = """ - ["codeMeta20"] + ["astrophysics"] //"""; - String datasetType = "software"; - Response linkSoftwareToCodemeta = UtilIT.updateDatasetTypeLinksWithMetadataBlocks(datasetType, metadataBlockLink, apiToken); - linkSoftwareToCodemeta.prettyPrint(); - linkSoftwareToCodemeta.then().assertThat(). + String datasetType = "instrument"; + Response linkInstrumentToAstro = UtilIT.updateDatasetTypeLinksWithMetadataBlocks(datasetType, metadataBlockLink, apiToken); + linkInstrumentToAstro.prettyPrint(); + linkInstrumentToAstro.then().assertThat(). statusCode(OK.getStatusCode()) - .body("data.linkedMetadataBlocks.after[0]", CoreMatchers.is("codeMeta20")); + .body("data.linkedMetadataBlocks.after[0]", CoreMatchers.is("astrophysics")); Response createDataverse = UtilIT.createRandomDataverse(apiToken); createDataverse.then().assertThat().statusCode(CREATED.getStatusCode()); @@ -428,6 +432,10 @@ public void testLinkSoftwareToCodemeta() { UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode()); + // displayOnCreate will only be true for fields that are set this way in the database. + // We set it here so we can make assertions below. + UtilIT.setDisplayOnCreate("astroInstrument", true); + Response listBlocks = null; System.out.println("listing root collection blocks with display on create using dataset type " + datasetType); listBlocks = UtilIT.listMetadataBlocks(":root", true, true, datasetType, apiToken); @@ -435,10 +443,10 @@ public void testLinkSoftwareToCodemeta() { listBlocks.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].name", is("citation")) - .body("data[1].name", is("codeMeta20")) + .body("data[1].name", is("astrophysics")) .body("data[2].name", nullValue()) .body("data[0].fields.title.displayOnCreate", equalTo(true)) - .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true)); + .body("data[1].fields.astroInstrument.displayOnCreate", equalTo(true)); System.out.println("listing root collection blocks with all fields (not display on create) using dataset type " + datasetType); listBlocks = UtilIT.listMetadataBlocks(":root", false, true, datasetType, apiToken); @@ -446,12 +454,12 @@ public void testLinkSoftwareToCodemeta() { listBlocks.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].name", is("citation")) - .body("data[1].name", is("codeMeta20")) + .body("data[1].name", is("astrophysics")) .body("data[2].name", nullValue()) .body("data[0].fields.title.displayOnCreate", equalTo(true)) .body("data[0].fields.subtitle.displayOnCreate", equalTo(false)) - .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true)) - .body("data[1].fields.issueTracker.displayOnCreate", equalTo(false)); + .body("data[1].fields.astroInstrument.displayOnCreate", equalTo(true)) + .body("data[1].fields.astroObject.displayOnCreate", equalTo(false)); System.out.println("listing " + dataverseAlias + " collection blocks with display on create using dataset type " + datasetType); listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, datasetType, apiToken); @@ -459,14 +467,14 @@ public void testLinkSoftwareToCodemeta() { listBlocks.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].name", is("citation")) - .body("data[1].name", is("codeMeta20")) + .body("data[1].name", is("astrophysics")) .body("data[2].name", nullValue()) .body("data[0].fields.title.displayOnCreate", equalTo(true)) // subtitle is hidden because it is not "display on create" .body("data[0].fields.subtitle", nullValue()) - .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true)) - // issueTracker is hidden because it is not "display on create" - .body("data[1].fields.issueTracker", nullValue()); + .body("data[1].fields.astroInstrument.displayOnCreate", equalTo(true)) + // astroObject is hidden because it is not "display on create" + .body("data[1].fields.astroObject", nullValue()); System.out.println("listing " + dataverseAlias + " collection blocks with all fields (not display on create) using dataset type " + datasetType); listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, false, true, datasetType, apiToken); @@ -474,12 +482,12 @@ public void testLinkSoftwareToCodemeta() { listBlocks.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].name", is("citation")) - .body("data[1].name", is("codeMeta20")) + .body("data[1].name", is("astrophysics")) .body("data[2].name", nullValue()) .body("data[0].fields.title.displayOnCreate", equalTo(true)) .body("data[0].fields.subtitle.displayOnCreate", equalTo(false)) - .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true)) - .body("data[1].fields.issueTracker.displayOnCreate", equalTo(false)); + .body("data[1].fields.astroInstrument.displayOnCreate", equalTo(true)) + .body("data[1].fields.astroObject.displayOnCreate", equalTo(false)); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java index 28cf6228a59..316ac579de4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java @@ -27,7 +27,7 @@ public static void setUpClass() { void testListMetadataBlocks() { // No optional params enabled Response listMetadataBlocksResponse = UtilIT.listMetadataBlocks(false, false); - int expectedDefaultNumberOfMetadataBlocks = 7; + int expectedDefaultNumberOfMetadataBlocks = 6; listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].fields", equalTo(null)) @@ -35,12 +35,11 @@ void testListMetadataBlocks() { // onlyDisplayedOnCreate=true listMetadataBlocksResponse = UtilIT.listMetadataBlocks(true, false); - int expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks = 2; + int expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks = 1; listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].fields", equalTo(null)) .body("data[0].displayName", equalTo("Citation Metadata")) - .body("data[1].displayName", equalTo("Software Metadata (CodeMeta v2.0)")) .body("data.size()", equalTo(expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks)); // returnDatasetFieldTypes=true diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 55baa965314..142b9cd8371 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -828,6 +828,13 @@ static Response getMetadataBlock(String block) { .get("/api/metadatablocks/" + block); } + static Response setDisplayOnCreate(String datasetFieldType, boolean setDisplayOnCreate) { + return given() + .queryParam("datasetFieldType", datasetFieldType) + .queryParam("setDisplayOnCreate", setDisplayOnCreate) + .post("/api/admin/datasetfield/setDisplayOnCreate"); + } + static private String getDatasetXml(String title, String author, String description) { String nullLicense = null; String nullRights = null;