From 3967820d673d4068c6c35b1189939b431fff143d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 29 Jul 2022 13:57:30 -0400 Subject: [PATCH 01/14] getEditVersion will create a new version if one doesn't exist and we don't want that in a check of the terms --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 251f3865104..37174d02e14 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -404,7 +404,7 @@ public Boolean isHasValidTermsOfAccess() { hasValidTermsOfAccess = true; return hasValidTermsOfAccess; } else { - hasValidTermsOfAccess = TermsOfUseAndAccessValidator.isTOUAValid(dataset.getEditVersion().getTermsOfUseAndAccess(), null); + hasValidTermsOfAccess = TermsOfUseAndAccessValidator.isTOUAValid(dataset.getLatestVersion().getTermsOfUseAndAccess(), null); return hasValidTermsOfAccess; } } From 05e026ed137dcbd4339ba218615bb686bb5bc498 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Jul 2022 20:01:44 -0400 Subject: [PATCH 02/14] fix for file delete --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 37174d02e14..c61b159e685 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -3439,10 +3439,6 @@ private void deleteFiles(List filesToDelete) { // passed to the UpdateDatasetCommand. -- L.A. Aug 2017 FileMetadataUtil.removeFileMetadataFromList(workingVersion.getFileMetadatas(), markedForDelete); - - FileMetadataUtil.removeDataFileFromList(newFiles, markedForDelete.getDataFile()); - FileUtil.deleteTempFile(markedForDelete.getDataFile(), dataset, ingestService); - } } From 23f84b0c162fabbf1313880cf21947aa2f8d08ca Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Jul 2022 20:07:49 -0400 Subject: [PATCH 03/14] null dataset thumb if being deleted --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index c61b159e685..2d51d52b17b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -3416,6 +3416,10 @@ private void deleteFiles(List filesToDelete) { refreshSelectedFiles(filesToDelete); } + if(filesToDelete.contains(dataset.getThumbnailFile().getFileMetadata())) { + dataset.setThumbnailFile(null); + } + for (FileMetadata markedForDelete : filesToDelete) { if (markedForDelete.getId() != null) { @@ -3438,6 +3442,7 @@ private void deleteFiles(List filesToDelete) { // NOT adding the file to the filesToBeDeleted list that will be // passed to the UpdateDatasetCommand. -- L.A. Aug 2017 + FileMetadataUtil.removeFileMetadataFromList(workingVersion.getFileMetadatas(), markedForDelete); } } From aa2dc3631e5ab7f0558c94e32662c581123828ea Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Jul 2022 20:16:17 -0400 Subject: [PATCH 04/14] add null check --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 2d51d52b17b..22c5d0dea8a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -3416,7 +3416,7 @@ private void deleteFiles(List filesToDelete) { refreshSelectedFiles(filesToDelete); } - if(filesToDelete.contains(dataset.getThumbnailFile().getFileMetadata())) { + if((dataset.getThumbnailFile()!=null) && filesToDelete.contains(dataset.getThumbnailFile().getFileMetadata())) { dataset.setThumbnailFile(null); } From 277d16c7f7cda79247b90bd274a25299e8485854 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Jul 2022 20:37:01 -0400 Subject: [PATCH 05/14] also clear from new file list --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 22c5d0dea8a..f54e69695a7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -3442,8 +3442,9 @@ private void deleteFiles(List filesToDelete) { // NOT adding the file to the filesToBeDeleted list that will be // passed to the UpdateDatasetCommand. -- L.A. Aug 2017 - FileMetadataUtil.removeFileMetadataFromList(workingVersion.getFileMetadatas(), markedForDelete); + FileMetadataUtil.removeDataFileFromList(newFiles, markedForDelete.getDataFile()); + } } From 9fac1fd1c4b99bdd645b6562adce164b53e00737 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 1 Aug 2022 13:15:46 -0400 Subject: [PATCH 06/14] loop and compare on datafile FWIW: looking for the dataset.getThumbnailFile.getFileMetadata() in the filesToDelete list fails in the delete from a published version case (assuming getFileMetadata() doesn't return the dynamically created fmd for the new draft version) --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index f54e69695a7..48e8309a80b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -3416,8 +3416,11 @@ private void deleteFiles(List filesToDelete) { refreshSelectedFiles(filesToDelete); } - if((dataset.getThumbnailFile()!=null) && filesToDelete.contains(dataset.getThumbnailFile().getFileMetadata())) { - dataset.setThumbnailFile(null); + for (FileMetadata fmd : filesToDelete) { + if(fmd.getDataFile().equals(dataset.getThumbnailFile())) { + dataset.setThumbnailFile(null); + break; + } } for (FileMetadata markedForDelete : filesToDelete) { From d50412a45008a3dde364cda549c328664053bd84 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 1 Aug 2022 14:43:19 -0400 Subject: [PATCH 07/14] don't loop unless there is a thumb --- .../java/edu/harvard/iq/dataverse/DatasetPage.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 48e8309a80b..1a2bcee4b12 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -3416,10 +3416,12 @@ private void deleteFiles(List filesToDelete) { refreshSelectedFiles(filesToDelete); } - for (FileMetadata fmd : filesToDelete) { - if(fmd.getDataFile().equals(dataset.getThumbnailFile())) { - dataset.setThumbnailFile(null); - break; + if (dataset.getThumbnailFile() != null) { + for (FileMetadata fmd : filesToDelete) { + if (fmd.getDataFile().equals(dataset.getThumbnailFile())) { + dataset.setThumbnailFile(null); + break; + } } } From de25eac4351863eaf3a9c07a092a6e9d957d45b3 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 1 Aug 2022 14:54:52 -0400 Subject: [PATCH 08/14] test work-around per pdurbin --- .../edu/harvard/iq/dataverse/export/OpenAireExporterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExporterTest.java b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExporterTest.java index 62f6d3037a1..505496fca7a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExporterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExporterTest.java @@ -97,7 +97,7 @@ public void testValidateExportDataset() throws Exception { } InputStream xmlStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray()); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setValidating(true); + //factory.setValidating(true); factory.setNamespaceAware(true); factory.setAttribute("http://java.sun.com/xml/jaxp/properties/schemaLanguage", "http://www.w3.org/2001/XMLSchema"); From d48f797d143dec38614dbe62529c2aa2bccc45bb Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 1 Aug 2022 17:00:15 -0400 Subject: [PATCH 09/14] add 5.11.1 release notes #8878 --- doc/release-notes/5.11.1-release-notes.md | 62 +++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 doc/release-notes/5.11.1-release-notes.md diff --git a/doc/release-notes/5.11.1-release-notes.md b/doc/release-notes/5.11.1-release-notes.md new file mode 100644 index 00000000000..c0978069379 --- /dev/null +++ b/doc/release-notes/5.11.1-release-notes.md @@ -0,0 +1,62 @@ +# Dataverse Software 5.11.1 + +This is a bug fix release of the Dataverse Software. Dataverse v5.11 will no longer be made available and installations should upgrade directly from v5.10.1 to v5.11.1. + +## Release Highlights + +Dataverse Software 5.11 contains two critical issues that are fixed in this release. + +First, if you delete a file from a published version of a dataset that has restricted files, the file will be deleted from the file system (or S3) and lose its "owner id" in the database. For details, see Issue #8867. + +Second, if you are a superuser, it's possible to click "Delete Draft" and delete a published dataset if it has restricted files. For details, see #8845 and #8742. + +## Notes for Dataverse Installation Administrators + +### Identifying Datasets with Deleted Files + +If you have been running 5.11, check if any files show "null" for the owner id. The "owner" of a file is the parent dataset: + +``` +select * from dvobject where dtype = 'DataFile' and owner_id is null; +``` + +For any of these files, change the owner id to the database id of the parent dataset. In addition, the file on disk (or in S3) is likely gone. Look at the "storageidentifier" field from the query above to determine the location of the file then restore the file from backup. + +## Installation + +If this is a new installation, please see our [Installation Guide](https://guides.dataverse.org/en/5.11.1/installation/). Please also contact us to get added to the [Dataverse Project Map](https://guides.dataverse.org/en/5.11.1/installation/config.html#putting-your-dataverse-installation-on-the-map-at-dataverse-org) if you have not done so already. + +## Upgrade Instructions + +0\. These instructions assume that you've already successfully upgraded from Dataverse Software 4.x to Dataverse Software 5 following the instructions in the [Dataverse Software 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). After upgrading from the 4.x series to 5.0, you should progress through the other 5.x releases before attempting the upgrade to 5.11.1. + +If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user. + +In the following commands we assume that Payara 5 is installed in `/usr/local/payara5`. If not, adjust as needed. + +`export PAYARA=/usr/local/payara5` + +(or `setenv PAYARA /usr/local/payara5` if you are using a `csh`-like shell) + +1\. Undeploy the previous version. + +- `$PAYARA/bin/asadmin list-applications` +- `$PAYARA/bin/asadmin undeploy dataverse<-version>` + +2\. Stop Payara and remove the generated directory + +- `service payara stop` +- `rm -rf $PAYARA/glassfish/domains/domain1/generated` + +3\. Start Payara + +- `service payara start` + +4\. Deploy this version. + +- `$PAYARA/bin/asadmin deploy dataverse-5.11.1war` + +5\. Restart Payara + +- `service payara stop` +- `service payara start` From 1361067d058f8cbfffd8f492d28389df94966672 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 1 Aug 2022 17:05:18 -0400 Subject: [PATCH 10/14] bump version to 5.11.1 #8878 --- doc/sphinx-guides/source/conf.py | 4 ++-- doc/sphinx-guides/source/versions.rst | 4 ++-- modules/dataverse-parent/pom.xml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 82914a35544..187d6ee73f1 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -66,9 +66,9 @@ # built documents. # # The short X.Y version. -version = '5.11' +version = '5.11.' # The full version, including alpha/beta/rc tags. -release = '5.11' +release = '5.11.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index e4518f4ca65..f46b9477d92 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -6,8 +6,8 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. -- 5.11 - +- 5.11.1 +- `5.11 `__ - `5.10.1 `__ - `5.10 `__ - `5.9 `__ diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index c2f536c4d41..22ea30795ba 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -129,7 +129,7 @@ - 5.11 + 5.11.1 11 UTF-8 From f79a95420d7f857d4aa3373f7d4f16e8f29a4e19 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 2 Aug 2022 10:12:36 -0400 Subject: [PATCH 11/14] tweak release notes #8878 Add suggestion on detecting destroyed datasets. Add milestone. --- doc/release-notes/5.11.1-release-notes.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/5.11.1-release-notes.md b/doc/release-notes/5.11.1-release-notes.md index c0978069379..d183f6ceb0b 100644 --- a/doc/release-notes/5.11.1-release-notes.md +++ b/doc/release-notes/5.11.1-release-notes.md @@ -1,6 +1,6 @@ # Dataverse Software 5.11.1 -This is a bug fix release of the Dataverse Software. Dataverse v5.11 will no longer be made available and installations should upgrade directly from v5.10.1 to v5.11.1. +This is a bug fix release of the Dataverse Software. Dataverse Software v5.11 will no longer be made available and installations should upgrade directly from v5.10.1 to v5.11.1. ## Release Highlights @@ -22,6 +22,16 @@ select * from dvobject where dtype = 'DataFile' and owner_id is null; For any of these files, change the owner id to the database id of the parent dataset. In addition, the file on disk (or in S3) is likely gone. Look at the "storageidentifier" field from the query above to determine the location of the file then restore the file from backup. +### Identifying Datasets Superusers May Have Accidentally Destroyed + +Check the "actionlogrecord" table for DestroyDatasetCommand. While these "destroy" entries are normal when a superuser uses the API to destroy datasets, an entry is also created if a superuser has accidentally deleted a published dataset in the web interface with the "Delete Draft" button. + +## Complete List of Changes + +For the complete list of code changes in this release, see the [5.11.1 Milestone](https://github.com/IQSS/dataverse/milestone/105?closed=1) in GitHub. + +For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org. + ## Installation If this is a new installation, please see our [Installation Guide](https://guides.dataverse.org/en/5.11.1/installation/). Please also contact us to get added to the [Dataverse Project Map](https://guides.dataverse.org/en/5.11.1/installation/config.html#putting-your-dataverse-installation-on-the-map-at-dataverse-org) if you have not done so already. From d463e486c266e954b435690e02fd86ffafbca7b7 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 2 Aug 2022 11:19:35 -0400 Subject: [PATCH 12/14] typo in war file name #8878 --- doc/release-notes/5.11.1-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/5.11.1-release-notes.md b/doc/release-notes/5.11.1-release-notes.md index d183f6ceb0b..f6000b10bb0 100644 --- a/doc/release-notes/5.11.1-release-notes.md +++ b/doc/release-notes/5.11.1-release-notes.md @@ -64,7 +64,7 @@ In the following commands we assume that Payara 5 is installed in `/usr/local/pa 4\. Deploy this version. -- `$PAYARA/bin/asadmin deploy dataverse-5.11.1war` +- `$PAYARA/bin/asadmin deploy dataverse-5.11.1.war` 5\. Restart Payara From 08ae12bd34e38c0600e7e03e156e304ae7e0d0fc Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 2 Aug 2022 11:21:30 -0400 Subject: [PATCH 13/14] fix version in sphinx #8878 --- doc/sphinx-guides/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 187d6ee73f1..c8175f21591 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -66,7 +66,7 @@ # built documents. # # The short X.Y version. -version = '5.11.' +version = '5.11.1' # The full version, including alpha/beta/rc tags. release = '5.11.1' From 0f185b59218091db726ad60a580cf960f88499b7 Mon Sep 17 00:00:00 2001 From: landreev Date: Tue, 2 Aug 2022 12:59:31 -0400 Subject: [PATCH 14/14] Update 5.11.1-release-notes.md Added some bold-faced upgrade guidelines (using language suggested by @qqmyers and my own) --- doc/release-notes/5.11.1-release-notes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/5.11.1-release-notes.md b/doc/release-notes/5.11.1-release-notes.md index f6000b10bb0..9baa61cbc6e 100644 --- a/doc/release-notes/5.11.1-release-notes.md +++ b/doc/release-notes/5.11.1-release-notes.md @@ -1,6 +1,6 @@ # Dataverse Software 5.11.1 -This is a bug fix release of the Dataverse Software. Dataverse Software v5.11 will no longer be made available and installations should upgrade directly from v5.10.1 to v5.11.1. +This is a bug fix release of the Dataverse Software. The .war file for v5.11 will no longer be made available and installations should upgrade directly from v5.10.1 to v5.11.1. To do so you will need **to follow the instructions for installing release 5.11 using the v5.11.1 war file**. (Note specifically the upgrade steps 6-9 from the 5.11 release note; most importantly, the ones related to the citation block and the Solr schema). **If you had previously installed v5.11** (no longer available), follow the simplified instructions below. ## Release Highlights @@ -38,7 +38,7 @@ If this is a new installation, please see our [Installation Guide](https://guide ## Upgrade Instructions -0\. These instructions assume that you've already successfully upgraded from Dataverse Software 4.x to Dataverse Software 5 following the instructions in the [Dataverse Software 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). After upgrading from the 4.x series to 5.0, you should progress through the other 5.x releases before attempting the upgrade to 5.11.1. +0\. These instructions assume that you've already successfully upgraded from Dataverse Software 4.x to Dataverse Software 5 following the instructions in the [Dataverse Software 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). After upgrading from the 4.x series to 5.0, you should progress through the other 5.x releases before attempting the upgrade to 5.11.1. **To upgrade from 5.10.1, follow the instructions for installing release 5.11 using the v5.11.1 war file**. If you had previously installed v5.11 (no longer available), follow the simplified instructions below. If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user.