[pull] develop from IQSS:develop

.env

@@ -1,4 +1,5 @@
 APP_IMAGE=gdcc/dataverse:unstable
-POSTGRES_VERSION=13
+POSTGRES_VERSION=16
 DATAVERSE_DB_USER=dataverse
 SOLR_VERSION=9.3.0
+SKIP_DEPLOY=0

.gitattributes

@@ -0,0 +1,4 @@
+# https://www.git-scm.com/docs/gitattributes
+
+# This set mandatory LF line endings for .sh files preventing from windows users to having to change the value of their git config --global core.autocrlf to 'false' or 'input'
+*.sh text eol=lf

.github/ISSUE_TEMPLATE/bug_report.md

@@ -3,7 +3,7 @@ name: Bug report
 about: Did you encounter something unexpected or incorrect in the Dataverse software?
   We'd like to hear about it!
 title: ''
-labels: ''
+labels: 'Type: Bug'
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,7 +2,7 @@
 name: Feature request
 about: Suggest an idea or new feature for the Dataverse software!
 title: 'Feature Request/Idea:'
-labels: ''
+labels: 'Type: Feature'
 assignees: ''
 
 ---

.github/workflows/maven_cache_management.yml

@@ -0,0 +1,101 @@
+name: Maven Cache Management
+
+on:
+    # Every push to develop should trigger cache rejuvenation (dependencies might have changed)
+    push:
+        branches:
+            - develop
+    # According to https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy
+    # all caches are deleted after 7 days of no access. Make sure we rejuvenate every 7 days to keep it available.
+    schedule:
+        - cron: '23 2 * * 0' # Run for 'develop' every Sunday at 02:23 UTC (3:23 CET, 21:23 ET)
+    # Enable manual cache management
+    workflow_dispatch:
+    # Delete branch caches once a PR is merged
+    pull_request:
+        types:
+            - closed
+
+env:
+    COMMON_CACHE_KEY: "dataverse-maven-cache"
+    COMMON_CACHE_PATH: "~/.m2/repository"
+
+jobs:
+    seed:
+        name: Drop and Re-Seed Local Repository
+        runs-on: ubuntu-latest
+        if: ${{ github.event_name != 'pull_request' }}
+        permissions:
+            # Write permission needed to delete caches
+            # See also: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-a-github-actions-cache-for-a-repository-using-a-cache-id
+            actions: write
+            contents: read
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+            - name: Determine Java version from Parent POM
+              run: echo "JAVA_VERSION=$(grep '<target.java.version>' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV}
+            - name: Set up JDK ${{ env.JAVA_VERSION }}
+              uses: actions/setup-java@v4
+              with:
+                  java-version: ${{ env.JAVA_VERSION }}
+                  distribution: temurin
+            - name: Seed common cache
+              run: |
+                  mvn -B -f modules/dataverse-parent dependency:go-offline dependency:resolve-plugins
+            # This non-obvious order is due to the fact that the download via Maven above will take a very long time (7-8 min).
+            # Jobs should not be left without a cache. Deleting and saving in one go leaves only a small chance for a cache miss.
+            - name: Drop common cache
+              run: |
+                  gh extension install actions/gh-actions-cache                  
+                  echo "🛒 Fetching list of cache keys"
+                  cacheKeys=$(gh actions-cache list -R ${{ github.repository }} -B develop | cut -f 1 )
+
+                  ## Setting this to not fail the workflow while deleting cache keys. 
+                  set +e
+                  echo "🗑️ Deleting caches..."
+                  for cacheKey in $cacheKeys
+                  do
+                      gh actions-cache delete $cacheKey -R ${{ github.repository }} -B develop --confirm
+                  done
+                  echo "✅ Done"
+              env:
+                  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+            - name: Save the common cache
+              uses: actions/cache@v4
+              with:
+                  path: ${{ env.COMMON_CACHE_PATH }}
+                  key: ${{ env.COMMON_CACHE_KEY }}
+                  enableCrossOsArchive: true
+
+    # Let's delete feature branch caches once their PR is merged - we only have 10 GB of space before eviction kicks in
+    deplete:
+        name: Deplete feature branch caches
+        runs-on: ubuntu-latest
+        if: ${{ github.event_name == 'pull_request' }}
+        permissions:
+            # `actions:write` permission is required to delete caches
+            # See also: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-a-github-actions-cache-for-a-repository-using-a-cache-id
+            actions: write
+            contents: read
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+            - name: Cleanup caches
+              run: |
+                  gh extension install actions/gh-actions-cache
+
+                  BRANCH=refs/pull/${{ github.event.pull_request.number }}/merge
+                  echo "🛒 Fetching list of cache keys"
+                  cacheKeysForPR=$(gh actions-cache list -R ${{ github.repository }} -B $BRANCH | cut -f 1 )
+
+                  ## Setting this to not fail the workflow while deleting cache keys. 
+                  set +e
+                  echo "🗑️ Deleting caches..."
+                  for cacheKey in $cacheKeysForPR
+                  do
+                      gh actions-cache delete $cacheKey -R ${{ github.repository }} -B $BRANCH --confirm
+                  done
+                  echo "✅ Done"
+              env:
+                  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/maven_unit_test.yml

@@ -1,16 +1,18 @@
-name: Maven Unit Tests
+name: Maven Tests
 
 on:
     push:
         paths:
             - "**.java"
+            - "**.sql"
             - "pom.xml"
             - "modules/**/pom.xml"
             - "!modules/container-base/**"
             - "!modules/dataverse-spi/**"
     pull_request:
         paths:
             - "**.java"
+            - "**.sql"
             - "pom.xml"
             - "modules/**/pom.xml"
             - "!modules/container-base/**"
@@ -28,6 +30,7 @@ jobs:
         continue-on-error: ${{ matrix.experimental }}
         runs-on: ubuntu-latest
         steps:
+          # Basic setup chores
           - uses: actions/checkout@v3
           - name: Set up JDK ${{ matrix.jdk }}
             uses: actions/setup-java@v3
@@ -37,16 +40,110 @@ jobs:
                 cache: maven
 
           # The reason why we use "install" here is that we want the submodules to be available in the next step.
-          # Also, we can cache them this way for jobs triggered by this one.
-          - name: Build with Maven
+          # Also, we can cache them this way for jobs triggered by this one. We need to skip ITs here, as we run
+          # them in the next job - but install usually runs through verify phase.
+          - name: Build with Maven and run unit tests
             run: > 
                 mvn -B -f modules/dataverse-parent
                 -Dtarget.java.version=${{ matrix.jdk }}
                 -DcompilerArgument=-Xlint:unchecked -P all-unit-tests
+                -DskipIntegrationTests
                 -pl edu.harvard.iq:dataverse -am
                 install
 
-          - name: Maven Code Coverage
+          # We don't want to cache the WAR file, so delete it
+          - run: rm -rf ~/.m2/repository/edu/harvard/iq/dataverse
+
+          # Upload the built war file. For download, it will be wrapped in a ZIP by GitHub.
+          # See also https://github.com/actions/upload-artifact#zipped-artifact-downloads
+          - uses: actions/upload-artifact@v3
+            with:
+                name: dataverse-java${{ matrix.jdk }}.war
+                path: target/dataverse*.war
+                retention-days: 7
+
+          # Store the build for the next step (integration test) to avoid recompilation and to transfer coverage reports
+          - run: |
+                tar -cvf java-builddir.tar target
+                tar -cvf java-m2-selection.tar ~/.m2/repository/io/gdcc/dataverse-*
+          - uses: actions/upload-artifact@v3
+            with:
+                name: java-artifacts
+                path: |
+                    java-builddir.tar
+                    java-m2-selection.tar
+                retention-days: 3
+
+    integration-test:
+        runs-on: ubuntu-latest
+        needs: unittest
+        name: (${{ matrix.status}} / JDK ${{ matrix.jdk }}) Integration Tests
+        strategy:
+            fail-fast: false
+            matrix:
+                jdk: [ '17' ]
+                experimental: [ false ]
+                status: [ "Stable" ]
+                #
+                # JDK 17 builds disabled due to non-essential fails marking CI jobs as completely failed within
+                # Github Projects, PR lists etc. This was consensus on Slack #dv-tech. See issue #8094
+                # (This is a limitation of how Github is currently handling these things.)
+                #
+                #include:
+                #    - jdk: '17'
+                #      experimental: true
+                #      status: "Experimental"
+        continue-on-error: ${{ matrix.experimental }}
+        steps:
+            # Basic setup chores
+            - uses: actions/checkout@v3
+            - name: Set up JDK ${{ matrix.jdk }}
+              uses: actions/setup-java@v3
+              with:
+                  java-version: ${{ matrix.jdk }}
+                  distribution: temurin
+                  cache: maven
+
+            # Get the build output from the unit test job
+            - uses: actions/download-artifact@v3
+              with:
+                  name: java-artifacts
+            - run: |
+                tar -xvf java-builddir.tar
+                tar -xvf java-m2-selection.tar -C /
+
+            # Run integration tests (but not unit tests again)
+            - run: mvn -DskipUnitTests -Dtarget.java.version=${{ matrix.jdk }} verify
+
+            # Wrap up and send to coverage job
+            - run: tar -cvf java-reportdir.tar target/site
+            - uses: actions/upload-artifact@v3
+              with:
+                  name: java-reportdir
+                  path: java-reportdir.tar
+                  retention-days: 3
+
+    coverage-report:
+        runs-on: ubuntu-latest
+        needs: integration-test
+        name: Coverage Report Submission
+        steps:
+            # Basic setup chores
+          - uses: actions/checkout@v3
+          - uses: actions/setup-java@v3
+            with:
+                java-version: '17'
+                distribution: temurin
+                cache: maven
+
+          # Get the build output from the integration test job
+          - uses: actions/download-artifact@v3
+            with:
+                name: java-reportdir
+          - run: tar -xvf java-reportdir.tar
+
+          # Deposit Code Coverage
+          - name: Deposit Code Coverage
             env:
                 CI_NAME: github
                 COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }}
@@ -57,22 +154,14 @@ jobs:
                 -DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }}
                 jacoco:report coveralls:report
 
-          # We don't want to cache the WAR file, so delete it
-          - run: rm -rf ~/.m2/repository/edu/harvard/iq/dataverse
+          # NOTE: this may be extended with adding a report to the build output, leave a comment, send to Sonarcloud, ...
 
-          # Upload the built war file. For download, it will be wrapped in a ZIP by GitHub.
-          # See also https://github.com/actions/upload-artifact#zipped-artifact-downloads
-          - uses: actions/upload-artifact@v3
-            with:
-                name: dataverse-java${{ matrix.jdk }}.war
-                path: target/dataverse*.war
-                retention-days: 7
     push-app-img:
         name: Publish App Image
         permissions:
             contents: read
             packages: write
             pull-requests: write
-        needs: unittest
+        needs: integration-test
         uses: ./.github/workflows/container_app_push.yml
         secrets: inherit

.gitignore

@@ -34,6 +34,7 @@ oauth-credentials.md
 /src/main/webapp/oauth2/newAccount.html
 scripts/api/setup-all.sh*
 scripts/api/setup-all.*.log
+src/main/resources/edu/harvard/iq/dataverse/openapi/
 
 # ctags generated tag file
 tags
@@ -61,3 +62,4 @@ src/main/webapp/resources/images/dataverseproject.png.thumb140
 
 # Docker development volumes
 /docker-dev-volumes
+/.vs

CONTRIBUTING.md

@@ -56,12 +56,12 @@ If you are interested in working on the main Dataverse code, great! Before you s
 
 Please read http://guides.dataverse.org/en/latest/developers/version-control.html to understand how we use the "git flow" model of development and how we will encourage you to create a GitHub issue (if it doesn't exist already) to associate with your pull request. That page also includes tips on making a pull request.
 
-After making your pull request, your goal should be to help it advance through our kanban board at https://github.com/orgs/IQSS/projects/2 . If no one has moved your pull request to the code review column in a timely manner, please reach out. Note that once a pull request is created for an issue, we'll remove the issue from the board so that we only track one card (the pull request).
+After making your pull request, your goal should be to help it advance through our kanban board at https://github.com/orgs/IQSS/projects/34 . If no one has moved your pull request to the code review column in a timely manner, please reach out. Note that once a pull request is created for an issue, we'll remove the issue from the board so that we only track one card (the pull request).
 
 Thanks for your contribution!
 
 [dataverse-community Google Group]: https://groups.google.com/group/dataverse-community
 [Community Call]: https://dataverse.org/community-calls
 [dataverse-dev Google Group]: https://groups.google.com/group/dataverse-dev
 [community contributors]: https://docs.google.com/spreadsheets/d/1o9DD-MQ0WkrYaEFTD5rF_NtyL8aUISgURsAXSL7Budk/edit?usp=sharing
-[dev efforts]: https://github.com/orgs/IQSS/projects/2#column-5298405
+[dev efforts]: https://github.com/orgs/IQSS/projects/34/views/6

README.md

@@ -1,21 +1,21 @@
 Dataverse®  
 ===============
 
-Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Data Science and Products team](http://www.iq.harvard.edu/people/people/data-science-products) at the [Institute for Quantitative Social Science](http://iq.harvard.edu/) and the [Dataverse community][]).
+Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Dataverse team](https://dataverse.org/about) at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][]).
 
-[dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our development [roadmap][], and more.
+[dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our [project board][], our development [roadmap][], and more.
 
 We maintain a demo site at [demo.dataverse.org][] which you are welcome to use for testing and evaluating Dataverse.
 
-To install Dataverse, please see our [Installation Guide][] which will prompt you to download our [latest release][].
+To install Dataverse, please see our [Installation Guide][] which will prompt you to download our [latest release][]. Docker users should consult the [Container Guide][].
 
 To discuss Dataverse with the community, please join our [mailing list][], participate in a [community call][], chat with us at [chat.dataverse.org][], or attend our annual [Dataverse Community Meeting][].
 
 We love contributors! Please see our [Contributing Guide][] for ways you can help.
 
 Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States.
 
-[![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg?raw=true "Dataverse Project")](http://dataverse.org)
+[![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg "Dataverse Project")](http://dataverse.org)
 
 [![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/)
 [![API Test Coverage](https://img.shields.io/jenkins/coverage/jacoco?jobUrl=https%3A%2F%2Fjenkins.dataverse.org%2Fjob%2FIQSS-dataverse-develop&label=API%20Test%20Coverage)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/ws/target/coverage-it/index.html)
@@ -26,15 +26,17 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi
 [dataverse.org]: https://dataverse.org
 [demo.dataverse.org]: https://demo.dataverse.org
 [Dataverse community]: https://dataverse.org/developers
-[Installation Guide]: http://guides.dataverse.org/en/latest/installation/index.html
+[Installation Guide]: https://guides.dataverse.org/en/latest/installation/index.html
 [latest release]: https://github.com/IQSS/dataverse/releases
+[Container Guide]: https://guides.dataverse.org/en/latest/container/index.html
 [features]: https://dataverse.org/software-features
+[project board]: https://github.com/orgs/IQSS/projects/34
 [roadmap]: https://www.iq.harvard.edu/roadmap-dataverse-project
 [integrations]: https://dataverse.org/integrations
-[REST APIs]: http://guides.dataverse.org/en/latest/api/index.html
+[REST APIs]: https://guides.dataverse.org/en/latest/api/index.html
 [Contributing Guide]: CONTRIBUTING.md
 [mailing list]: https://groups.google.com/group/dataverse-community
 [community call]: https://dataverse.org/community-calls
-[chat.dataverse.org]: http://chat.dataverse.org
+[chat.dataverse.org]: https://chat.dataverse.org
 [Dataverse Community Meeting]: https://dataverse.org/events
 [open source]: LICENSE.md

conf/keycloak/docker-compose.yml

@@ -3,13 +3,15 @@ version: "3.9"
 services:
 
   keycloak:
-    image: 'jboss/keycloak:16.1.1'
+    image: 'quay.io/keycloak/keycloak:21.0'
+    command:
+      - "start-dev"
+      - "--import-realm"
     environment:
-      - KEYCLOAK_USER=kcadmin
-      - KEYCLOAK_PASSWORD=kcpassword
-      - KEYCLOAK_IMPORT=/tmp/oidc-realm.json
+      - KEYCLOAK_ADMIN=kcadmin
+      - KEYCLOAK_ADMIN_PASSWORD=kcpassword
       - KEYCLOAK_LOGLEVEL=DEBUG
     ports:
       - "8090:8080"
     volumes:
-      - './oidc-realm.json:/tmp/oidc-realm.json'
+      - './test-realm.json:/opt/keycloak/data/import/test-realm.json'