Merge pull request IQSS#7422 from poikilotherm/7418-datasourcedefinition

7418 datasourcedefinition

doc/release-notes/7418-datasourcedefinition.md

@@ -0,0 +1,88 @@
+## Release Highlights
+
+### Easier Configuration of Database Connections
+
+Dataverse now being able to use up-to-date Java technologies, transforms
+the way how to configure the connection to your PostgreSQL database.
+
+In the past, the configuration of the connection has been quite static
+and not very easy to update. This has been an issue especially for cloud
+and container usage.
+
+Using MicroProfile Config API (#7000, #7418), you can much more easily specify configuration
+details. For an overview of supported options, please see the
+[installation guide](https://guides.dataverse.org/en/5.3/installation/config.html#jvm-options).
+
+Note that some settings have been moved from domain.xml to code such as min and max pool size.
+
+## Notes for Dataverse Installation Administrators
+
+### New JVM Options
+
+- dataverse.db.name
+- dataverse.db.user
+- dataverse.db.password
+- dataverse.db.host
+- dataverse.db.port
+
+<!-- ## Update to Payara Platform 5.2020.6 -->
+<!-- ... -->
+
+<!-- PLACEHOLDER REPLACEMENT TEXT FOR PAYARA UPGRADE NOTE #7417 -->
+🚨 THIS VERSION OF DATAVERSE **REQUIRES** UPGRADING TO PAYARA 5.2020.6. 🚨
+
+<!-- ... -->
+
+## Upgrading from earlier releases
+
+ℹ️ You need to update the Payara Application Server before continuing here. See above.
+
+1. Undeploy the previous version.
+```
+<payara install path>/asadmin list-applications
+<payara install path>/asadmin undeploy dataverse-<version>
+```
+
+(where `<payara install path>` is where Payara 5 is installed, for example: `/usr/local/payara5`)
+
+2. Update your database connection before updating.
+
+Please configure your connection details, replacing all the `${DB_...}`.
+(If you are using a PostgreSQL server on `localhost:5432`, you can omit `dataverse.db.host` and `dataverse.db.port`.)
+
+```
+<payara install path>/asadmin create-system-properties "dataverse.db.user=${DB_USER}"
+<payara install path>/asadmin create-system-properties "dataverse.db.host=${DB_HOST}"
+<payara install path>/asadmin create-system-properties "dataverse.db.port=${DB_PORT}"
+<payara install path>/asadmin create-system-properties "dataverse.db.name=${DB_NAME}"
+echo "AS_ADMIN_ALIASPASSWORD=${DB_PASS}" > /tmp/password.txt
+<payara install path>/asadmin create-password-alias --passwordfile /tmp/password.txt dataverse.db.password
+rm /tmp/password.txt
+```
+
+<!-- PLACE HOLDER FOR EJB TIMER DATABASE RESET NOTE #5345 -->
+
+Now you are safe to delete the old password alias and DB pool:
+```
+<payara install path>/asadmin delete-jdbc-connection-pool --cascade=true dvnDbPool
+<payara install path>/asadmin delete-password-alias db_password_alias
+```
+
+3. Stop payara and remove the generated directory, start.
+```
+service payara stop
+# remove the generated directory:
+rm -rf <payara install path>/payara/domains/domain1/generated
+service payara start
+```
+
+3. Deploy this version.
+```
+<payara install path>/bin/asadmin deploy dataverse-5.3.war
+```
+
+4. Restart Payara
+```
+service payara stop
+service payara start
+```

doc/sphinx-guides/source/installation/config.rst

@@ -1032,6 +1032,57 @@ dataverse.auth.password-reset-timeout-in-minutes
 
 Users have 60 minutes to change their passwords by default. You can adjust this value here.
 
+dataverse.db.name
++++++++++++++++++
+
+The PostgreSQL database name to use for Dataverse.
+
+Defaults to ``dataverse`` (but the installer sets it to ``dvndb``).
+
+Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_DB_NAME``.
+
+dataverse.db.user
++++++++++++++++++
+
+The PostgreSQL user name to connect with.
+
+Defaults to ``dataverse`` (but the installer sets it to ``dvnapp``).
+
+Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_DB_USER``.
+
+dataverse.db.password
++++++++++++++++++++++
+
+The PostgreSQL users password to connect with.
+
+Preferrably use a JVM alias, as passwords in environment variables aren't safe.
+
+.. code-block:: shell
+
+  echo "AS_ADMIN_ALIASPASSWORD=change-me-super-secret" > /tmp/password.txt
+  asadmin create-password-alias --passwordfile /tmp/password.txt dataverse.db.password
+  rm /tmp/password.txt
+
+Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_DB_PASSWORD``.
+
+dataverse.db.host
++++++++++++++++++
+
+The PostgreSQL server to connect to.
+
+Defaults to ``localhost``.
+
+Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_DB_HOST``.
+
+dataverse.db.port
++++++++++++++++++
+
+The PostgreSQL server port to connect to.
+
+Defaults to ``5432``, the default PostgreSQL port.
+
+Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_DB_PORT``.
+
 dataverse.rserve.host
 +++++++++++++++++++++
 
@@ -1168,7 +1219,7 @@ This JVM setting is also part of **handles** configuration. The Handle.Net insta
 .. _dataverse.handlenet.index:
 
 dataverse.handlenet.index
-+++++++++++++++++++++++++++++++++
++++++++++++++++++++++++++
 If you want to use different index than the default 300
 
 .. _dataverse.timerServer:

pom.xml

@@ -685,6 +685,7 @@
                     <include>**/*.xml</include>
                     <include>**/firstNames/*.*</include>
                     <include>**/*.xsl</include>
+                    <include>**/*.properties</include>
                 </includes>
             </resource>
         </resources>

scripts/installer/as-setup.sh

@@ -73,7 +73,7 @@ function preliminary_setup()
   ./asadmin $ASADMIN_OPTS create-jvm-options "-XX\:+DisableExplicitGC"
 
   # alias passwords
-  for alias in "rserve_password_alias ${RSERVE_PASS}" "doi_password_alias ${DOI_PASSWORD}" "db_password_alias ${DB_PASS}"
+  for alias in "rserve_password_alias ${RSERVE_PASS}" "doi_password_alias ${DOI_PASSWORD}" "dataverse.db.password ${DB_PASS}"
   do
       set -- $alias
       echo "AS_ADMIN_ALIASPASSWORD=$2" > /tmp/$1.txt
@@ -150,17 +150,11 @@ function final_setup(){
         ./asadmin $ASADMIN_OPTS delete-jvm-options -Xmx512m
         ./asadmin $ASADMIN_OPTS create-jvm-options "-Xmx${MEM_HEAP_SIZE}m"
 
-
-        ./asadmin $ASADMIN_OPTS create-jdbc-connection-pool --restype javax.sql.DataSource \
-                                        --datasourceclassname org.postgresql.ds.PGPoolingDataSource \
-                                        --property create=true:User=$DB_USER:PortNumber=$DB_PORT:databaseName=$DB_NAME:ServerName=$DB_HOST \
-                                        dvnDbPool
-
-       ./asadmin $ASADMIN_OPTS set resources.jdbc-connection-pool.dvnDbPool.property.password='${ALIAS=db_password_alias}'
-
-        ###
-        # Create data sources
-        ./asadmin $ASADMIN_OPTS create-jdbc-resource --connectionpoolid dvnDbPool jdbc/VDCNetDS
+         # Set up the database connection properties
+        ./asadmin $ASADMIN_OPTS create-system-properties "dataverse.db.user=${DB_USER}"
+        ./asadmin $ASADMIN_OPTS create-system-properties "dataverse.db.host=${DB_HOST}"
+        ./asadmin $ASADMIN_OPTS create-system-properties "dataverse.db.port=${DB_PORT}"
+        ./asadmin $ASADMIN_OPTS create-system-properties "dataverse.db.name=${DB_NAME}"
 
         ./asadmin $ASADMIN_OPTS create-jvm-options "\-Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl"
 

src/main/java/edu/harvard/iq/dataverse/flyway/StartupFlywayMigrator.java

@@ -15,7 +15,7 @@
 @TransactionManagement(value = TransactionManagementType.BEAN)
 public class StartupFlywayMigrator {
 
-    @Resource(lookup = "jdbc/VDCNetDS")
+    @Resource(lookup = "java:app/jdbc/dataverse")
     private DataSource dataSource;
 
     @PostConstruct

src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java

@@ -0,0 +1,54 @@
+package edu.harvard.iq.dataverse.util;
+
+import javax.annotation.Resource;
+import javax.annotation.sql.DataSourceDefinition;
+import javax.enterprise.inject.Produces;
+import javax.inject.Singleton;
+import javax.sql.DataSource;
+
+// Find docs here: https://javaee.github.io/javaee-spec/javadocs/javax/annotation/sql/DataSourceDefinition.html
+@Singleton
+@DataSourceDefinition(
+        name = "java:app/jdbc/dataverse",
+        // The app server (Payara) deploys a managed pool for this data source for us.
+        // We don't need to deal with this on our own.
+        //
+        // HINT: PGSimpleDataSource would work too, but as we use a connection pool, go with a javax.sql.ConnectionPoolDataSource
+        // HINT: PGXADataSource is unnecessary (no distributed transactions used) and breaks ingest.
+        className = "org.postgresql.ds.PGConnectionPoolDataSource",
+        user = "${MPCONFIG=dataverse.db.user}",
+        password = "${MPCONFIG=dataverse.db.password}",
+        url = "jdbc:postgresql://${MPCONFIG=dataverse.db.host}:${MPCONFIG=dataverse.db.port}/${MPCONFIG=dataverse.db.name}",
+        // If we ever need to change these pool settings, we need to remove this class and create the resource
+        // from web.xml. We can use MicroProfile Config in there for these values, impossible to do in the annotation.
+        //
+        // See also https://blog.payara.fish/an-intro-to-connection-pools-in-payara-server-5
+        // Payara DataSourceDefinitionDeployer default value = 8
+        minPoolSize = 10,
+        // HINT: Payara DataSourceDefinitionDeployer default value = 32
+        // HINT: Harvard Dataverse is fine for a while with 64
+        maxPoolSize = 100,
+        // "The number of seconds that a physical connection should remain unused in the pool before the connection is closed for a connection pool. "
+        // Payara DataSourceDefinitionDeployer default value = 300 (seconds)
+        maxIdleTime = 300)
+// It's possible to add additional properties like this...
+//
+//properties = {
+//    "fish.payara.log-jdbc-calls=true"
+//})
+//
+// ... but at this time we don't think we need any. The full list
+// of properties can be found at https://docs.payara.fish/community/docs/5.2020.6/documentation/payara-server/jdbc/advanced-connection-pool-properties.html#full-list-of-properties
+//
+// All these properties cannot be configured via MPCONFIG as Payara doesn't support this (yet). To be enhanced.
+// See also https://github.com/payara/Payara/issues/5024
+public class DataSourceProducer {
+
+    @Resource(lookup = "java:app/jdbc/dataverse")
+    DataSource ds;
+
+    @Produces
+    public DataSource getDatasource() {
+        return ds;
+    }
+}

src/main/resources/META-INF/microprofile-config.properties

@@ -0,0 +1,4 @@
+dataverse.db.host=localhost
+dataverse.db.port=5432
+dataverse.db.user=dataverse
+dataverse.db.name=dataverse

src/main/resources/META-INF/persistence.xml

@@ -3,7 +3,7 @@
     <persistence-unit name="VDCNet-ejbPU" transaction-type="JTA">
         <!-- provider>oracle.toplink.essentials.ejb.cmp3.EntityManagerFactoryProvider</provider-->
         <provider>org.eclipse.persistence.jpa.PersistenceProvider</provider>
-        <jta-data-source>jdbc/VDCNetDS</jta-data-source>
+        <jta-data-source>java:app/jdbc/dataverse</jta-data-source>
         <properties>
             <!--property name="toplink.logging.level" value="FINE"/-->
 	    <!-- disabling weaving, as an experiment: - L.A. -->
@@ -20,15 +20,4 @@
             <!--<property name="eclipselink.logging.parameters" value="true"/>-->
         </properties>
     </persistence-unit>
-    <!-- the persistence unit below is no longer necessary; needs to be removed 
-	 from gnrs/TestNRS and from here: - L.A. 
-    <persistence-unit name="VDCNet-test" transaction-type="RESOURCE_LOCAL">
-        <provider>oracle.toplink.essentials.ejb.cmp3.EntityManagerFactoryProvider</provider>
-        <properties>
-            <property name="toplink.jdbc.url" value="jdbc:postgresql://localhost:5432/VDCNet-dev"/>
-            <property name="toplink.jdbc.user" value="vdcApp"/>
-            <property name="toplink.jdbc.driver" value="org.postgresql.jdbc3.Jdbc3PoolingDataSource"/>
-            <property name="toplink.jdbc.password" value="secret"/>
-        </properties>
-    </persistence-unit -->
 </persistence>