Merge pull request #307 from NOAA-OWP/github_296

GitHub 296 and 300, changes to specify certs and passwords using environment variables and adding monitoring endpoints

compose-entry.template.yml

@@ -84,7 +84,7 @@ services:
          - /mnt/wres_share/logs/tasker/:/mnt/wres_share/logs/tasker/
         environment:
          # Make sure to pass through WRES_TASKER_SERVER_P12 to tasker at runtime
-         - JAVA_OPTS=-Dwres.monitorPassword=${WRES_MONITOR_PASSWORD} -Dwres.adminToken=${WRES_ADMIN_TOKEN} -Dwres.broker=broker -Dwres.redisHost=persister -Dwres.trustStore=${WRES_TRUST_STORE} -Dwres.taskerPathToServerP12=${WRES_TASKER_SERVER_P12} -Dcom.redhat.fips=false -Djava.io.tmpdir=/mnt/wres_share/input_data -Dwres.dataDirectDiskThreshold=90 -Dwres.numberOfWorkers=5 -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/tasker -XX:OnOutOfMemoryError='mv /mnt/wres_share/heap_dumps/tasker/java_pid%p.hprof /mnt/wres_share/heap_dumps/tasker/java_pid%p_$$CON_HOSTNAME.hprof; chmod 775 /mnt/wres_share/heap_dumps/tasker/java_pid%p_$$CON_HOSTNAME.hprof'
+         - JAVA_OPTS=-Dwres.monitorPassword=${WRES_MONITOR_PASSWORD} -Dwres.adminToken=${WRES_ADMIN_TOKEN} -Dwres.broker=broker -Dwres.redisHost=persister -Dwres.trustStore=${WRES_TRUST_STORE} -Dwres.trustStorePassword=${WRES_TRUST_STORE_PASSWORD} -Dwres.taskerPathToServerP12=${WRES_TASKER_SERVER_P12} -Dwres.taskerPathToServerP12Password=${WRES_TASKER_SERVER_P12_PASSWORD} -Dwres.taskerPathToClientP12Bundle=${WRES_TASKER_CLIENT_P12} -Dwres.taskerPathToClientP12Password=${WRES_TASKER_CLIENT_P12_PASSWORD} -Dcom.redhat.fips=false -Djava.io.tmpdir=/mnt/wres_share/input_data -Dwres.dataDirectDiskThreshold=90 -Dwres.numberOfWorkers=5 -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/tasker -XX:OnOutOfMemoryError='mv /mnt/wres_share/heap_dumps/tasker/java_pid%p.hprof /mnt/wres_share/heap_dumps/tasker/java_pid%p_$$CON_HOSTNAME.hprof; chmod 775 /mnt/wres_share/heap_dumps/tasker/java_pid%p_$$CON_HOSTNAME.hprof'
          - LANG=C.UTF-8
         # Tasker JVM should have 340m max heap specified at launch
         # The total limit includes stack space which depends on Thread count
@@ -99,6 +99,7 @@ services:
         ports:
          - "5671:5671"
          - "15671:15671"
+         - "15691:15691"
         image: "${DOCKER_REGISTRY}/wres/wres-broker:BROKER_IMAGE"
         restart: always
         volumes:
@@ -108,6 +109,15 @@ services:
          - /mnt/wres_share/rabbitmq/:/var/lib/rabbitmq/
         environment:
          - RABBITMQ_CONFIG_FILE=rabbitmq.conf
+         - WRES_RABBITMQ_SSL_OPTIONS_CACERTFILE=${WRES_RABBITMQ_SSL_OPTIONS_CACERTFILE}
+         - WRES_RABBITMQ_SSL_OPTIONS_CERTFILE=${WRES_RABBITMQ_SSL_OPTIONS_CERTFILE}
+         - WRES_RABBITMQ_SSL_OPTIONS_KEYFILE=${WRES_RABBITMQ_SSL_OPTIONS_KEYFILE}
+         - WRES_RABBITMQ_MANAGEMENT_SSL_CACERTFILE=${WRES_RABBITMQ_MANAGEMENT_SSL_CACERTFILE}
+         - WRES_RABBITMQ_MANAGEMENT_SSL_CERTFILE=${WRES_RABBITMQ_MANAGEMENT_SSL_CERTFILE}
+         - WRES_RABBITMQ_MANAGEMENT_SSL_KEYFILE=${WRES_RABBITMQ_MANAGEMENT_SSL_KEYFILE}
+         - WRES_RABBITMQ_PROMETHEUS_SSL_CACERTFILE=${WRES_RABBITMQ_PROMETHEUS_SSL_CACERTFILE}
+         - WRES_RABBITMQ_PROMETHEUS_SSL_CERTFILE=${WRES_RABBITMQ_PROMETHEUS_SSL_CERTFILE}
+         - WRES_RABBITMQ_PROMETHEUS_SSL_KEYFILE=${WRES_RABBITMQ_PROMETHEUS_SSL_KEYFILE}
         # rabbitmq.conf should have 360m specified as high watermark
         mem_limit: 720m
         cap_drop:
@@ -145,7 +155,7 @@ services:
          # Writing all log outputs:
          - /mnt/wres_share/logs/worker/:/mnt/wres_share/logs/worker/
         environment:
-         - JAVA_OPTS=-Dwres.broker=broker -Dcom.redhat.fips=false -Dwres.trustStore=${WRES_TRUST_STORE} -Djava.io.tmpdir=/mnt/wres_share/evaluations -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/worker-shim -XX:OnOutOfMemoryError='mv /mnt/wres_share/heap_dumps/worker-shim/java_pid%p.hprof /mnt/wres_share/heap_dumps/worker-shim/java_pid%p_$$CON_HOSTNAME.hprof; chmod 775 /mnt/wres_share/heap_dumps/worker-shim/java_pid%p_$$CON_HOSTNAME.hprof'
+         - JAVA_OPTS=-Dwres.broker=broker -Dcom.redhat.fips=false -Dwres.trustStore=${WRES_TRUST_STORE} -Dwres.trustStorePassword=${WRES_TRUST_STORE_PASSWORD} -Dwres.workerPathToClientP12Bundle=${WRES_WORKER_CLIENT_P12} -Dwres.workerPathToClientP12Password=${WRES_WORKER_CLIENT_P12_PASSWORD} -Djava.io.tmpdir=/mnt/wres_share/evaluations -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/worker-shim -XX:OnOutOfMemoryError='mv /mnt/wres_share/heap_dumps/worker-shim/java_pid%p.hprof /mnt/wres_share/heap_dumps/worker-shim/java_pid%p_$$CON_HOSTNAME.hprof; chmod 775 /mnt/wres_share/heap_dumps/worker-shim/java_pid%p_$$CON_HOSTNAME.hprof'
          - PGPASSFILE=/wres_secrets/.pgpass
          # Use caller-specified db hostname from env var WRES_DB_FQDN.
          # Do not auto-liquibase-migrate on each evaluation. This requires an
@@ -177,6 +187,10 @@ services:
          - /mnt/wres_keys:/wres_secrets:ro
         environment:
          - BROKER_WORK=/container_home
+         - BROKER_KEYSTORE_PATH=${EVENTSBROKER_KEYSTORE_PATH}
+         - BROKER_KEYSTORE_PASSWORD=${EVENTSBROKER_KEYSTORE_PASSWORD}
+         - BROKER_TRUSTSTORE_PATH=${EVENTSBROKER_TRUSTSTORE_PATH}
+         - BROKER_TRUSTSTORE_PASSWORD=${EVENTSBROKER_TRUSTSTORE_PASSWORD}
          - JAVA_ARGS=-XX:+PrintClassHistogram -XX:+UseG1GC -XX:+UseStringDeduplication -Xms2048m -Xmx2048m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/eventsbroker -Dcom.redhat.fips=false
          - LANG=C.UTF-8
         # Broker heap is 2048m in JAVA_ARGS

compose-workers.template.yml

@@ -72,7 +72,7 @@ services:
          # Writing all log outputs:
          - /mnt/wres_share/logs/worker/:/mnt/wres_share/logs/worker/
         environment:
-         - JAVA_OPTS=-Dwres.broker=${WRES_BROKER_HOST} -Dcom.redhat.fips=false -Dwres.trustStore=${WRES_TRUST_STORE} -Djava.io.tmpdir=/mnt/wres_share/evaluations -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/worker-shim -XX:OnOutOfMemoryError='mv /mnt/wres_share/heap_dumps/worker-shim/java_pid%p.hprof /mnt/wres_share/heap_dumps/worker-shim/java_pid%p_$$CON_HOSTNAME.hprof; chmod 775 /mnt/wres_share/heap_dumps/worker-shim/java_pid%p_$$CON_HOSTNAME.hprof'
+         - JAVA_OPTS=-Dwres.broker=${WRES_BROKER_HOST} -Dcom.redhat.fips=false -Dwres.trustStore=${WRES_TRUST_STORE} -Dwres.trustStorePassword=${WRES_TRUST_STORE_PASSWORD} -Dwres.workerPathToClientP12Bundle=${WRES_WORKER_CLIENT_P12} -Dwres.workerPathToClientP12Password=${WRES_WORKER_CLIENT_P12_PASSWORD} -Djava.io.tmpdir=/mnt/wres_share/evaluations -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/worker-shim -XX:OnOutOfMemoryError='mv /mnt/wres_share/heap_dumps/worker-shim/java_pid%p.hprof /mnt/wres_share/heap_dumps/worker-shim/java_pid%p_$$CON_HOSTNAME.hprof; chmod 775 /mnt/wres_share/heap_dumps/worker-shim/java_pid%p_$$CON_HOSTNAME.hprof'
          - PGPASSFILE=/wres_secrets/.pgpass
          # Use caller-specified db hostname from env var WRES_DB_FQDN.
          # Do not auto-liquibase-migrate on each evaluation. This requires an
@@ -104,6 +104,10 @@ services:
          - /mnt/wres_keys:/wres_secrets:ro
         environment:
          - BROKER_WORK=/container_home
+         - BROKER_KEYSTORE_PATH=${EVENTSBROKER_KEYSTORE_PATH}
+         - BROKER_KEYSTORE_PASSWORD=${EVENTSBROKER_KEYSTORE_PASSWORD}
+         - BROKER_TRUSTSTORE_PATH=${EVENTSBROKER_TRUSTSTORE_PATH}
+         - BROKER_TRUSTSTORE_PASSWORD=${EVENTSBROKER_TRUSTSTORE_PASSWORD}
          - JAVA_ARGS=-XX:+PrintClassHistogram -XX:+UseG1GC -XX:+UseStringDeduplication -Xms2048m -Xmx2048m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/eventsbroker -Dcom.redhat.fips=false
          - LANG=C.UTF-8
         # Broker heap is 2048m in JAVA_ARGS

wres-broker/nonsrc/rabbitmq.conf

@@ -7,9 +7,9 @@ ssl_options.ciphers.2  = TLS_AES_128_GCM_SHA256
 ssl_options.ciphers.3  = TLS_CHACHA20_POLY1305_SHA256
 ssl_options.ciphers.4  = TLS_AES_128_CCM_SHA256
 ssl_options.ciphers.5  = TLS_AES_128_CCM_8_SHA256
-ssl_options.cacertfile = /wres_secrets/ca_x509_cert.pem
-ssl_options.certfile = /wres_secrets/broker_server_x509_cert.pem
-ssl_options.keyfile = /wres_secrets/broker_server_private_rsa_key.pem
+ssl_options.cacertfile = $(WRES_RABBITMQ_SSL_OPTIONS_CACERTFILE)
+ssl_options.certfile = $(WRES_RABBITMQ_SSL_OPTIONS_CERTFILE)
+ssl_options.keyfile = $(WRES_RABBITMQ_SSL_OPTIONS_KEYFILE)
 ssl_options.verify = verify_peer
 ssl_options.fail_if_no_peer_cert = true
 ssl_options.honor_cipher_order = true
@@ -18,9 +18,9 @@ ssl_cert_login_from = common_name
 vm_memory_high_watermark.absolute = 360MB
 
 management.ssl.port = 15671
-management.ssl.cacertfile = /wres_secrets/ca_x509_cert.pem
-management.ssl.certfile = /wres_secrets/broker_server_x509_cert.pem
-management.ssl.keyfile = /wres_secrets/broker_server_private_rsa_key.pem
+management.ssl.cacertfile = $(WRES_RABBITMQ_MANAGEMENT_SSL_CACERTFILE)
+management.ssl.certfile = $(WRES_RABBITMQ_MANAGEMENT_SSL_CERTFILE) 
+management.ssl.keyfile = $(WRES_RABBITMQ_MANAGEMENT_SSL_KEYFILE)
 management.ssl.verify = verify_peer
 management.ssl.fail_if_no_peer_cert = true
 management.ssl.honor_cipher_order = true
@@ -38,3 +38,11 @@ load_definitions = /etc/rabbitmq/definitions.json
 
 #log.console.level = debug
 #ssl_options.log_level = debug
+
+# Monitoring
+prometheus.ssl.port       = 15691
+prometheus.ssl.cacertfile = $(WRES_RABBITMQ_PROMETHEUS_SSL_CACERTFILE)
+prometheus.ssl.certfile   = $(WRES_RABBITMQ_PROMETHEUS_SSL_CERTFILE) 
+prometheus.ssl.keyfile    = $(WRES_RABBITMQ_PROMETHEUS_SSL_KEYFILE)
+## To enforce TLS (disable the non-TLS port):
+# prometheus.tcp.listener = none

wres-eventsbroker/docker-entrypoint.sh

@@ -6,7 +6,9 @@ set -e
 BROKER_CONFIG_PATH=$BROKER_INSTANCE/etc/
 
 # Properties passed to the broker and then accessible as system properties in the xml configuration files
-ARTEMIS_CLUSTER_PROPS="-Dactivemq.remoting.amqp.port=${BROKER_AMQP_PORT} -Dactivemq.remoting.http.port=${BROKER_HTTP_PORT} -Dhawtio.disableProxy=true -Dhawtio.realm=activemq-cert -Dhawtio.role=wres-eventsbroker-admin -Dhawtio.offline=true -Dhawtio.sessionTimeout=86400 -Dhawtio.rolePrincipalClasses=org.apache.activemq.artemis.spi.core.security.jaas.RolePrincipal"
+ARTEMIS_CLUSTER_PROPS="-Dactivemq.remoting.amqp.port=${BROKER_AMQP_PORT} -Dactivemq.remoting.http.port=${BROKER_HTTP_PORT} -Dbroker.keystore.path=${BROKER_KEYSTORE_PATH} -Dbroker.keystore.password=${BROKER_KEYSTORE_PASSWORD} -Dbroker.truststore.path=${BROKER_TRUSTSTORE_PATH} -Dbroker.truststore.password=${BROKER_TRUSTSTORE_PASSWORD} -Dhawtio.disableProxy=true -Dhawtio.realm=activemq-cert -Dhawtio.role=wres-eventsbroker-admin -Dhawtio.offline=true -Dhawtio.sessionTimeout=86400 -Dhawtio.rolePrincipalClasses=org.apache.activemq.artemis.spi.core.security.jaas.RolePrincipal"
+
+echo "HANK HERR ####>> $ARTEMIS_CLUSTER_PROPS"
 
 # Set some JVM arguments if not already set
 if [[ -z $JAVA_ARGS ]]; then

wres-eventsbroker/nonsrc/bootstrap.xml

@@ -29,10 +29,11 @@
    <web path="web">
        <binding uri="https://0.0.0.0:${activemq.remoting.http.port}" 
                 clientAuth="true" 
-                keyStorePath="/wres_secrets/wres-eventsbroker_server_keystore.p12" 
-                keyStorePassword="wres-eventsbroker-passphrase" 
-                trustStorePath="/wres_secrets/wres-eventsbroker_server_truststore.p12" 
-                trustStorePassword="wres-eventsbroker-passphrase" includedTLSProtocols="TLSv1.2,TLSv1.3">
+                keyStorePath="${broker.keystore.path}" 
+                keyStorePassword="${broker.keystore.password}" 
+                trustStorePath="${broker.truststore.path}" 
+                trustStorePassword="${broker.truststore.password}" 
+                includedTLSProtocols="TLSv1.2,TLSv1.3">
            <app url="activemq-branding" war="activemq-branding.war"/>
            <app url="artemis-plugin" war="artemis-plugin.war"/>
            <app url="console" war="console.war"/>

wres-messages/src/main/java/wres/messages/BrokerHelper.java

@@ -50,17 +50,6 @@ public class BrokerHelper
 
     static final String TRUST_STORE_PROPERTY_NAME = "wres.trustStore";
 
-    /**
-     * The role.
-     */
-    public enum Role
-    {
-        /** Worker role. */
-        WORKER,
-        /** Tasker role. */
-        TASKER
-    }
-
     private BrokerHelper()
     {
         // Static helper class, no construction
@@ -74,7 +63,7 @@ private BrokerHelper()
 
     public static String getBrokerHost()
     {
-        String brokerFromDashD= System.getProperty( BROKER_HOST_PROPERTY_NAME );
+        String brokerFromDashD = System.getProperty( BROKER_HOST_PROPERTY_NAME );
 
         if ( brokerFromDashD != null )
         {
@@ -95,7 +84,7 @@ public static String getBrokerHost()
 
     public static String getBrokerVhost()
     {
-        String brokerVhostFromDashD= System.getProperty( BROKER_VHOST_PROPERTY_NAME );
+        String brokerVhostFromDashD = System.getProperty( BROKER_VHOST_PROPERTY_NAME );
 
         if ( brokerVhostFromDashD != null )
         {
@@ -166,7 +155,7 @@ public static TrustManager getDefaultTrustManager()
              && alternativeTrustStoreFile.canRead() )
         {
             try ( InputStream alternativeTrustStream =
-                          new FileInputStream( alternativeTrustStoreFile ) )
+                    new FileInputStream( alternativeTrustStoreFile ) )
             {
                 customTrustStore.load( alternativeTrustStream,
                                        "changeit".toCharArray() );
@@ -179,7 +168,8 @@ public static TrustManager getDefaultTrustManager()
             catch ( IOException | NoSuchAlgorithmException | CertificateException e )
             {
                 LOGGER.warn( "Could not use alternative Certificate Authority at '{}'",
-                             trustStore, e );
+                             trustStore,
+                             e );
                 // Continue and use the default trust store.
             }
         }
@@ -247,29 +237,40 @@ private static TrustManagerFactory getPKIXTrustManagerFactory()
         catch ( NoSuchAlgorithmException nsae )
         {
             throw new IllegalStateException( "WRES expected JRE to support algorithm '"
-                                             + algorithm + "'.", nsae );
+                                             + algorithm
+                                             + "'.",
+                                             nsae );
         }
     }
 
 
     /**
      * Get an SSLContext that is set up with a client certificate,
      * used to authenticate to the wres-broker.
-     * @param role the role of the module connecting to the broker
+     * @param pathToP12 The path to the p12 file to use. Must be non-null and not empty.
+     * @param passwordForP12 The password to use. If null or empty, no password isused.
      * @return SSLContext ready to go for connecting to the broker
      * @throws IllegalStateException when anything goes wrong setting up
-     * keystores, trust managers, factories, reading files, parsing certificate,
-     * decrypting contents, etc.
+     * keystores, trust managers, factories, reading files, parsing certificate, 
+     * decrypting contents, etc. 
      */
 
-    public static SSLContext getSSLContextWithClientCertificate( Role role )
+    public static SSLContext getSSLContextWithClientCertificate( String pathToP12, String passwordForP12 )
     {
-        String ourClientCertificateFilename = BrokerHelper.getSecretsDir() + "/"
-                                              + "wres-" + role.name()
-                                                             .toLowerCase()
-                                              + "_client_private_key_and_x509_cert.p12";
-        char[] keyPassphrase = ("wres-" + role.name().toLowerCase()
-                                + "-passphrase").toCharArray();
+        if (pathToP12 == null || pathToP12.isEmpty())
+        {
+            throw new IllegalArgumentException("Argument pathToP12 cannot be null or empty, but was.");
+        }
+        char[] keyPassphrase = new char[]{};
+        if ( passwordForP12 != null && !passwordForP12.isEmpty() )
+        {
+            keyPassphrase = passwordForP12.toCharArray();
+        }
+        else
+        {
+            LOGGER.warn("For file " + pathToP12 + " password provided is null or empty, so no password is assumed.");
+        }
+
         KeyStore keyStore;
 
         try
@@ -283,14 +284,14 @@ public static SSLContext getSSLContextWithClientCertificate( Role role )
         }
 
         try ( InputStream clientCertificateInputStream =
-                      new FileInputStream( ourClientCertificateFilename ) )
+                new FileInputStream( pathToP12 ) )
         {
             keyStore.load( clientCertificateInputStream, keyPassphrase );
         }
         catch ( IOException | NoSuchAlgorithmException | CertificateException e )
         {
             throw new IllegalStateException( "WRES expected to find a file '"
-                                             + ourClientCertificateFilename
+                                             + pathToP12
                                              + "' with PKCS#12 format, with"
                                              + " both a client certificate AND"
                                              + " the private key inside, used "
@@ -318,7 +319,8 @@ public static SSLContext getSSLContextWithClientCertificate( Role role )
         {
             throw new IllegalStateException( "WRES expected to be able to read "
                                              + "and decrypt the file '"
-                                             + ourClientCertificateFilename +
+                                             + pathToP12
+                                             +
                                              "'.",
                                              e );
         }
@@ -333,7 +335,8 @@ public static SSLContext getSSLContextWithClientCertificate( Role role )
         catch ( NoSuchAlgorithmException nsae )
         {
             throw new IllegalStateException( "WRES expected to be able to use protocol '"
-                                             + protocol + "'",
+                                             + protocol
+                                             + "'",
                                              nsae );
         }
 
@@ -353,3 +356,4 @@ public static SSLContext getSSLContextWithClientCertificate( Role role )
         return sslContext;
     }
 }
+

wres-messages/test/wres/messages/BrokerHelperTest.java

@@ -82,6 +82,6 @@ public void getSSLContextThrowsIllegalStateExceptionWhenNothingSpecified()
     {
         exception.expect( IllegalStateException.class );
         SSLContext sslContext = BrokerHelper.getSSLContextWithClientCertificate(
-                BrokerHelper.Role.TASKER );
+                "wres-tasker_client_private_key_and_x509_cert.p12", "wres-tasker-passphrase" );
     }
 }

wres-reading/src/wres/reading/ReaderUtilities.java

@@ -672,6 +672,7 @@ public static Pair<SSLContext, X509TrustManager> getSslContextTrustingDodSignerF
     {
         // Look for a system property first: #106160
         String pathToTrustFile = System.getProperty( "wres.wrdsCertificateFileToTrust" );
+        String passwordForInternalTrustStore = System.getProperty( "wres.wrdsInternalTrustStorePassword" );
         if ( Objects.nonNull( pathToTrustFile ) )
         {
             LOGGER.debug( "Discovered the system property wres.wrdsCertificateFileToTrust with value {}.",
@@ -681,7 +682,7 @@ public static Pair<SSLContext, X509TrustManager> getSslContextTrustingDodSignerF
             try ( InputStream trustStream = Files.newInputStream( path ) )
             {
                 SSLStuffThatTrustsOneCertificate sslGoo =
-                        new SSLStuffThatTrustsOneCertificate( trustStream );
+                        new SSLStuffThatTrustsOneCertificate( trustStream, passwordForInternalTrustStore );
                 return Pair.of( sslGoo.getSSLContext(), sslGoo.getTrustManager() );
             }
             catch ( IOException e )
@@ -725,7 +726,7 @@ public static Pair<SSLContext, X509TrustManager> getSslContextTrustingDodSignerF
                 return Pair.of( SSLContext.getDefault(), theTrustManager );
             }
             SSLStuffThatTrustsOneCertificate sslGoo =
-                    new SSLStuffThatTrustsOneCertificate( inputStream );
+                    new SSLStuffThatTrustsOneCertificate( inputStream, passwordForInternalTrustStore );
             return Pair.of( sslGoo.getSSLContext(), sslGoo.getTrustManager() );
         }
         catch ( IOException ioe )