From c4fa6bf3d461738bca5cd290cfe85395b1d88eb3 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 22 Jan 2025 16:12:18 -0800 Subject: [PATCH 1/4] First pass at a reprodicible env for running under FIPS mode This commit adds a dockerfile intended to serve as a portable environment for exploring running logstash in a FIPS compliant manner. The iteration at this commit focuses on running *only* the tests with the BC provider and trust stores. The same dockerfile has some commented out and unused config for doing manual exploration with LS. Eventually this will be split out into separate concerns. --- Dockerfile | 62 ++++++++++++++++++++ java.policy | 15 +++++ java.security | 117 +++++++++++++++++++++++++++++++++++++ logstash-core/build.gradle | 4 ++ test-init.gradle | 45 ++++++++++++++ 5 files changed, 243 insertions(+) create mode 100644 Dockerfile create mode 100644 java.policy create mode 100644 java.security create mode 100644 test-init.gradle diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..a3f11817dcc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,62 @@ +# Start from the FIPS-compliant base image +FROM docker.elastic.co/wolfi/chainguard-base-fips:latest + +# Install OpenJDK 21 +RUN apk add --no-cache \ + openjdk-21 \ + bash + +# Create directory for security configuration +RUN mkdir -p /etc/java/security +RUN mkdir -p /root/.gradle + +# Copy configuration files +COPY java.security /etc/java/security/ +COPY java.policy /etc/java/security/ +COPY test-init.gradle /tmp/test-init.gradle + +# Set environment variables +ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk +ENV PATH="${JAVA_HOME}/bin:${PATH}" + +# Create working directory +WORKDIR /logstash + +# Copy the local Logstash source +COPY . . + +# Initial build using JKS truststore +# ENV JAVA_OPTS="-Djavax.net.ssl.trustStore=$JAVA_HOME/lib/security/cacerts -Djavax.net.ssl.trustStoreType=JKS -Djavax.net.ssl.trustStorePassword=changeit" +RUN ./gradlew clean bootstrap assemble installDefaultGems --no-daemon + +# Convert to BCFKS trust store +RUN keytool -importkeystore \ + -srckeystore $JAVA_HOME/lib/security/cacerts \ + -destkeystore /etc/java/security/cacerts.bcfks \ + -srcstoretype jks \ + -deststoretype bcfks \ + -providerpath /root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bc-fips/2.0.0/ee9ac432cf08f9a9ebee35d7cf8a45f94959a7ab/bc-fips-2.0.0.jar \ + -provider org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider \ + -deststorepass changeit \ + -srcstorepass changeit \ + -noprompt && \ + cp /etc/java/security/cacerts.bcfks /etc/java/security/keystore.bcfks + +# ENV JAVA_SECURITY_PROPERTIES=/etc/java/security/java.security +# ENV JAVA_OPTS="\ +# -Djava.security.debug=ssl,provider \ +# -Djava.security.properties=${JAVA_SECURITY_PROPERTIES} \ +# -Djavax.net.ssl.keyStore=/etc/java/security/keystore.bcfks \ +# -Djavax.net.ssl.keyStoreType=BCFKS \ +# -Djavax.net.ssl.keyStoreProvider=BCFIPS \ +# -Djavax.net.ssl.keyStorePassword=changeit \ +# -Djavax.net.ssl.trustStore=/etc/java/security/cacerts.bcfks \ +# -Djavax.net.ssl.trustStoreType=BCFKS \ +# -Djavax.net.ssl.trustStoreProvider=BCFIPS \ +# -Djavax.net.ssl.trustStorePassword=changeit \ +# -Dssl.KeyManagerFactory.algorithm=PKIX \ +# -Dssl.TrustManagerFactory.algorithm=PKIX \ +# -Dorg.bouncycastle.fips.approved_only=true" + +# Run tests with BCFKS truststore +CMD ["./gradlew", "--init-script", "/tmp/test-init.gradle", "--info", "--stacktrace", "test"] \ No newline at end of file diff --git a/java.policy b/java.policy new file mode 100644 index 00000000000..68f3cb7c49d --- /dev/null +++ b/java.policy @@ -0,0 +1,15 @@ +grant { + // Your existing permissions + permission java.lang.PropertyPermission "java.runtime.name", "read"; + permission java.lang.RuntimePermission "accessClassInPackage.sun.security.internal.spec"; + permission java.lang.RuntimePermission "getProtectionDomain"; + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission org.bouncycastle.crypto.CryptoServicesPermission "tlsAlgorithmsEnabled"; + permission org.bouncycastle.crypto.CryptoServicesPermission "exportKeys"; + + // Add provider permissions + permission java.security.SecurityPermission "putProviderProperty.BCFIPS"; + permission java.security.SecurityPermission "insertProvider.BCFIPS"; + permission java.security.SecurityPermission "putProviderProperty.BCJSSE"; + permission java.security.SecurityPermission "insertProvider.BCJSSE"; +}; \ No newline at end of file diff --git a/java.security b/java.security new file mode 100644 index 00000000000..bd093581666 --- /dev/null +++ b/java.security @@ -0,0 +1,117 @@ +security.provider.1=org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider +security.provider.2=org.bouncycastle.jsse.provider.BouncyCastleJsseProvider fips:BCFIPS +security.provider.3=SUN + +securerandom.source=file:/dev/random +securerandom.strongAlgorithms=NativePRNGBlocking:SUN,DRBG:SUN +securerandom.drbg.config= + +login.configuration.provider=sun.security.provider.ConfigFile + +policy.provider=sun.security.provider.PolicyFile +policy.url.1=file:/etc/java/security/java.policy +policy.expandProperties=true +policy.allowSystemProperty=true +policy.ignoreIdentityScope=false + +keystore.type=bcfks +keystore.type.compat=true + +package.access=sun.misc.,\ + sun.reflect. +package.definition=sun.misc.,\ + sun.reflect. + +security.overridePropertiesFile=true + +ssl.KeyManagerFactory.algorithm=PKIX +ssl.TrustManagerFactory.algorithm=PKIX + +networkaddress.cache.negative.ttl=10 + +krb5.kdc.bad.policy = tryLast + +sun.security.krb5.disableReferrals=false +sun.security.krb5.maxReferrals=5 + +jdk.disabled.namedCurves = secp112r1, secp112r2, secp128r1, secp128r2, \ + secp160k1, secp160r1, secp160r2, secp192k1, secp192r1, secp224k1, \ + secp224r1, secp256k1, sect113r1, sect113r2, sect131r1, sect131r2, \ + sect163k1, sect163r1, sect163r2, sect193r1, sect193r2, sect233k1, \ + sect233r1, sect239k1, sect283k1, sect283r1, sect409k1, sect409r1, \ + sect571k1, sect571r1, X9.62 c2tnb191v1, X9.62 c2tnb191v2, \ + X9.62 c2tnb191v3, X9.62 c2tnb239v1, X9.62 c2tnb239v2, X9.62 c2tnb239v3, \ + X9.62 c2tnb359v1, X9.62 c2tnb431r1, X9.62 prime192v2, X9.62 prime192v3, \ + X9.62 prime239v1, X9.62 prime239v2, X9.62 prime239v3, brainpoolP256r1, \ + brainpoolP320r1, brainpoolP384r1, brainpoolP512r1 + +jdk.certpath.disabledAlgorithms=MD2, MD5, \ + RSA keySize < 1024, DSA keySize < 1024, EC keySize < 224, \ + SHA1, \ + secp112r1, secp112r2, secp128r1, secp128r2, \ + secp160k1, secp160r1, secp160r2, secp192k1, secp192r1, secp224k1, \ + secp224r1, secp256k1, sect113r1, sect113r2, sect131r1, sect131r2, \ + sect163k1, sect163r1, sect163r2, sect193r1, sect193r2, sect233k1, \ + sect233r1, sect239k1, sect283k1, sect283r1, sect409k1, sect409r1, \ + sect571k1, sect571r1, \ + brainpoolP256r1, brainpoolP320r1, brainpoolP384r1, brainpoolP512r1 + +jdk.security.legacyAlgorithms=SHA1, \ + RSA keySize < 2048, DSA keySize < 2048 + +jdk.jar.disabledAlgorithms=MD2, MD5, RSA keySize < 1024, \ + DSA keySize < 1024, SHA1, \ + secp112r1, secp112r2, secp128r1, secp128r2, \ + secp160k1, secp160r1, secp160r2, secp192k1, secp192r1, secp224k1, \ + secp224r1, secp256k1, sect113r1, sect113r2, sect131r1, sect131r2, \ + sect163k1, sect163r1, sect163r2, sect193r1, sect193r2, sect233k1, \ + sect233r1, sect239k1, sect283k1, sect283r1, sect409k1, sect409r1, \ + sect571k1, sect571r1, X9.62 c2tnb191v1, X9.62 c2tnb191v2, \ + X9.62 c2tnb191v3, X9.62 c2tnb239v1, X9.62 c2tnb239v2, X9.62 c2tnb239v3, \ + X9.62 c2tnb359v1, X9.62 c2tnb431r1, X9.62 prime192v2, X9.62 prime192v3, \ + X9.62 prime239v1, X9.62 prime239v2, X9.62 prime239v3, brainpoolP256r1, \ + brainpoolP320r1, brainpoolP384r1, brainpoolP512r1 + +jdk.tls.disabledAlgorithms=MD5, SSLv3, TLSv1, TLSv1.1, RC4, DES, MD5withRSA, \ + DH keySize < 1024, EC keySize < 224, 3DES_EDE_CBC, anon, NULL, \ + secp112r1, secp112r2, secp128r1, secp128r2, \ + secp160k1, secp160r1, secp160r2, secp192k1, secp192r1, secp224k1, \ + secp224r1, secp256k1, sect113r1, sect113r2, sect131r1, sect131r2, \ + sect163k1, sect163r1, sect163r2, sect193r1, sect193r2, sect233k1, \ + sect233r1, sect239k1, sect283k1, sect283r1, sect409k1, sect409r1, \ + sect571k1, sect571r1, brainpoolP256r1, \ + brainpoolP320r1, brainpoolP384r1, brainpoolP512r1 +jdk.tls.legacyAlgorithms= \ + K_NULL, C_NULL, M_NULL, \ + DH_anon, ECDH_anon, \ + RC4_128, RC4_40, DES_CBC, DES40_CBC, \ + 3DES_EDE_CBC +jdk.tls.keyLimits=AES/GCM/NoPadding KeyUpdate 2^37, \ + ChaCha20-Poly1305 KeyUpdate 2^37 + +crypto.policy=unlimited + +jdk.xml.dsig.secureValidationPolicy=\ + disallowAlg http://www.w3.org/TR/1999/REC-xslt-19991116,\ + disallowAlg http://www.w3.org/2001/04/xmldsig-more#rsa-md5,\ + disallowAlg http://www.w3.org/2001/04/xmldsig-more#hmac-md5,\ + disallowAlg http://www.w3.org/2001/04/xmldsig-more#md5,\ + maxTransforms 5,\ + maxReferences 30,\ + disallowReferenceUriSchemes file http https,\ + minKeySize RSA 1024,\ + minKeySize DSA 1024,\ + minKeySize EC 224,\ + noDuplicateIds,\ + noRetrievalMethodLoops + +jceks.key.serialFilter = java.base/java.lang.Enum;java.base/java.security.KeyRep;\ + java.base/java.security.KeyRep$Type;java.base/javax.crypto.spec.SecretKeySpec;!* + +jdk.sasl.disabledMechanisms=CRAM-MD5, DIGEST-MD5 +jdk.security.caDistrustPolicies=SYMANTEC_TLS +jdk.io.permissionsUseCanonicalPath=false + +jdk.tls.alpnCharset=ISO_8859_1 + +org.bouncycastle.fips.approved_only=true diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index 831f0fefde5..2bb66d739eb 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -235,6 +235,10 @@ dependencies { runtimeOnly 'commons-logging:commons-logging:1.3.1' // also handle libraries relying on log4j 1.x to redirect their logs runtimeOnly "org.apache.logging.log4j:log4j-1.2-api:${log4jVersion}" + runtimeOnly("org.bouncycastle:bc-fips:2.0.0") + runtimeOnly("org.bouncycastle:bcpkix-fips:2.0.7") + runtimeOnly("org.bouncycastle:bctls-fips:2.0.19") + runtimeOnly("org.bouncycastle:bcutil-fips:2.0.3") implementation('org.reflections:reflections:0.10.2') { exclude group: 'com.google.guava', module: 'guava' } diff --git a/test-init.gradle b/test-init.gradle new file mode 100644 index 00000000000..8b7a34ab056 --- /dev/null +++ b/test-init.gradle @@ -0,0 +1,45 @@ +initscript { + repositories { + mavenCentral() + } + dependencies { + classpath files( + '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bc-fips/2.0.0/ee9ac432cf08f9a9ebee35d7cf8a45f94959a7ab/bc-fips-2.0.0.jar', + '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bcpkix-fips/2.0.7/1eea0f325315ca6295b0a6926ff862d8001cdf9/bcpkix-fips-2.0.7.jar', + '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bctls-fips/2.0.19/9cc33650ede63bc1a8281ed5c8e1da314d50bc76/bctls-fips-2.0.19.jar', + '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bcutil-fips/2.0.3/a1857cd639295b10cc90e6d31ecbc523cdafcc19/bcutil-fips-2.0.3.jar' + ) + } +} + +gradle.taskGraph.whenReady { graph -> + if (graph.hasTask(":logstash-core:test") || + graph.hasTask(":logstash-core:rubyTests") || + graph.hasTask(":logstash-core:integrationTests")) { + + println "Registering BC FIPS provider for test execution..." + System.setProperty("javax.net.ssl.trustStore", "/etc/java/security/cacerts.bcfks") + System.setProperty("javax.net.ssl.trustStoreType", "BCFKS") + System.setProperty("javax.net.ssl.trustStoreProvider", "BCFIPS") + System.setProperty("javax.net.ssl.trustStorePassword", "changeit") + + def provider = Class.forName('org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider').getDeclaredConstructor().newInstance() + java.security.Security.insertProviderAt(provider, 1) + def configureTask = { task -> + task.systemProperty "java.security.properties", "/etc/java/security/java.security" + task.systemProperty "javax.net.ssl.keyStore", "/etc/java/security/keystore.bcfks" + task.systemProperty "javax.net.ssl.keyStoreType", "BCFKS" + task.systemProperty "javax.net.ssl.keyStoreProvider", "BCFIPS" + task.systemProperty "javax.net.ssl.keyStorePassword", "changeit" + task.systemProperty "javax.net.ssl.trustStore", "/etc/java/security/cacerts.bcfks" + task.systemProperty "javax.net.ssl.trustStoreType", "BCFKS" + task.systemProperty "javax.net.ssl.trustStoreProvider", "BCFIPS" + } + + graph.allTasks.each { task -> + if (task.path in [":logstash-core:test", ":logstash-core:rubyTests", ":logstash-core:integrationTests"]) { + configureTask(task) + } + } + } +} \ No newline at end of file From e30e728513f0a3093487adba013ddf32af30cef9 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Thu, 23 Jan 2025 16:13:39 -0800 Subject: [PATCH 2/4] Attempt to figure out timing The major issue i'm running in to now is where exactly to make the tests use the BC providers. Too early, and I run in to issues not being able to download dependencies but too late and the tests will fail due to not being able to load. This iteration looks at moving away from a script wrapping tasks and instead moving it to the build.gradle in the tasks directly. At this point its actually regressed in the java tests. My current thinking is that by not explicitly adding the cached locations to the classpath i've broken something. --- Dockerfile | 21 +++++++++++++++++---- logstash-core/build.gradle | 27 +++++++++++++++++++++++++++ test-init.gradle | 8 +++++--- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index a3f11817dcc..146f3e375df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,6 @@ COPY . . # ENV JAVA_OPTS="-Djavax.net.ssl.trustStore=$JAVA_HOME/lib/security/cacerts -Djavax.net.ssl.trustStoreType=JKS -Djavax.net.ssl.trustStorePassword=changeit" RUN ./gradlew clean bootstrap assemble installDefaultGems --no-daemon -# Convert to BCFKS trust store RUN keytool -importkeystore \ -srckeystore $JAVA_HOME/lib/security/cacerts \ -destkeystore /etc/java/security/cacerts.bcfks \ @@ -39,9 +38,22 @@ RUN keytool -importkeystore \ -provider org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider \ -deststorepass changeit \ -srcstorepass changeit \ - -noprompt && \ - cp /etc/java/security/cacerts.bcfks /etc/java/security/keystore.bcfks + -noprompt +RUN keytool -importkeystore \ + -srckeystore $JAVA_HOME/lib/security/cacerts \ + -destkeystore /etc/java/security/keystore.bcfks \ + -srcstoretype jks \ + -deststoretype bcfks \ + -providerpath /root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bc-fips/2.0.0/ee9ac432cf08f9a9ebee35d7cf8a45f94959a7ab/bc-fips-2.0.0.jar \ + -provider org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider \ + -deststorepass changeit \ + -srcstorepass changeit \ + -noprompt + +# Uncomment this out and build for interactive terminal sessions in a container +# Currently this dockerfile is just for running tests, but I have been also using it to +# do manual validation. Need to think more about how to separate concerns. # ENV JAVA_SECURITY_PROPERTIES=/etc/java/security/java.security # ENV JAVA_OPTS="\ # -Djava.security.debug=ssl,provider \ @@ -59,4 +71,5 @@ RUN keytool -importkeystore \ # -Dorg.bouncycastle.fips.approved_only=true" # Run tests with BCFKS truststore -CMD ["./gradlew", "--init-script", "/tmp/test-init.gradle", "--info", "--stacktrace", "test"] \ No newline at end of file +CMD ["./gradlew", "--info", "--stacktrace", "test"] + diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index 2bb66d739eb..5c10d86df63 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -110,7 +110,26 @@ configurations.archives { extendsFrom configurations.javadoc } +def setupBCProvider() { + println "Setting up BC provider" + System.setProperty("javax.net.ssl.trustStore", "/etc/java/security/cacerts.bcfks") + System.setProperty("javax.net.ssl.trustStoreType", "BCFKS") + System.setProperty("javax.net.ssl.trustStoreProvider", "BCFIPS") + System.setProperty("javax.net.ssl.trustStorePassword", "changeit") + + def provider = Class.forName('org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider').getDeclaredConstructor().newInstance() + java.security.Security.insertProviderAt(provider, 1) +} + tasks.register("javaTests", Test) { + doFirst { + configurations.testRuntimeClasspath.resolvedConfiguration.files.each { file -> + if (file.name.startsWith("bc-")) { + println "Found BC jar: ${file}" + } + } + setupBCProvider() + } dependsOn ':bootstrap' exclude '/org/logstash/RSpecTests.class' exclude '/org/logstash/config/ir/ConfigCompilerTest.class' @@ -141,6 +160,14 @@ jacocoTestReport { javaTests.finalizedBy(jacocoTestReport) tasks.register("rubyTests", Test) { + doFirst { + configurations.testRuntimeClasspath.resolvedConfiguration.files.each { file -> + if (file.name.startsWith("bc-")) { + println "Found BC jar: ${file}" + } + } + setupBCProvider() + } dependsOn compileTestJava inputs.files fileTree("${projectDir}/lib") inputs.files fileTree("${projectDir}/spec") diff --git a/test-init.gradle b/test-init.gradle index 8b7a34ab056..98304712769 100644 --- a/test-init.gradle +++ b/test-init.gradle @@ -1,3 +1,4 @@ +// test-init.gradle initscript { repositories { mavenCentral() @@ -5,7 +6,7 @@ initscript { dependencies { classpath files( '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bc-fips/2.0.0/ee9ac432cf08f9a9ebee35d7cf8a45f94959a7ab/bc-fips-2.0.0.jar', - '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bcpkix-fips/2.0.7/1eea0f325315ca6295b0a6926ff862d8001cdf9/bcpkix-fips-2.0.7.jar', + '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bcpkix-fips/2.0.7/1eea0f325315ca6295b0a6926ff862d8001cdf9/bcpkix-fips-2.0.7.jar', '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bctls-fips/2.0.19/9cc33650ede63bc1a8281ed5c8e1da314d50bc76/bctls-fips-2.0.19.jar', '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bcutil-fips/2.0.3/a1857cd639295b10cc90e6d31ecbc523cdafcc19/bcutil-fips-2.0.3.jar' ) @@ -18,13 +19,14 @@ gradle.taskGraph.whenReady { graph -> graph.hasTask(":logstash-core:integrationTests")) { println "Registering BC FIPS provider for test execution..." + def provider = Class.forName('org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider').getDeclaredConstructor().newInstance() + java.security.Security.insertProviderAt(provider, 1) + System.setProperty("javax.net.ssl.trustStore", "/etc/java/security/cacerts.bcfks") System.setProperty("javax.net.ssl.trustStoreType", "BCFKS") System.setProperty("javax.net.ssl.trustStoreProvider", "BCFIPS") System.setProperty("javax.net.ssl.trustStorePassword", "changeit") - def provider = Class.forName('org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider').getDeclaredConstructor().newInstance() - java.security.Security.insertProviderAt(provider, 1) def configureTask = { task -> task.systemProperty "java.security.properties", "/etc/java/security/java.security" task.systemProperty "javax.net.ssl.keyStore", "/etc/java/security/keystore.bcfks" From a7224b0d11d1536f8c5a809e616b9a3087385eee Mon Sep 17 00:00:00 2001 From: donoghuc Date: Fri, 24 Jan 2025 13:25:15 -0800 Subject: [PATCH 3/4] Pattern for testing using standarized JVM config In a previous iteration jvm options were attempted to be configured at the gradle level. This presented a challeng in rspec loading because webmock was not being loaded correctly with the fips providers. That initial approach was also problematic because it introduced a fork in configuration changes between configuring the JVM via the enviornment variables and files for running logstash in the container vs in the tests. This new apprach attempts to separate aout all of the test setup and building in gradle from a pure "just run the tests" task. The fundamental idea is that we dont want to use FIPS mode for downloading dependencies and building/preparing an environment we only want that configured at the very end. This apprach accomplishes that by teasing out the dependencies that trigger downloads etc in gradel from tasks that will only run the tests. The dockerfile order will call all the gradle tasks for setup, then configure FIPS mode and call the tests that should be run under fips mode. --- Dockerfile | 42 +++++++-------- build.gradle | 13 +++-- logstash-core/build.gradle | 101 +++++++++++++++++-------------------- rubyUtils.gradle | 1 + test-init.gradle | 47 ----------------- 5 files changed, 76 insertions(+), 128 deletions(-) delete mode 100644 test-init.gradle diff --git a/Dockerfile b/Dockerfile index 146f3e375df..3b7d9a9966e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,6 @@ RUN mkdir -p /root/.gradle # Copy configuration files COPY java.security /etc/java/security/ COPY java.policy /etc/java/security/ -COPY test-init.gradle /tmp/test-init.gradle # Set environment variables ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk @@ -27,14 +26,15 @@ COPY . . # Initial build using JKS truststore # ENV JAVA_OPTS="-Djavax.net.ssl.trustStore=$JAVA_HOME/lib/security/cacerts -Djavax.net.ssl.trustStoreType=JKS -Djavax.net.ssl.trustStorePassword=changeit" -RUN ./gradlew clean bootstrap assemble installDefaultGems --no-daemon +RUN ./gradlew clean bootstrap assemble installDefaultGems testSetup --no-daemon +# CMD ["sleep", "infinity"] RUN keytool -importkeystore \ -srckeystore $JAVA_HOME/lib/security/cacerts \ -destkeystore /etc/java/security/cacerts.bcfks \ -srcstoretype jks \ -deststoretype bcfks \ - -providerpath /root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bc-fips/2.0.0/ee9ac432cf08f9a9ebee35d7cf8a45f94959a7ab/bc-fips-2.0.0.jar \ + -providerpath /logstash/logstash-core/lib/jars/bc-fips-2.0.0.jar \ -provider org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider \ -deststorepass changeit \ -srcstorepass changeit \ @@ -45,31 +45,33 @@ RUN keytool -importkeystore \ -destkeystore /etc/java/security/keystore.bcfks \ -srcstoretype jks \ -deststoretype bcfks \ - -providerpath /root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bc-fips/2.0.0/ee9ac432cf08f9a9ebee35d7cf8a45f94959a7ab/bc-fips-2.0.0.jar \ + -providerpath /logstash/logstash-core/lib/jars/bc-fips-2.0.0.jar \ -provider org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider \ -deststorepass changeit \ -srcstorepass changeit \ -noprompt + # Uncomment this out and build for interactive terminal sessions in a container # Currently this dockerfile is just for running tests, but I have been also using it to # do manual validation. Need to think more about how to separate concerns. -# ENV JAVA_SECURITY_PROPERTIES=/etc/java/security/java.security -# ENV JAVA_OPTS="\ -# -Djava.security.debug=ssl,provider \ -# -Djava.security.properties=${JAVA_SECURITY_PROPERTIES} \ -# -Djavax.net.ssl.keyStore=/etc/java/security/keystore.bcfks \ -# -Djavax.net.ssl.keyStoreType=BCFKS \ -# -Djavax.net.ssl.keyStoreProvider=BCFIPS \ -# -Djavax.net.ssl.keyStorePassword=changeit \ -# -Djavax.net.ssl.trustStore=/etc/java/security/cacerts.bcfks \ -# -Djavax.net.ssl.trustStoreType=BCFKS \ -# -Djavax.net.ssl.trustStoreProvider=BCFIPS \ -# -Djavax.net.ssl.trustStorePassword=changeit \ -# -Dssl.KeyManagerFactory.algorithm=PKIX \ -# -Dssl.TrustManagerFactory.algorithm=PKIX \ -# -Dorg.bouncycastle.fips.approved_only=true" +ENV JAVA_SECURITY_PROPERTIES=/etc/java/security/java.security +ENV JAVA_OPTS="\ + -Djava.security.debug=ssl,provider \ + -Djava.security.properties=${JAVA_SECURITY_PROPERTIES} \ + -Djavax.net.ssl.keyStore=/etc/java/security/keystore.bcfks \ + -Djavax.net.ssl.keyStoreType=BCFKS \ + -Djavax.net.ssl.keyStoreProvider=BCFIPS \ + -Djavax.net.ssl.keyStorePassword=changeit \ + -Djavax.net.ssl.trustStore=/etc/java/security/cacerts.bcfks \ + -Djavax.net.ssl.trustStoreType=BCFKS \ + -Djavax.net.ssl.trustStoreProvider=BCFIPS \ + -Djavax.net.ssl.trustStorePassword=changeit \ + -Dssl.KeyManagerFactory.algorithm=PKIX \ + -Dssl.TrustManagerFactory.algorithm=PKIX \ + -Dorg.bouncycastle.fips.approved_only=true" +ENV LS_JAVA_OPTS="${JAVA_OPTS}" # Run tests with BCFKS truststore -CMD ["./gradlew", "--info", "--stacktrace", "test"] +CMD ["./gradlew","-PskipJRubySetup=true", "--info", "--stacktrace", "javaTestsOnly", "rubyTestsOnly"] diff --git a/build.gradle b/build.gradle index 3f865ab7686..1410899332b 100644 --- a/build.gradle +++ b/build.gradle @@ -295,15 +295,18 @@ clean { delete "${projectDir}/logstash-core/src/main/resources/org/logstash/plugins/plugin_aliases.yml" } -def assemblyDeps = [downloadAndInstallJRuby, assemble] + subprojects.collect { - it.tasks.findByName("assemble") +def getAssemblyDeps() { + return project.hasProperty('skipJRubySetup') ? [] : [downloadAndInstallJRuby, assemble] + subprojects.collect { + it.tasks.findByName("assemble") + } } tasks.register("bootstrap") { - dependsOn assemblyDeps + onlyIf { !project.hasProperty('skipJRubySetup') } + dependsOn getAssemblyDeps() doLast { - setupJruby(projectDir, buildDir) - } + setupJruby(projectDir, buildDir) + } } diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index 5c10d86df63..713c057dd28 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -27,27 +27,12 @@ buildscript { } } -plugins { - id "jacoco" - id "org.sonarqube" version "4.3.0.3225" -} -apply plugin: 'jacoco' -apply plugin: "org.sonarqube" repositories { mavenCentral() } -sonarqube { - properties { - property 'sonar.coverage.jacoco.xmlReportPaths', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml" - } -} - -jacoco { - toolVersion = "0.8.9" -} import org.yaml.snakeyaml.Yaml @@ -110,26 +95,7 @@ configurations.archives { extendsFrom configurations.javadoc } -def setupBCProvider() { - println "Setting up BC provider" - System.setProperty("javax.net.ssl.trustStore", "/etc/java/security/cacerts.bcfks") - System.setProperty("javax.net.ssl.trustStoreType", "BCFKS") - System.setProperty("javax.net.ssl.trustStoreProvider", "BCFIPS") - System.setProperty("javax.net.ssl.trustStorePassword", "changeit") - - def provider = Class.forName('org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider').getDeclaredConstructor().newInstance() - java.security.Security.insertProviderAt(provider, 1) -} - tasks.register("javaTests", Test) { - doFirst { - configurations.testRuntimeClasspath.resolvedConfiguration.files.each { file -> - if (file.name.startsWith("bc-")) { - println "Found BC jar: ${file}" - } - } - setupBCProvider() - } dependsOn ':bootstrap' exclude '/org/logstash/RSpecTests.class' exclude '/org/logstash/config/ir/ConfigCompilerTest.class' @@ -143,31 +109,10 @@ tasks.register("javaTests", Test) { exclude '/org/logstash/plugins/factory/PluginFactoryExtTest.class' exclude '/org/logstash/execution/ObservedExecutionTest.class' - jacoco { - enabled = true - destinationFile = layout.buildDirectory.file('jacoco/test.exec').get().asFile - classDumpDir = layout.buildDirectory.dir('jacoco/classpathdumps').get().asFile - } } -jacocoTestReport { - reports { - xml.required = true - html.required = true - } -} - -javaTests.finalizedBy(jacocoTestReport) tasks.register("rubyTests", Test) { - doFirst { - configurations.testRuntimeClasspath.resolvedConfiguration.files.each { file -> - if (file.name.startsWith("bc-")) { - println "Found BC jar: ${file}" - } - } - setupBCProvider() - } dependsOn compileTestJava inputs.files fileTree("${projectDir}/lib") inputs.files fileTree("${projectDir}/spec") @@ -223,6 +168,50 @@ task generateVersionInfoResources(type: DefaultTask) { resourceFile.text = "logstash-core: ${logstashCoreVersion}" } } +// Test execution tasks without dependencies +tasks.register("javaTestsOnly", Test) { + setDependsOn([]) + exclude '/org/logstash/RSpecTests.class' + exclude '/org/logstash/config/ir/ConfigCompilerTest.class' + exclude '/org/logstash/config/ir/CompiledPipelineTest.class' + exclude '/org/logstash/config/ir/EventConditionTest.class' + exclude '/org/logstash/config/ir/PipelineConfigTest.class' + exclude '/org/logstash/config/ir/compiler/OutputDelegatorTest.class' + exclude '/org/logstash/config/ir/compiler/JavaCodecDelegatorTest.class' + exclude '/org/logstash/plugins/NamespacedMetricImplTest.class' + exclude '/org/logstash/plugins/CounterMetricImplTest.class' + exclude '/org/logstash/plugins/factory/PluginFactoryExtTest.class' + exclude '/org/logstash/execution/ObservedExecutionTest.class' +} + +tasks.register("rubyTestsOnly", Test) { + setDependsOn([]) + inputs.files fileTree("${projectDir}/lib") + inputs.files fileTree("${projectDir}/spec") + systemProperty 'logstash.root.dir', projectDir.parent + + include '/org/logstash/RSpecTests.class' + include '/org/logstash/config/ir/ConfigCompilerTest.class' + include '/org/logstash/config/ir/CompiledPipelineTest.class' + include '/org/logstash/config/ir/EventConditionTest.class' + include '/org/logstash/config/ir/PipelineConfigTest.class' + include '/org/logstash/config/ir/compiler/OutputDelegatorTest.class' + include '/org/logstash/config/ir/compiler/JavaCodecDelegatorTest.class' + include '/org/logstash/plugins/NamespacedMetricImplTest.class' + include '/org/logstash/plugins/CounterMetricImplTest.class' + include '/org/logstash/plugins/factory/PluginFactoryExtTest.class' + include '/org/logstash/execution/ObservedExecutionTest.class' +} + +// Setup task combining all prerequisites +tasks.register("testSetup") { + dependsOn ':bootstrap' + dependsOn 'classes' + dependsOn 'testClasses' + dependsOn 'compileTestJava' + dependsOn 'copyRuntimeLibs' + dependsOn 'generateVersionInfoResources' +} sourceSets { main { output.dir(generateVersionInfoResources.outputs.files) } } @@ -294,4 +283,4 @@ dependencies { api group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.14' api group: 'commons-codec', name: 'commons-codec', version: '1.17.0' api group: 'org.apache.httpcomponents', name: 'httpcore', version: '4.4.16' -} +} \ No newline at end of file diff --git a/rubyUtils.gradle b/rubyUtils.gradle index 94d020543c2..ab2281d377a 100644 --- a/rubyUtils.gradle +++ b/rubyUtils.gradle @@ -228,6 +228,7 @@ def customJRubyVersion = customJRubyDir == "" ? "" : Files.readAllLines(Paths.ge def customJRubyTar = customJRubyDir == "" ? "" : (customJRubyDir + "/maven/jruby-dist/target/jruby-dist-${customJRubyVersion}-bin.tar.gz") tasks.register("downloadJRuby", Download) { + onlyIf { !project.hasProperty('skipJRubySetup') } description "Download JRuby artifact from this specific URL: ${jRubyURL}" src jRubyURL onlyIfNewer true diff --git a/test-init.gradle b/test-init.gradle deleted file mode 100644 index 98304712769..00000000000 --- a/test-init.gradle +++ /dev/null @@ -1,47 +0,0 @@ -// test-init.gradle -initscript { - repositories { - mavenCentral() - } - dependencies { - classpath files( - '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bc-fips/2.0.0/ee9ac432cf08f9a9ebee35d7cf8a45f94959a7ab/bc-fips-2.0.0.jar', - '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bcpkix-fips/2.0.7/1eea0f325315ca6295b0a6926ff862d8001cdf9/bcpkix-fips-2.0.7.jar', - '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bctls-fips/2.0.19/9cc33650ede63bc1a8281ed5c8e1da314d50bc76/bctls-fips-2.0.19.jar', - '/root/.gradle/caches/modules-2/files-2.1/org.bouncycastle/bcutil-fips/2.0.3/a1857cd639295b10cc90e6d31ecbc523cdafcc19/bcutil-fips-2.0.3.jar' - ) - } -} - -gradle.taskGraph.whenReady { graph -> - if (graph.hasTask(":logstash-core:test") || - graph.hasTask(":logstash-core:rubyTests") || - graph.hasTask(":logstash-core:integrationTests")) { - - println "Registering BC FIPS provider for test execution..." - def provider = Class.forName('org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider').getDeclaredConstructor().newInstance() - java.security.Security.insertProviderAt(provider, 1) - - System.setProperty("javax.net.ssl.trustStore", "/etc/java/security/cacerts.bcfks") - System.setProperty("javax.net.ssl.trustStoreType", "BCFKS") - System.setProperty("javax.net.ssl.trustStoreProvider", "BCFIPS") - System.setProperty("javax.net.ssl.trustStorePassword", "changeit") - - def configureTask = { task -> - task.systemProperty "java.security.properties", "/etc/java/security/java.security" - task.systemProperty "javax.net.ssl.keyStore", "/etc/java/security/keystore.bcfks" - task.systemProperty "javax.net.ssl.keyStoreType", "BCFKS" - task.systemProperty "javax.net.ssl.keyStoreProvider", "BCFIPS" - task.systemProperty "javax.net.ssl.keyStorePassword", "changeit" - task.systemProperty "javax.net.ssl.trustStore", "/etc/java/security/cacerts.bcfks" - task.systemProperty "javax.net.ssl.trustStoreType", "BCFKS" - task.systemProperty "javax.net.ssl.trustStoreProvider", "BCFIPS" - } - - graph.allTasks.each { task -> - if (task.path in [":logstash-core:test", ":logstash-core:rubyTests", ":logstash-core:integrationTests"]) { - configureTask(task) - } - } - } -} \ No newline at end of file From d5302be30c0bc8612265fcebbedcf779fcad4e0a Mon Sep 17 00:00:00 2001 From: donoghuc Date: Mon, 27 Jan 2025 16:15:12 -0800 Subject: [PATCH 4/4] Ensure gradle subtasks are configured to use bc Previously while the gradle process itself was configured with bc providers it was not actually using them for test execution due to precedence issues. This commit explicitly ensures they are configured. Will need to look at a better shared/long term solution for this. --- build.gradle | 2 +- logstash-core/build.gradle | 31 ++++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 1410899332b..02e406abcc4 100644 --- a/build.gradle +++ b/build.gradle @@ -302,7 +302,7 @@ def getAssemblyDeps() { } tasks.register("bootstrap") { - onlyIf { !project.hasProperty('skipJRubySetup') } + // onlyIf { !project.hasProperty('skipJRubySetup') } dependsOn getAssemblyDeps() doLast { setupJruby(projectDir, buildDir) diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index 713c057dd28..49e47670607 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -182,6 +182,20 @@ tasks.register("javaTestsOnly", Test) { exclude '/org/logstash/plugins/CounterMetricImplTest.class' exclude '/org/logstash/plugins/factory/PluginFactoryExtTest.class' exclude '/org/logstash/execution/ObservedExecutionTest.class' + + // SSL and security settings + systemProperty "java.security.properties", System.getenv("JAVA_SECURITY_PROPERTIES") + systemProperty "javax.net.ssl.keyStore", "/etc/java/security/keystore.bcfks" + systemProperty "javax.net.ssl.keyStoreType", "BCFKS" + systemProperty "javax.net.ssl.keyStoreProvider", "BCFIPS" + systemProperty "javax.net.ssl.keyStorePassword", "changeit" + systemProperty "javax.net.ssl.trustStore", "/etc/java/security/cacerts.bcfks" + systemProperty "javax.net.ssl.trustStoreType", "BCFKS" + systemProperty "javax.net.ssl.trustStoreProvider", "BCFIPS" + systemProperty "javax.net.ssl.trustStorePassword", "changeit" + systemProperty "ssl.KeyManagerFactory.algorithm", "PKIX" + systemProperty "ssl.TrustManagerFactory.algorithm", "PKIX" + systemProperty "org.bouncycastle.fips.approved_only", "true" } tasks.register("rubyTestsOnly", Test) { @@ -201,8 +215,21 @@ tasks.register("rubyTestsOnly", Test) { include '/org/logstash/plugins/CounterMetricImplTest.class' include '/org/logstash/plugins/factory/PluginFactoryExtTest.class' include '/org/logstash/execution/ObservedExecutionTest.class' -} + // SSL and security settings + systemProperty "java.security.properties", System.getenv("JAVA_SECURITY_PROPERTIES") + systemProperty "javax.net.ssl.keyStore", "/etc/java/security/keystore.bcfks" + systemProperty "javax.net.ssl.keyStoreType", "BCFKS" + systemProperty "javax.net.ssl.keyStoreProvider", "BCFIPS" + systemProperty "javax.net.ssl.keyStorePassword", "changeit" + systemProperty "javax.net.ssl.trustStore", "/etc/java/security/cacerts.bcfks" + systemProperty "javax.net.ssl.trustStoreType", "BCFKS" + systemProperty "javax.net.ssl.trustStoreProvider", "BCFIPS" + systemProperty "javax.net.ssl.trustStorePassword", "changeit" + systemProperty "ssl.KeyManagerFactory.algorithm", "PKIX" + systemProperty "ssl.TrustManagerFactory.algorithm", "PKIX" + systemProperty "org.bouncycastle.fips.approved_only", "true" +} // Setup task combining all prerequisites tasks.register("testSetup") { dependsOn ':bootstrap' @@ -211,6 +238,8 @@ tasks.register("testSetup") { dependsOn 'compileTestJava' dependsOn 'copyRuntimeLibs' dependsOn 'generateVersionInfoResources' + dependsOn ':copyPluginTestAlias' + dependsOn ':installDevelopmentGems' } sourceSets { main { output.dir(generateVersionInfoResources.outputs.files) }