Add support for testing against multiple FDB clusters

.gitignore

@@ -80,6 +80,7 @@ protogen/
 
 # Local FDB settings
 fdb-environment.properties
+fdb-environment.yaml
 
 # Docker local support
 run/

actions/setup-fdb/action.yml

@@ -31,12 +31,57 @@ runs:
     - name: Install FDB Server
       shell: bash
       run: sudo dpkg -i ~/.fdb-cache/${{ steps.fdb_filenames.outputs.client_deb }} ~/.fdb-cache/${{ steps.fdb_filenames.outputs.server_deb }}
-    - name: Fix FDB Network Addresses
+    - name: Stop default fdb
       shell: bash
-      run: sudo sed -i -e "s/public_address = auto:\$ID/public_address = 127.0.0.1:\$ID/g" -e "s/listen_address = public/listen_address = 0.0.0.0:\$ID/g" /etc/foundationdb/foundationdb.conf
-    - name: Start FDB Server
+      run: sudo service foundationdb stop
+
+    - name: Create cluster1 config
+      shell: bash
+      run: |
+        sudo cp /etc/foundationdb/foundationdb.conf /etc/foundationdb/foundationdb1.conf
+
+        sudo sed -i -e "s/\/etc\/foundationdb\/fdb.cluster/\/etc\/foundationdb\/fdb1.cluster/g" \
+        -e "s/\/var\/log\/foundationdb/\/var\/log\/foundationdb1/" \
+        -e "s/fdbserver.4500/fdbserver.4600/g" \
+        /etc/foundationdb/foundationdb1.conf
+
+        sudo bash -c "echo 'fdb1:$(mktemp -u XXXXXXXX)@127.0.0.1:4600' > /etc/foundationdb/fdb1.cluster"
+    - name: Create cluster2 config
       shell: bash
-      run: sudo /usr/lib/foundationdb/fdbmonitor /etc/foundationdb/foundationdb.conf --daemonize
-    - name: Switch FDB to SSD
+      run: |
+        sudo cp /etc/foundationdb/foundationdb.conf /etc/foundationdb/foundationdb2.conf
+
+        sudo sed -i -e "s/\/etc\/foundationdb\/fdb.cluster/\/etc\/foundationdb\/fdb2.cluster/g" \
+        -e "s/\/var\/log\/foundationdb/\/var\/log\/foundationdb2/" \
+        -e "s/fdbserver.4500/fdbserver.4700/g" \
+        /etc/foundationdb/foundationdb2.conf
+
+        sudo bash -c "echo 'fdb2:$(mktemp -u XXXXXXXX)@127.0.0.1:4700' > /etc/foundationdb/fdb2.cluster"
+
+    - name: create dirs & set permissions
       shell: bash
-      run: fdbcli --exec "configure single ssd storage_migration_type=aggressive; status"
+      run: |
+        sudo mkdir /var/log/foundationdb1 /var/log/foundationdb2
+        sudo chown foundationdb:foundationdb /etc/foundationdb/fdb1.cluster /etc/foundationdb/fdb2.cluster /var/log/foundationdb1 /var/log/foundationdb2
+        sudo chmod 664 /etc/foundationdb/fdb1.cluster /etc/foundationdb/fdb2.cluster
+        sudo chmod 700 /var/log/foundationdb1 /var/log/foundationdb2
+
+    - name: Start FDB Server 1
+      shell: bash
+      run: sudo /usr/lib/foundationdb/fdbmonitor --conffile /etc/foundationdb/foundationdb1.conf --lockfile /var/run/fdbmonitor1.pid --loggroup fdb1 --daemonize
+    - name: Start FDB Server 2
+      shell: bash
+      run: sudo /usr/lib/foundationdb/fdbmonitor --conffile /etc/foundationdb/foundationdb2.conf --lockfile /var/run/fdbmonitor2.pid --loggroup fdb2 --daemonize
+
+    - name: Switch FDB 1 to SSD
+      shell: bash
+      run: fdbcli -C /etc/foundationdb/fdb1.cluster --exec "configure new single ssd storage_migration_type=aggressive; status"
+    - name: Switch FDB 2 to SSD
+      shell: bash
+      run: fdbcli -C /etc/foundationdb/fdb2.cluster --exec "configure new single ssd storage_migration_type=aggressive; status"
+    - name: Create fdb-environment.yaml
+      shell: bash
+      run: |
+        echo "clusterFiles: " >> fdb-environment.yaml
+        echo "  - /etc/foundationdb/fdb1.cluster" >> fdb-environment.yaml
+        echo "  - /etc/foundationdb/fdb2.cluster" >> fdb-environment.yaml

build.gradle

@@ -18,6 +18,10 @@
  * limitations under the License.
  */
 
+import org.yaml.snakeyaml.Yaml
+
+import org.apache.tools.ant.taskdefs.condition.Os
+
 buildscript {
     repositories {
         if (Boolean.parseBoolean(mavenLocalEnabled)) {
@@ -29,6 +33,7 @@ buildscript {
 
     dependencies {
         classpath 'org.jboss.tattletale:tattletale:1.2.0.Beta2'
+        classpath libs.snakeyaml
     }
 }
 
@@ -341,6 +346,8 @@ if (!JavaVersion.current().isJava8Compatible()) {
     throw new Exception("Java 8 is required to build fdb-record-layer")
 }
 
+// fdb-environment.properties is the old way we configured the library path and cluster file, it does not scale well
+// to multiple cluster files, so is being replaced with a yaml file.
 def fdbEnvironmentFile = new File("${rootProject.projectDir}/fdb-environment.properties")
 if (fdbEnvironmentFile.exists()) {
     fdbEnvironmentFile.eachLine { line ->
@@ -356,3 +363,15 @@ if (!ext.fdbEnvironment.isEmpty()) {
         }
     }
 }
+
+def fdbEnvironmentYamlFile = new File("${rootProject.projectDir}/fdb-environment.yaml")
+if (fdbEnvironmentYamlFile.exists()) {
+    def libraryPath = new Yaml().loadAll(fdbEnvironmentYamlFile.newInputStream()).first().libraryPath
+    def libraryPathVariable = Os.isFamily(Os.FAMILY_MAC) ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH"
+    allprojects {
+        tasks.withType(Test) { task ->
+            task.environment([(libraryPathVariable): libraryPath])
+            task.environment(FDB_ENVIRONMENT_YAML: fdbEnvironmentYamlFile.absolutePath)
+        }
+    }
+}

docker-local/start.sh

@@ -95,6 +95,13 @@ FDB_CLUSTER_FILE=${FDB_CLUSTER_FILE}
 ${LIBRARY_PATH}=${RUNDIR}
 EOF
 
+cat >${ROOTDIR}/fdb-environment.yaml <<EOF
+# docker-local
+clusterFiles:
+  - ${FDB_CLUSTER_FILE}
+libraryPath: ${RUNDIR}
+EOF
+
 cat ${ROOTDIR}/fdb-environment.properties
 
 echo "Docker-based FDB cluster is now up."

docs/sphinx/source/Building.md

@@ -63,7 +63,24 @@ If you enable the local repo in whatever uses the Record Layer, the following wi
 ./gradlew publishToMavenLocal -PpublishBuild=true
 ```
 
-## Configuring tests for a non-standard FDB cluster file location
+## Configuring tests for a non-standard FDB cluster file location, or to run against multiple clusters
+
+If a file `fdb-environment.yaml` exists in the root of the working directory, it contains the configuration for the C API library
+and a list of cluster files. Most tests will chose randomly from the provided cluster files when testing.
+
+Here is an example file:
+``` yaml
+libraryPath: /Users/scott/fdb/bin/fdb-server-7.3.42-macos_arm64/lib
+clusterFiles:
+  - /Users/scott/fdb/data/fdb-one.cluster
+  - /Users/scott/fdb/data/fdb-two.cluster
+```
+
+
+
+### deprecated properties file
+
+[ this is being replaced by the yaml file described above, to better support multiple cluster files ]
 
 If a file `fdb-environment.properties` exists in the root of the working directory, it contains environment variables that specify where to find
 the local FDB. These settings will apply when running inside IntelliJ as well as from a command line and so are easier to manage than a shell script.

fdb-extensions/fdb-extensions.gradle

@@ -31,11 +31,13 @@ dependencies {
     compileOnly(libs.jsr305)
 
     testImplementation(libs.bundles.test.impl)
+    testImplementation(libs.snakeyaml)
     testRuntimeOnly(libs.bundles.test.runtime)
     testCompileOnly(libs.bundles.test.compileOnly)
     testAnnotationProcessor(libs.autoService)
 
     testFixturesImplementation(libs.bundles.test.impl)
+    testFixturesImplementation(libs.snakeyaml)
     testFixturesCompileOnly(libs.bundles.test.compileOnly)
     testFixturesImplementation(libs.slf4j.api)
     testFixturesAnnotationProcessor(libs.autoService)

fdb-extensions/src/testFixtures/java/com/apple/foundationdb/test/FDBTestEnvironment.java

@@ -0,0 +1,86 @@
+/*
+ * FDBTestEnvironment.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb.test;
+
+import org.assertj.core.api.Assumptions;
+import org.yaml.snakeyaml.Yaml;
+
+import javax.annotation.Nullable;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Utility class for accessing {@code fdb-environment.yaml}, so that tests can know about fdb clusters
+ * that are available.
+ */
+public final class FDBTestEnvironment {
+    private static final List<String> clusterFiles;
+
+    static {
+        final String fdbEnvironment = System.getenv("FDB_ENVIRONMENT_YAML");
+
+        if (fdbEnvironment != null && !fdbEnvironment.isEmpty()) {
+            clusterFiles = List.copyOf(parseFDBEnvironmentYaml(fdbEnvironment));
+        } else {
+            clusterFiles = Collections.singletonList(null); // List.of does not allow null
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static List<String> parseFDBEnvironmentYaml(final String fdbEnvironment) {
+        Yaml yaml = new Yaml();
+        try (FileInputStream yamlInput = new FileInputStream(fdbEnvironment)) {
+            Object fdbConfig = yaml.load(yamlInput);
+            return (List<String>)((Map<?, ?>)fdbConfig).get("clusterFiles");
+        } catch (IOException e) {
+            throw new IllegalStateException("Could not read fdb-environment.yaml", e);
+        } catch (ClassCastException e) {
+            throw new IllegalStateException("Could not parse fdb environment file " + fdbEnvironment, e);
+        }
+    }
+
+    @Nullable
+    public static String getClusterFile(int i) {
+        return clusterFiles.get(i);
+    }
+
+    public static List<String> allClusterFiles() {
+        return clusterFiles;
+    }
+
+    public static String randomClusterFile() {
+        return clusterFiles.get(ThreadLocalRandom.current().nextInt(clusterFiles.size()));
+    }
+
+    /**
+     * Marks the current test as skipped if there are not the desired number of clusters available to test against.
+     * @param desiredCount the number of clusters to test against
+     */
+    public static void assumeClusterCount(final int desiredCount) {
+        if (desiredCount > 1) {
+            Assumptions.assumeThat(clusterFiles.size()).as("Cluster file count").isGreaterThanOrEqualTo(desiredCount);
+        }
+    }
+}

fdb-extensions/src/testFixtures/java/com/apple/foundationdb/test/TestDatabaseExtension.java

@@ -95,7 +95,7 @@ public void beforeAll(final ExtensionContext extensionContext) {
     @Nonnull
     public Database getDatabase() {
         if (db == null) {
-            db = FDB.instance().open();
+            db = FDB.instance().open(FDBTestEnvironment.randomClusterFile());
         }
         return db;
     }

fdb-record-layer-core/fdb-record-layer-core.gradle

@@ -44,6 +44,7 @@ dependencies {
     testImplementation(libs.junit.platform)
     testImplementation(libs.protobuf.util)
     testCompileOnly(libs.spotbugs.annotations)
+    testImplementation(libs.snakeyaml)
 
     testAnnotationProcessor(libs.autoService)
 }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/BlockingInAsyncDetectionTest.java

@@ -24,6 +24,7 @@
 import com.apple.foundationdb.record.RecordCoreException;
 import com.apple.foundationdb.record.TestHelpers;
 import com.apple.foundationdb.record.test.FDBDatabaseExtension;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.test.Tags;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
@@ -57,7 +58,7 @@ void testBlockingInAsyncException() {
         // Make sure that we aren't holding on to previously created databases
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         assertEquals(BlockingInAsyncDetection.IGNORE_COMPLETE_EXCEPTION_BLOCKING, database.getBlockingInAsyncDetection());
         assertThrows(BlockingInAsyncException.class, () -> callAsyncBlocking(database));
     }
@@ -68,7 +69,7 @@ void testBlockingInAsyncWarning() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.IGNORE_COMPLETE_WARN_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_IN_ASYNC_CONTEXT_MESSAGE,
                 () -> {
                     callAsyncBlocking(database, true);
@@ -82,7 +83,7 @@ void testCompletedBlockingInAsyncWarning() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.WARN_COMPLETE_EXCEPTION_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_IN_ASYNC_CONTEXT_MESSAGE,
                 () -> database.asyncToSync(new FDBStoreTimer(), FDBStoreTimer.Waits.WAIT_ERROR_CHECK,
                         CompletableFuture.supplyAsync(() ->
@@ -95,7 +96,7 @@ void testBlockingCreatingAsyncDetection() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.WARN_COMPLETE_EXCEPTION_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_RETURNING_ASYNC_MESSAGE,
                 () -> returnAnAsync(database, MoreAsyncUtil.delayedFuture(200L, TimeUnit.MILLISECONDS, database.getScheduledExecutor())));
     }
@@ -106,7 +107,7 @@ void testCompletedBlockingCreatingAsyncDetection() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.WARN_COMPLETE_EXCEPTION_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertDidNotLog(FDBDatabase.class, FDBDatabase.BLOCKING_RETURNING_ASYNC_MESSAGE,
                 () -> returnAnAsync(database, CompletableFuture.completedFuture(10L)));
     }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBDatabaseTest.java

@@ -36,6 +36,7 @@
 import com.apple.foundationdb.record.test.TestKeySpace;
 import com.apple.foundationdb.record.test.TestKeySpacePathManagerExtension;
 import com.apple.foundationdb.subspace.Subspace;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.foundationdb.tuple.Tuple;
 import com.apple.test.BooleanSource;
 import com.apple.test.Tags;
@@ -54,6 +55,7 @@
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -66,6 +68,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -240,7 +243,7 @@ void testJoinNowOnNonCompletedFuture(BlockingInAsyncDetection behavior) {
         if (behavior.throwExceptionOnBlocking()) {
             assertThrows(BlockingInAsyncException.class, () -> database.joinNow(new CompletableFuture<>()));
         } else {
-            FDBDatabase database2 = factory.getDatabase();
+            FDBDatabase database2 = factory.getDatabase(database.getClusterFile());
             TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_FOR_FUTURE_MESSAGE, () -> {
                 long val = database2.joinNow(MoreAsyncUtil.delayedFuture(100, TimeUnit.MILLISECONDS, database2.getScheduledExecutor())
                         .thenApply(vignore -> 1066L));
@@ -494,4 +497,31 @@ void cannotChangeAPIVersionAfterInit() {
             assertEquals(initApiVersion, database.getAPIVersion());
         }
     }
+
+    @Test
+    void canAccessMultipleClusters() {
+        FDBTestEnvironment.assumeClusterCount(2);
+        final FDBDatabase database0 = dbExtension.getDatabase(0);
+        final FDBDatabase database1 = dbExtension.getDatabase(1);
+        final byte[] key = Tuple.from(UUID.randomUUID()).pack();
+        final byte[] value0 = Tuple.from("cluster0").pack();
+        final byte[] value1 = Tuple.from("cluster1").pack();
+        try (FDBRecordContext context0 = database0.openContext()) {
+            context0.ensureActive().set(key, value0);
+            context0.commit();
+        }
+        try (FDBRecordContext context1 = database1.openContext()) {
+            assertNull(context1.ensureActive().get(key).join());
+            context1.ensureActive().set(key, value1);
+            context1.commit();
+        }
+        try (FDBRecordContext context0 = database0.openContext()) {
+            assertArrayEquals(value0, context0.ensureActive().get(key).join());
+            context0.commit();
+        }
+        try (FDBRecordContext context1 = database1.openContext()) {
+            assertArrayEquals(value1, context1.ensureActive().get(key).join());
+            context1.commit();
+        }
+    }
 }