From bbcc7edd3b86c3bc7e0cfe79f6da335f4e65d80b Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Wed, 20 Mar 2024 16:15:03 -0600
Subject: [PATCH 01/28] Added support for supplying branch name in place of
 image for adhoc runs

---
 .github/scripts/build-docker-image-remote.sh  | 83 +++++++++++++++++++
 .github/scripts/setup-test-server-remote.sh   |  8 +-
 .github/workflows/adhoc-remote-benchmarks.yml |  2 +-
 .github/workflows/remote-benchmarks.yml       | 20 ++---
 4 files changed, 90 insertions(+), 23 deletions(-)
 create mode 100755 .github/scripts/build-docker-image-remote.sh

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
new file mode 100755
index 00000000..b20e8bf4
--- /dev/null
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# Build a local docker image on the remote side if needed
+# Ensure the docker image is running in the Deephaven directory
+
+HOST=`hostname`
+GIT_DIR=/root/git
+DEEPHAVEN_DIR=/root/deephaven
+DOCKER_IMG=$1
+BRANCH_DELIM="::"
+
+if [ ! -d "${DEEPHAVEN_DIR}" ]; then
+  echo "$0: Missing one or more Benchmark setup directories"
+  exit 1
+fi
+
+if [[ $# != 1 ]]; then
+  echo "$0: Missing docker image/branch argument"
+  exit 1
+fi
+
+title () { echo; echo $1; }
+
+title "- Setting up Remote Docker Image on ${HOST} -"
+
+if [[ ${DOCKER_IMG} != *"${BRANCH_DELIM}"* ]]; then
+  echo "DOCKER_IMG=ghcr.io/deephaven/server:${DOCKER_IMG}" > .env
+  docker compose pull
+  title "-- Starting Deephaven and Redpanda --"
+  docker compose up -d
+  exit 0
+fi
+
+readarray -d "${BRANCH_DELIM}" -t splitarr <<< "${DOCKER_IMG}"
+OWNER=${splitarr[0]}
+BRANCH_NAME=${splitarr[1]}
+
+title "-- Cloning deephaven-core --"
+cd ${GIT_DIR}
+rm -rf deephaven-core 
+git clone git@github.com:${OWNER}/deephaven-core.git
+cd deephaven-core
+git checkout ${BRANCH_NAME}
+
+title "-- Cloning deephaven-server-docker --"
+cd ${GIT_DIR}
+rm -rf deephaven-server-docker
+git clone git@github.com:deephaven/deephaven-server-docker.git
+cd deephaven-server-docker
+git checkout main
+
+title "-- Assembling Python Deephaven Core Server --"
+cd ${GIT_DIR}/deephaven-core
+OLD_JAVA_HOME="${JAVA_HOME}"
+export JAVA_HOME=/usr/lib/jvm/${JAVA}
+./gradlew outputVersion server-jetty-app:assemble py-server:assemble
+export DEEPHAVEN_VERSION=$(cat build/version)
+export JAVA_HOME="${OLD_JAVA_HOME}"
+
+title "-- Building Deephaven Docker Image --"
+cd ${GIT_DIR}/deephaven-server-docker
+cp ${GIT_DIR}/deephaven-core/server/jetty-app/build/distributions/server-jetty-*.tar contexts/server/
+cp ${GIT_DIR}/deephaven-core/server/jetty-app/build/distributions/server-jetty-*.tar contexts/server-slim/
+cp ${GIT_DIR}/deephaven-core/py/server/build/wheel/deephaven_core-*-py3-none-any.whl contexts/server/
+
+export DEEPHAVEN_SOURCES=custom
+export DEEPHAVEN_CORE_WHEEL=$(find . -type f -name "*.whl" | xargs -n 1 basename)
+export TAG=benchmark-local
+
+echo "DEEPHAVEN_VERSION: ${DEEPHAVEN_VERSION}"
+echo "DEEPHAVEN_CORE_WHEEL: ${DEEPHAVEN_CORE_WHEEL}"
+docker buildx bake -f server.hcl
+
+title "-- Starting Deephaven and Redpanda --"
+echo "DOCKER_IMG=deephaven/server:benchmark-local" > .env
+docker compose up -d
+
+
+
diff --git a/.github/scripts/setup-test-server-remote.sh b/.github/scripts/setup-test-server-remote.sh
index 62b005fe..f631b342 100755
--- a/.github/scripts/setup-test-server-remote.sh
+++ b/.github/scripts/setup-test-server-remote.sh
@@ -75,16 +75,10 @@ docker images -a -q | xargs --no-run-if-empty -n 1 docker rmi
 docker system prune --volumes --force
 rm -rf ${DEEPHAVEN_DIR}
 
-title "-- Installing Deephaven and Redpanda --"
+title "-- Staging Docker Resources --"
 mkdir -p ${DEEPHAVEN_DIR}
 cd ${DEEPHAVEN_DIR}
 cp ${GIT_DIR}/benchmark/.github/resources/${RUN_TYPE}-benchmark-docker-compose.yml docker-compose.yml
-echo "DOCKER_IMG=${DOCKER_IMG}" > .env
-docker compose pull
-
-title "-- Starting Deephaven and Redpanda --"
-docker compose up -d
-
 
 
 
diff --git a/.github/workflows/adhoc-remote-benchmarks.yml b/.github/workflows/adhoc-remote-benchmarks.yml
index 2e9dd6c7..bf9724fe 100644
--- a/.github/workflows/adhoc-remote-benchmarks.yml
+++ b/.github/workflows/adhoc-remote-benchmarks.yml
@@ -11,7 +11,7 @@ on:
   workflow_dispatch:
    inputs:
      docker_image:
-       description: 'Docker Image Name'
+       description: 'Docker Image Name or DH Core Branch'
        required: true
        default: 'edge'
        type: string
diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index 47c5ade4..10739b9c 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -53,21 +53,7 @@ jobs:
         java-version: '21'
         distribution: 'temurin'
         cache: maven
-
-    - name: Docker Pull Deephaven and Redpanda
-      run: |
-        cp .github/resources/${RUN_TYPE}-benchmark-docker-compose.yml docker-compose.yml
-        echo "DOCKER_IMG=${DOCKER_IMG}" > .env
-        docker compose pull
-
-    - name: Docker Up Deephaven and Redpanda
-      run: docker compose up -d
-
-    - name: Build with Maven
-      run: |
-        mvn -B verify --file pom.xml
-        rm -rf results
-    
+        
     - name: Setup Local and Remote Scripts
       run: |
         sudo chmod +x ${SD}/*
@@ -77,6 +63,10 @@ jobs:
       run: |
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} setup-test-server-remote ${REPO} ${BRANCH} ${RUN_TYPE} "${DOCKER_IMG}"
 
+    - name: Run Remote Docker Image Build
+      run: |
+        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-dokcer-image-remote ${DOCKER_IMG}
+
     - name: Run Remote Benchmark Artifact Build
       run: |
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-benchmark-artifact-remote

From a472a5d689bc0e01fc3d288ec51b2d5278cc5a68 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Wed, 20 Mar 2024 16:25:54 -0600
Subject: [PATCH 02/28] sigh. Mispelled script name

---
 .github/workflows/remote-benchmarks.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index 10739b9c..07215117 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -65,7 +65,7 @@ jobs:
 
     - name: Run Remote Docker Image Build
       run: |
-        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-dokcer-image-remote ${DOCKER_IMG}
+        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-docker-image-remote ${DOCKER_IMG}
 
     - name: Run Remote Benchmark Artifact Build
       run: |

From f7b9e66fbea78b9f082c2b7a341c6a2d527e36f1 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Wed, 20 Mar 2024 16:52:25 -0600
Subject: [PATCH 03/28] Add correct directories for docker compose pull

---
 .github/scripts/build-docker-image-remote.sh | 2 ++
 .github/workflows/remote-benchmarks.yml      | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
index b20e8bf4..b17094cb 100755
--- a/.github/scripts/build-docker-image-remote.sh
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -28,6 +28,7 @@ title () { echo; echo $1; }
 title "- Setting up Remote Docker Image on ${HOST} -"
 
 if [[ ${DOCKER_IMG} != *"${BRANCH_DELIM}"* ]]; then
+  cd ${DEEPHAVEN_DIR}
   echo "DOCKER_IMG=ghcr.io/deephaven/server:${DOCKER_IMG}" > .env
   docker compose pull
   title "-- Starting Deephaven and Redpanda --"
@@ -76,6 +77,7 @@ echo "DEEPHAVEN_CORE_WHEEL: ${DEEPHAVEN_CORE_WHEEL}"
 docker buildx bake -f server.hcl
 
 title "-- Starting Deephaven and Redpanda --"
+cd ${DEEPHAVEN_DIR}
 echo "DOCKER_IMG=deephaven/server:benchmark-local" > .env
 docker compose up -d
 
diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index 07215117..e658ada3 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -65,7 +65,7 @@ jobs:
 
     - name: Run Remote Docker Image Build
       run: |
-        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-docker-image-remote ${DOCKER_IMG}
+        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-docker-image-remote "${DOCKER_IMG}"
 
     - name: Run Remote Benchmark Artifact Build
       run: |

From 739174e351d2c074a3dd105e316776d9abfc3cab Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Wed, 20 Mar 2024 16:58:24 -0600
Subject: [PATCH 04/28] Updated docker compose files with variables

---
 .github/resources/adhoc-benchmark-docker-compose.yml   | 2 +-
 .github/resources/compare-benchmark-docker-compose.yml | 2 +-
 .github/resources/release-benchmark-docker-compose.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/resources/adhoc-benchmark-docker-compose.yml b/.github/resources/adhoc-benchmark-docker-compose.yml
index 4dcf6a37..68a85ff8 100644
--- a/.github/resources/adhoc-benchmark-docker-compose.yml
+++ b/.github/resources/adhoc-benchmark-docker-compose.yml
@@ -2,7 +2,7 @@ version: "3.4"
 
 services:
   deephaven:
-    image: ghcr.io/deephaven/server:${DOCKER_IMG}
+    image: ${DOCKER_IMG}
     ports:
       - "${DEEPHAVEN_PORT:-10000}:10000"
     volumes:
diff --git a/.github/resources/compare-benchmark-docker-compose.yml b/.github/resources/compare-benchmark-docker-compose.yml
index 4dcf6a37..68a85ff8 100644
--- a/.github/resources/compare-benchmark-docker-compose.yml
+++ b/.github/resources/compare-benchmark-docker-compose.yml
@@ -2,7 +2,7 @@ version: "3.4"
 
 services:
   deephaven:
-    image: ghcr.io/deephaven/server:${DOCKER_IMG}
+    image: ${DOCKER_IMG}
     ports:
       - "${DEEPHAVEN_PORT:-10000}:10000"
     volumes:
diff --git a/.github/resources/release-benchmark-docker-compose.yml b/.github/resources/release-benchmark-docker-compose.yml
index 4dcf6a37..68a85ff8 100644
--- a/.github/resources/release-benchmark-docker-compose.yml
+++ b/.github/resources/release-benchmark-docker-compose.yml
@@ -2,7 +2,7 @@ version: "3.4"
 
 services:
   deephaven:
-    image: ghcr.io/deephaven/server:${DOCKER_IMG}
+    image: ${DOCKER_IMG}
     ports:
       - "${DEEPHAVEN_PORT:-10000}:10000"
     volumes:

From c154485302fa652247de1c26109a1aabed283520 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Wed, 20 Mar 2024 18:19:41 -0600
Subject: [PATCH 05/28] Get jars from remote for artifact

---
 .github/scripts/fetch-results-local.sh | 1 +
 .github/scripts/run-publish-local.sh   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/scripts/fetch-results-local.sh b/.github/scripts/fetch-results-local.sh
index 0269e433..b7e1fdda 100755
--- a/.github/scripts/fetch-results-local.sh
+++ b/.github/scripts/fetch-results-local.sh
@@ -22,6 +22,7 @@ fi
 # Pull results from the benchmark server
 scp -r ${USER}@${HOST}:${RUN_DIR}/results .
 scp -r ${USER}@${HOST}:${RUN_DIR}/logs .
+scp -r ${USER}@${HOST}:${RUN_DIR}/*.jar .
 
 # If the RUN_TYPE is adhoc, userfy the destination directory
 DEST_DIR=${RUN_TYPE}
diff --git a/.github/scripts/run-publish-local.sh b/.github/scripts/run-publish-local.sh
index 15cda446..91a60413 100755
--- a/.github/scripts/run-publish-local.sh
+++ b/.github/scripts/run-publish-local.sh
@@ -22,7 +22,7 @@ BENCH_PROPS_NAME=${RUN_TYPE}-scale-benchmark.properties
 BENCH_PROPS_PATH=${GIT_DIR}/.github/resources/${BENCH_PROPS_NAME}
 
 mkdir -p ${RUN_DIR}
-cp ${GIT_DIR}/target/deephaven-benchmark-*.jar ${RUN_DIR}/
+cp ./deephaven-benchmark-*.jar ${RUN_DIR}/
 rm -f ${RUN_DIR}/deephaven-benchmark*-tests.jar
 cat ${BENCH_PROPS_PATH} | sed 's|${slackToken}|'"${SLACK_TOKEN}|g" | sed 's|${slackChannel}'"|${SLACK_CHANNEL}|g" > ${RUN_DIR}/${BENCH_PROPS_NAME}
 

From d5e2af0113d1987841837489c1aa4e91fb9bac32 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Wed, 20 Mar 2024 18:41:11 -0600
Subject: [PATCH 06/28] Fixed issue with slack publish

---
 .github/scripts/run-publish-local.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/scripts/run-publish-local.sh b/.github/scripts/run-publish-local.sh
index 91a60413..289ad658 100755
--- a/.github/scripts/run-publish-local.sh
+++ b/.github/scripts/run-publish-local.sh
@@ -27,6 +27,8 @@ rm -f ${RUN_DIR}/deephaven-benchmark*-tests.jar
 cat ${BENCH_PROPS_PATH} | sed 's|${slackToken}|'"${SLACK_TOKEN}|g" | sed 's|${slackChannel}'"|${SLACK_CHANNEL}|g" > ${RUN_DIR}/${BENCH_PROPS_NAME}
 
 cd ${DEEPHAVEN_DIR}
+cp ${GIT_DIR}/.github/resources/integration-docker-compose.yml docker-compose.yml
+docker compose pull
 sudo docker compose down
 sudo docker compose up -d
 sleep 10

From 223a63ca7c23bf302f3b632f7f6d38e60a2674f1 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Thu, 21 Mar 2024 11:54:03 -0600
Subject: [PATCH 07/28] Changed ssh to https for cloning for building docker
 image

---
 .github/scripts/build-docker-image-remote.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
index b17094cb..37d4d101 100755
--- a/.github/scripts/build-docker-image-remote.sh
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -43,14 +43,14 @@ BRANCH_NAME=${splitarr[1]}
 title "-- Cloning deephaven-core --"
 cd ${GIT_DIR}
 rm -rf deephaven-core 
-git clone git@github.com:${OWNER}/deephaven-core.git
+git clone https://github.com/${OWNER}/deephaven-core.git
 cd deephaven-core
 git checkout ${BRANCH_NAME}
 
 title "-- Cloning deephaven-server-docker --"
 cd ${GIT_DIR}
 rm -rf deephaven-server-docker
-git clone git@github.com:deephaven/deephaven-server-docker.git
+git clone https://github.com/${OWNER}/deephaven-server-docker.git
 cd deephaven-server-docker
 git checkout main
 

From ac3fb26bd335a9bbd3b8262db6a7116446e59f8b Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Thu, 21 Mar 2024 12:00:17 -0600
Subject: [PATCH 08/28] Fixed deephaven server docker URL

---
 .github/scripts/build-docker-image-remote.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
index 37d4d101..ca772b36 100755
--- a/.github/scripts/build-docker-image-remote.sh
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -50,7 +50,7 @@ git checkout ${BRANCH_NAME}
 title "-- Cloning deephaven-server-docker --"
 cd ${GIT_DIR}
 rm -rf deephaven-server-docker
-git clone https://github.com/${OWNER}/deephaven-server-docker.git
+git clone https://github.com/deephaven/deephaven-server-docker.git
 cd deephaven-server-docker
 git checkout main
 

From 88c3958ea695fa740e19d2d5e76e3cef07367897 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Thu, 21 Mar 2024 13:00:16 -0600
Subject: [PATCH 09/28] Added Temurin JDK install

---
 .github/scripts/build-docker-image-remote.sh | 2 +-
 .github/scripts/setup-test-server-remote.sh  | 7 +++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
index ca772b36..f452c397 100755
--- a/.github/scripts/build-docker-image-remote.sh
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -56,7 +56,7 @@ git checkout main
 
 title "-- Assembling Python Deephaven Core Server --"
 cd ${GIT_DIR}/deephaven-core
-OLD_JAVA_HOME="${JAVA_HOME}"
+OLD_JAVA_HOME="${JAVA_HOME:-}"
 export JAVA_HOME=/usr/lib/jvm/${JAVA}
 ./gradlew outputVersion server-jetty-app:assemble py-server:assemble
 export DEEPHAVEN_VERSION=$(cat build/version)
diff --git a/.github/scripts/setup-test-server-remote.sh b/.github/scripts/setup-test-server-remote.sh
index f631b342..6c34ee42 100755
--- a/.github/scripts/setup-test-server-remote.sh
+++ b/.github/scripts/setup-test-server-remote.sh
@@ -31,8 +31,11 @@ title "- Setting Up Remote Benchmark Testing on ${HOST} -"
 title "-- Adding OS Applications --"
 apt update
 
-title "-- Installing JDK 21 --"
-apt install openjdk-21-jre-headless
+title "-- Installing JVMs --"
+apt install temurin-11-jdk
+apt install temurin-21-jdk
+# Look at installed packages:  dpkg --list | grep jdk
+# Configure default java:  update-alternatives --config java
 
 title "-- Installing Maven --"
 apt install maven

From 75b22ef51b4c567c0c73a999dd996cc74ee5257c Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Thu, 21 Mar 2024 14:49:30 -0600
Subject: [PATCH 10/28] Added JDK checking and installation to Benchmark server
 setup

---
 .github/scripts/setup-test-server-remote.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/scripts/setup-test-server-remote.sh b/.github/scripts/setup-test-server-remote.sh
index 6c34ee42..e7daf37d 100755
--- a/.github/scripts/setup-test-server-remote.sh
+++ b/.github/scripts/setup-test-server-remote.sh
@@ -29,7 +29,13 @@ title () { echo; echo $1; }
 title "- Setting Up Remote Benchmark Testing on ${HOST} -"
 
 title "-- Adding OS Applications --"
-apt update
+UPDATED=$(update-alternatives --list java | grep -i temurin; echo $?)
+if [[ %{UPDATED} != 0 ]]; then
+  apt install -y wget apt-transport-https gpg
+  wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add -
+  echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list
+  apt update
+fi
 
 title "-- Installing JVMs --"
 apt install temurin-11-jdk

From 27846fae53886f5a9a639d0afc2cd3d5f4b5e6ff Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Thu, 21 Mar 2024 14:55:57 -0600
Subject: [PATCH 11/28] Added promptless apt jdk install

---
 .github/scripts/setup-test-server-remote.sh | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/scripts/setup-test-server-remote.sh b/.github/scripts/setup-test-server-remote.sh
index e7daf37d..6047c93b 100755
--- a/.github/scripts/setup-test-server-remote.sh
+++ b/.github/scripts/setup-test-server-remote.sh
@@ -30,7 +30,8 @@ title "- Setting Up Remote Benchmark Testing on ${HOST} -"
 
 title "-- Adding OS Applications --"
 UPDATED=$(update-alternatives --list java | grep -i temurin; echo $?)
-if [[ %{UPDATED} != 0 ]]; then
+if [[ ${UPDATED} != 0 ]]; then
+  title "-- Adding Adoptium to APT registry --"
   apt install -y wget apt-transport-https gpg
   wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add -
   echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list
@@ -38,13 +39,13 @@ if [[ %{UPDATED} != 0 ]]; then
 fi
 
 title "-- Installing JVMs --"
-apt install temurin-11-jdk
-apt install temurin-21-jdk
+apt -y install temurin-11-jdk
+apt -y install temurin-21-jdk
 # Look at installed packages:  dpkg --list | grep jdk
 # Configure default java:  update-alternatives --config java
 
 title "-- Installing Maven --"
-apt install maven
+apt -y install maven
 
 title "-- Installing Docker --"
 command_exists() {

From 9a80b26efc01d7ec644d1e77a21e3a6a3cb1e277 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Thu, 21 Mar 2024 15:07:16 -0600
Subject: [PATCH 12/28] Don't enforce set -u

---
 .github/scripts/build-docker-image-remote.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
index f452c397..b392fa4d 100755
--- a/.github/scripts/build-docker-image-remote.sh
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -2,7 +2,6 @@
 
 set -o errexit
 set -o pipefail
-set -o nounset
 
 # Build a local docker image on the remote side if needed
 # Ensure the docker image is running in the Deephaven directory
@@ -56,7 +55,7 @@ git checkout main
 
 title "-- Assembling Python Deephaven Core Server --"
 cd ${GIT_DIR}/deephaven-core
-OLD_JAVA_HOME="${JAVA_HOME:-}"
+OLD_JAVA_HOME="${JAVA_HOME}"
 export JAVA_HOME=/usr/lib/jvm/${JAVA}
 ./gradlew outputVersion server-jetty-app:assemble py-server:assemble
 export DEEPHAVEN_VERSION=$(cat build/version)

From af1d9f1276c08f346e358b30652fde50042c8e58 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Thu, 21 Mar 2024 15:13:58 -0600
Subject: [PATCH 13/28] Specify build java

---
 .github/scripts/build-docker-image-remote.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
index b392fa4d..299fff0d 100755
--- a/.github/scripts/build-docker-image-remote.sh
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -11,6 +11,7 @@ GIT_DIR=/root/git
 DEEPHAVEN_DIR=/root/deephaven
 DOCKER_IMG=$1
 BRANCH_DELIM="::"
+BUILD_JAVA=temurin-11-jdk-amd64
 
 if [ ! -d "${DEEPHAVEN_DIR}" ]; then
   echo "$0: Missing one or more Benchmark setup directories"
@@ -56,7 +57,7 @@ git checkout main
 title "-- Assembling Python Deephaven Core Server --"
 cd ${GIT_DIR}/deephaven-core
 OLD_JAVA_HOME="${JAVA_HOME}"
-export JAVA_HOME=/usr/lib/jvm/${JAVA}
+export JAVA_HOME=/usr/lib/jvm/${BUILD_JAVA}
 ./gradlew outputVersion server-jetty-app:assemble py-server:assemble
 export DEEPHAVEN_VERSION=$(cat build/version)
 export JAVA_HOME="${OLD_JAVA_HOME}"

From 8b05e61cc8e9ae19c71c27cfeb54bf5093e9c5cd Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Thu, 21 Mar 2024 17:47:28 -0600
Subject: [PATCH 14/28] Be more forceful in purge docker stuff

---
 .github/scripts/setup-test-server-remote.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/setup-test-server-remote.sh b/.github/scripts/setup-test-server-remote.sh
index 6047c93b..da70af1a 100755
--- a/.github/scripts/setup-test-server-remote.sh
+++ b/.github/scripts/setup-test-server-remote.sh
@@ -80,8 +80,8 @@ git checkout ${GIT_BRANCH}
 
 title "-- Stopping and Removing Docker Installations --"
 docker ps -a -q | xargs --no-run-if-empty -n 1 docker stop
-docker ps -a -q | xargs --no-run-if-empty -n 1 docker rm
-docker images -a -q | xargs --no-run-if-empty -n 1 docker rmi
+docker ps -a -q | xargs --no-run-if-empty -n 1 docker rm --force
+docker images -a -q | xargs --no-run-if-empty -n 1 docker rmi --force
 docker system prune --volumes --force
 rm -rf ${DEEPHAVEN_DIR}
 

From 522af532558a836e6d5cc38ea1b2c071ff264f80 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Thu, 21 Mar 2024 19:05:10 -0600
Subject: [PATCH 15/28] Added some debug lines

---
 .github/scripts/build-docker-image-remote.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
index 299fff0d..7186a728 100755
--- a/.github/scripts/build-docker-image-remote.sh
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -59,8 +59,11 @@ cd ${GIT_DIR}/deephaven-core
 OLD_JAVA_HOME="${JAVA_HOME}"
 export JAVA_HOME=/usr/lib/jvm/${BUILD_JAVA}
 ./gradlew outputVersion server-jetty-app:assemble py-server:assemble
+title "-- Finished Assembly Build --"
 export DEEPHAVEN_VERSION=$(cat build/version)
+title "-- Deephaven Version ${DEEPHAVEN_VERSION}--"
 export JAVA_HOME="${OLD_JAVA_HOME}"
+title "-- Restored Java Home --"
 
 title "-- Building Deephaven Docker Image --"
 cd ${GIT_DIR}/deephaven-server-docker

From a845d1b60ca9c96ed73577b3fdef2942b792c82a Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 11:05:32 -0600
Subject: [PATCH 16/28] More build issues experiments

---
 .github/scripts/build-docker-image-remote.sh | 6 +++---
 .github/scripts/setup-test-server-remote.sh  | 4 ++--
 .github/workflows/remote-benchmarks.yml      | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
index 7186a728..ceff87a6 100755
--- a/.github/scripts/build-docker-image-remote.sh
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -58,12 +58,12 @@ title "-- Assembling Python Deephaven Core Server --"
 cd ${GIT_DIR}/deephaven-core
 OLD_JAVA_HOME="${JAVA_HOME}"
 export JAVA_HOME=/usr/lib/jvm/${BUILD_JAVA}
+
+echo "org.gradle.daemon=false" >> gradle.properties
 ./gradlew outputVersion server-jetty-app:assemble py-server:assemble
-title "-- Finished Assembly Build --"
+
 export DEEPHAVEN_VERSION=$(cat build/version)
-title "-- Deephaven Version ${DEEPHAVEN_VERSION}--"
 export JAVA_HOME="${OLD_JAVA_HOME}"
-title "-- Restored Java Home --"
 
 title "-- Building Deephaven Docker Image --"
 cd ${GIT_DIR}/deephaven-server-docker
diff --git a/.github/scripts/setup-test-server-remote.sh b/.github/scripts/setup-test-server-remote.sh
index da70af1a..138e9117 100755
--- a/.github/scripts/setup-test-server-remote.sh
+++ b/.github/scripts/setup-test-server-remote.sh
@@ -68,9 +68,9 @@ fi
 
 title "-- Removing Git Benchmark Repositories --"
 rm -rf ${GIT_DIR}
+mkdir -p ${GIT_DIR}
 
 title "-- Clone Git Benchmark Repository ${GIT_REPO} --"
-mkdir -p ${GIT_DIR}
 cd ${GIT_DIR}
 git clone https://github.com/${GIT_REPO}.git
 cd benchmark
@@ -79,7 +79,7 @@ title "-- Clone Git Benchmark Branch ${GIT_BRANCH} --"
 git checkout ${GIT_BRANCH}
 
 title "-- Stopping and Removing Docker Installations --"
-docker ps -a -q | xargs --no-run-if-empty -n 1 docker stop
+docker ps -a -q | xargs --no-run-if-empty -n 1 docker kill
 docker ps -a -q | xargs --no-run-if-empty -n 1 docker rm --force
 docker images -a -q | xargs --no-run-if-empty -n 1 docker rmi --force
 docker system prune --volumes --force
diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index e658ada3..c9a3e465 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -54,7 +54,7 @@ jobs:
         distribution: 'temurin'
         cache: maven
         
-    - name: Setup Local and Remote Scripts
+    - name: Setup Local Scripts
       run: |
         sudo chmod +x ${SD}/*
         ${SD}/setup-ssh-local.sh ${HOST} "${{secrets.BENCHMARK_KEY}}"

From baa7860c6f86dc0e67c81de55a9e60e3b3a88c4b Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 13:36:15 -0600
Subject: [PATCH 17/28] Split up the DH distribution build and the docker image
 build

---
 .github/scripts/build-docker-image-remote.sh  | 39 ++---------
 .../build-server-distribution-remote.sh       | 66 +++++++++++++++++++
 .github/scripts/setup-test-server-remote.sh   | 10 ++-
 .github/workflows/remote-benchmarks.yml       |  9 ++-
 4 files changed, 87 insertions(+), 37 deletions(-)
 create mode 100755 .github/scripts/build-server-distribution-remote.sh

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
index ceff87a6..8fe1e41a 100755
--- a/.github/scripts/build-docker-image-remote.sh
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -3,23 +3,21 @@
 set -o errexit
 set -o pipefail
 
-# Build a local docker image on the remote side if needed
+# Build a local docker image on the remote side
 # Ensure the docker image is running in the Deephaven directory
 
 HOST=`hostname`
 GIT_DIR=/root/git
 DEEPHAVEN_DIR=/root/deephaven
-DOCKER_IMG=$1
-BRANCH_DELIM="::"
-BUILD_JAVA=temurin-11-jdk-amd64
+DEEPHAVEN_VERSION=build/version
 
 if [ ! -d "${DEEPHAVEN_DIR}" ]; then
   echo "$0: Missing one or more Benchmark setup directories"
   exit 1
 fi
 
-if [[ $# != 1 ]]; then
-  echo "$0: Missing docker image/branch argument"
+if [ ! -f "${DEEPHAVEN_VERSION}" ]; then
+  echo "$0: Missing Deephaven version file. Was the project built first?"
   exit 1
 fi
 
@@ -36,36 +34,9 @@ if [[ ${DOCKER_IMG} != *"${BRANCH_DELIM}"* ]]; then
   exit 0
 fi
 
-readarray -d "${BRANCH_DELIM}" -t splitarr <<< "${DOCKER_IMG}"
-OWNER=${splitarr[0]}
-BRANCH_NAME=${splitarr[1]}
-
-title "-- Cloning deephaven-core --"
-cd ${GIT_DIR}
-rm -rf deephaven-core 
-git clone https://github.com/${OWNER}/deephaven-core.git
-cd deephaven-core
-git checkout ${BRANCH_NAME}
-
-title "-- Cloning deephaven-server-docker --"
-cd ${GIT_DIR}
-rm -rf deephaven-server-docker
-git clone https://github.com/deephaven/deephaven-server-docker.git
-cd deephaven-server-docker
-git checkout main
-
-title "-- Assembling Python Deephaven Core Server --"
-cd ${GIT_DIR}/deephaven-core
-OLD_JAVA_HOME="${JAVA_HOME}"
-export JAVA_HOME=/usr/lib/jvm/${BUILD_JAVA}
-
-echo "org.gradle.daemon=false" >> gradle.properties
-./gradlew outputVersion server-jetty-app:assemble py-server:assemble
-
-export DEEPHAVEN_VERSION=$(cat build/version)
-export JAVA_HOME="${OLD_JAVA_HOME}"
 
 title "-- Building Deephaven Docker Image --"
+export DEEPHAVEN_VERSION=$(cat ${DEEPHAVEN_VERSION})
 cd ${GIT_DIR}/deephaven-server-docker
 cp ${GIT_DIR}/deephaven-core/server/jetty-app/build/distributions/server-jetty-*.tar contexts/server/
 cp ${GIT_DIR}/deephaven-core/server/jetty-app/build/distributions/server-jetty-*.tar contexts/server-slim/
diff --git a/.github/scripts/build-server-distribution-remote.sh b/.github/scripts/build-server-distribution-remote.sh
new file mode 100755
index 00000000..a8b4c9a3
--- /dev/null
+++ b/.github/scripts/build-server-distribution-remote.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+
+# Assemble the Deephaven server artifacts on the remote side if needed
+# The supplied argument can be an image name or <owner>::<branch>
+# Ensure that the artifacts and Deephaven version are available in standard directories
+
+HOST=`hostname`
+GIT_DIR=/root/git
+DEEPHAVEN_DIR=/root/deephaven
+DOCKER_IMG=$1
+BRANCH_DELIM="::"
+BUILD_JAVA=temurin-11-jdk-amd64
+
+if [ ! -d "${DEEPHAVEN_DIR}" ]; then
+  echo "$0: Missing one or more Benchmark setup directories"
+  exit 1
+fi
+
+if [[ $# != 1 ]]; then
+  echo "$0: Missing docker image/branch argument"
+  exit 1
+fi
+
+title () { echo; echo $1; }
+
+title "- Setting up Remote Docker Image on ${HOST} -"
+
+if [[ ${DOCKER_IMG} != *"${BRANCH_DELIM}"* ]]; then
+  cd ${DEEPHAVEN_DIR}
+  echo "DOCKER_IMG=ghcr.io/deephaven/server:${DOCKER_IMG}" > .env
+  docker compose pull
+  title "-- Starting Deephaven and Redpanda --"
+  docker compose up -d
+  exit 0
+fi
+
+readarray -d "${BRANCH_DELIM}" -t splitarr <<< "${DOCKER_IMG}"
+OWNER=${splitarr[0]}
+BRANCH_NAME=${splitarr[1]}
+
+title "-- Cloning deephaven-core --"
+cd ${GIT_DIR}
+rm -rf deephaven-core 
+git clone https://github.com/${OWNER}/deephaven-core.git
+cd deephaven-core
+git checkout ${BRANCH_NAME}
+
+title "-- Cloning deephaven-server-docker --"
+cd ${GIT_DIR}
+rm -rf deephaven-server-docker
+git clone https://github.com/deephaven/deephaven-server-docker.git
+cd deephaven-server-docker
+git checkout main
+
+title "-- Assembling Python Deephaven Core Server --"
+cd ${GIT_DIR}/deephaven-core
+OLD_JAVA_HOME="${JAVA_HOME}"
+export JAVA_HOME=/usr/lib/jvm/${BUILD_JAVA}
+
+echo "org.gradle.daemon=false" >> gradle.properties
+./gradlew outputVersion server-jetty-app:assemble py-server:assemble
+
+
diff --git a/.github/scripts/setup-test-server-remote.sh b/.github/scripts/setup-test-server-remote.sh
index 138e9117..a8c34c7b 100755
--- a/.github/scripts/setup-test-server-remote.sh
+++ b/.github/scripts/setup-test-server-remote.sh
@@ -78,10 +78,16 @@ cd benchmark
 title "-- Clone Git Benchmark Branch ${GIT_BRANCH} --"
 git checkout ${GIT_BRANCH}
 
-title "-- Stopping and Removing Docker Installations --"
-docker ps -a -q | xargs --no-run-if-empty -n 1 docker kill
+title "-- Stopping Docker Containers --"
+docker ps -a -q | xargs --no-run-if-empty -n 1 docker stop --force
+
+title "-- Removing Docker Containers --"
 docker ps -a -q | xargs --no-run-if-empty -n 1 docker rm --force
+
+title "-- Removing Docker Images --"
 docker images -a -q | xargs --no-run-if-empty -n 1 docker rmi --force
+
+title "-- Pruning Docker Volumes --"
 docker system prune --volumes --force
 rm -rf ${DEEPHAVEN_DIR}
 
diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index c9a3e465..45c72098 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -44,6 +44,7 @@ jobs:
       TEST_RGX: ${{inputs.test_class_regex}}
       ROW_CNT: ${{inputs.scale_row_count}}
       ACTOR: ${{github.actor}}
+      IS_IMAGE_BUILD: contains(${DOCKER_IMG}, "::")
 
     steps:
     - uses: actions/checkout@v3
@@ -63,9 +64,15 @@ jobs:
       run: |
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} setup-test-server-remote ${REPO} ${BRANCH} ${RUN_TYPE} "${DOCKER_IMG}"
 
+    - name: Run Remote Server Distribution Build
+      if: ${IS_IMAGE_BUILD}
+      run: |
+        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-server-distribution-remote "${DOCKER_IMG}"
+        
     - name: Run Remote Docker Image Build
+      if: ${IS_IMAGE_BUILD}
       run: |
-        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-docker-image-remote "${DOCKER_IMG}"
+        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-docker-image-remote
 
     - name: Run Remote Benchmark Artifact Build
       run: |

From 1e469c7c3f565e9d78a1817d9c6ce8be1ae7f273 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 13:41:31 -0600
Subject: [PATCH 18/28] Workflow syntax error

---
 .github/workflows/remote-benchmarks.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index 45c72098..837b0d0a 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -65,12 +65,12 @@ jobs:
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} setup-test-server-remote ${REPO} ${BRANCH} ${RUN_TYPE} "${DOCKER_IMG}"
 
     - name: Run Remote Server Distribution Build
-      if: ${IS_IMAGE_BUILD}
+      if: ${{.env.IS_IMAGE_BUILD}}
       run: |
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-server-distribution-remote "${DOCKER_IMG}"
         
     - name: Run Remote Docker Image Build
-      if: ${IS_IMAGE_BUILD}
+      if: ${{.env.IS_IMAGE_BUILD}}
       run: |
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-docker-image-remote
 

From 025f5de3d8c6d4698238d6b0a1629291b050b0cc Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 13:42:45 -0600
Subject: [PATCH 19/28] Sigh

---
 .github/workflows/remote-benchmarks.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index 837b0d0a..4dda316b 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -65,12 +65,12 @@ jobs:
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} setup-test-server-remote ${REPO} ${BRANCH} ${RUN_TYPE} "${DOCKER_IMG}"
 
     - name: Run Remote Server Distribution Build
-      if: ${{.env.IS_IMAGE_BUILD}}
+      if: ${{env.IS_IMAGE_BUILD}}
       run: |
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-server-distribution-remote "${DOCKER_IMG}"
         
     - name: Run Remote Docker Image Build
-      if: ${{.env.IS_IMAGE_BUILD}}
+      if: ${{env.IS_IMAGE_BUILD}}
       run: |
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-docker-image-remote
 

From 21ae73a6e7df83b62db8a9c36810c1b882fe7ba2 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 14:37:20 -0600
Subject: [PATCH 20/28] Fixed the checked location of DH version

---
 .github/scripts/build-docker-image-remote.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
index 8fe1e41a..ba424dc3 100755
--- a/.github/scripts/build-docker-image-remote.sh
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -9,7 +9,7 @@ set -o pipefail
 HOST=`hostname`
 GIT_DIR=/root/git
 DEEPHAVEN_DIR=/root/deephaven
-DEEPHAVEN_VERSION=build/version
+DEEPHAVEN_VERSION=${GIT_DIR}/deephaven-server-docker/build/version
 
 if [ ! -d "${DEEPHAVEN_DIR}" ]; then
   echo "$0: Missing one or more Benchmark setup directories"

From 3e5561d2b8000ecf737a347fb4064b43c04a057e Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 14:46:26 -0600
Subject: [PATCH 21/28] Fixed another path issue

---
 .github/scripts/build-docker-image-remote.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
index ba424dc3..f440eefa 100755
--- a/.github/scripts/build-docker-image-remote.sh
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -9,14 +9,14 @@ set -o pipefail
 HOST=`hostname`
 GIT_DIR=/root/git
 DEEPHAVEN_DIR=/root/deephaven
-DEEPHAVEN_VERSION=${GIT_DIR}/deephaven-server-docker/build/version
+DEEPHAVEN_VERSION_FILE=${GIT_DIR}/deephaven-core/build/version
 
 if [ ! -d "${DEEPHAVEN_DIR}" ]; then
   echo "$0: Missing one or more Benchmark setup directories"
   exit 1
 fi
 
-if [ ! -f "${DEEPHAVEN_VERSION}" ]; then
+if [ ! -f "${DEEPHAVEN_VERSION_FILE}" ]; then
   echo "$0: Missing Deephaven version file. Was the project built first?"
   exit 1
 fi
@@ -36,7 +36,7 @@ fi
 
 
 title "-- Building Deephaven Docker Image --"
-export DEEPHAVEN_VERSION=$(cat ${DEEPHAVEN_VERSION})
+export DEEPHAVEN_VERSION=$(cat ${DEEPHAVEN_VERSION_FILE})
 cd ${GIT_DIR}/deephaven-server-docker
 cp ${GIT_DIR}/deephaven-core/server/jetty-app/build/distributions/server-jetty-*.tar contexts/server/
 cp ${GIT_DIR}/deephaven-core/server/jetty-app/build/distributions/server-jetty-*.tar contexts/server-slim/

From d7cafd0d66ee7f63679a8d9e3b3efb40ee26a8cf Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 15:06:45 -0600
Subject: [PATCH 22/28] More GH workflow conditions

---
 .github/workflows/remote-benchmarks.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index 4dda316b..ad569b46 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -44,7 +44,7 @@ jobs:
       TEST_RGX: ${{inputs.test_class_regex}}
       ROW_CNT: ${{inputs.scale_row_count}}
       ACTOR: ${{github.actor}}
-      IS_IMAGE_BUILD: contains(${DOCKER_IMG}, "::")
+      IS_IMAGE_BUILD: ${{ contains(env.DOCKER_IMG, "::") }}
 
     steps:
     - uses: actions/checkout@v3

From b83571b1958036ee980e56178d7af9f1007614da Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 15:09:16 -0600
Subject: [PATCH 23/28] More GH workflow conditions

---
 .github/workflows/remote-benchmarks.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index ad569b46..ba45bdc3 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -44,7 +44,7 @@ jobs:
       TEST_RGX: ${{inputs.test_class_regex}}
       ROW_CNT: ${{inputs.scale_row_count}}
       ACTOR: ${{github.actor}}
-      IS_IMAGE_BUILD: ${{ contains(env.DOCKER_IMG, "::") }}
+      IS_IMAGE_BUILD: ${{ contains(inputs.docker_image, "::") }}
 
     steps:
     - uses: actions/checkout@v3

From 0bcbfae450860b552f2d483b10ee9859a3855d31 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 15:11:48 -0600
Subject: [PATCH 24/28] More GH workflow conditions

---
 .github/workflows/remote-benchmarks.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index ba45bdc3..542b0371 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -44,7 +44,7 @@ jobs:
       TEST_RGX: ${{inputs.test_class_regex}}
       ROW_CNT: ${{inputs.scale_row_count}}
       ACTOR: ${{github.actor}}
-      IS_IMAGE_BUILD: ${{ contains(inputs.docker_image, "::") }}
+      IS_IMAGE_BUILD: ${{ contains(inputs.docker_image, '::') }}
 
     steps:
     - uses: actions/checkout@v3

From 2c7148c4c8d7eb75096ace6a57ea28438eec6c8f Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 15:20:42 -0600
Subject: [PATCH 25/28] Removed force from docker stop

---
 .github/scripts/setup-test-server-remote.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/scripts/setup-test-server-remote.sh b/.github/scripts/setup-test-server-remote.sh
index a8c34c7b..0331b3a6 100755
--- a/.github/scripts/setup-test-server-remote.sh
+++ b/.github/scripts/setup-test-server-remote.sh
@@ -79,7 +79,7 @@ title "-- Clone Git Benchmark Branch ${GIT_BRANCH} --"
 git checkout ${GIT_BRANCH}
 
 title "-- Stopping Docker Containers --"
-docker ps -a -q | xargs --no-run-if-empty -n 1 docker stop --force
+docker ps -a -q | xargs --no-run-if-empty -n 1 docker kill
 
 title "-- Removing Docker Containers --"
 docker ps -a -q | xargs --no-run-if-empty -n 1 docker rm --force

From 57b03d267591441bb016cb86624d5a734c87d152 Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 15:47:03 -0600
Subject: [PATCH 26/28] More GH workflow trial and error

---
 .github/workflows/remote-benchmarks.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index 542b0371..36ef53b6 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -44,7 +44,6 @@ jobs:
       TEST_RGX: ${{inputs.test_class_regex}}
       ROW_CNT: ${{inputs.scale_row_count}}
       ACTOR: ${{github.actor}}
-      IS_IMAGE_BUILD: ${{ contains(inputs.docker_image, '::') }}
 
     steps:
     - uses: actions/checkout@v3
@@ -65,12 +64,12 @@ jobs:
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} setup-test-server-remote ${REPO} ${BRANCH} ${RUN_TYPE} "${DOCKER_IMG}"
 
     - name: Run Remote Server Distribution Build
-      if: ${{env.IS_IMAGE_BUILD}}
+      if: ${{ contains(env.DOCKER_IMG, '::') }}
       run: |
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-server-distribution-remote "${DOCKER_IMG}"
         
     - name: Run Remote Docker Image Build
-      if: ${{env.IS_IMAGE_BUILD}}
+      if: ${{ contains(env.DOCKER_IMG, '::') }}
       run: |
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-docker-image-remote
 

From 61902676ac37f3370374ce0859db89b1708bad1a Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 16:50:50 -0600
Subject: [PATCH 27/28] Reworked the GH workflow to separate conditional builds
 from start/stop

---
 .github/scripts/build-docker-image-remote.sh  | 17 -------
 .../build-server-distribution-remote.sh       | 11 -----
 .github/scripts/manage-deephaven-remote.sh    | 46 +++++++++++++++++++
 .github/workflows/remote-benchmarks.yml       |  4 ++
 4 files changed, 50 insertions(+), 28 deletions(-)
 create mode 100755 .github/scripts/manage-deephaven-remote.sh

diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh
index f440eefa..341ea2ca 100755
--- a/.github/scripts/build-docker-image-remote.sh
+++ b/.github/scripts/build-docker-image-remote.sh
@@ -25,16 +25,6 @@ title () { echo; echo $1; }
 
 title "- Setting up Remote Docker Image on ${HOST} -"
 
-if [[ ${DOCKER_IMG} != *"${BRANCH_DELIM}"* ]]; then
-  cd ${DEEPHAVEN_DIR}
-  echo "DOCKER_IMG=ghcr.io/deephaven/server:${DOCKER_IMG}" > .env
-  docker compose pull
-  title "-- Starting Deephaven and Redpanda --"
-  docker compose up -d
-  exit 0
-fi
-
-
 title "-- Building Deephaven Docker Image --"
 export DEEPHAVEN_VERSION=$(cat ${DEEPHAVEN_VERSION_FILE})
 cd ${GIT_DIR}/deephaven-server-docker
@@ -50,10 +40,3 @@ echo "DEEPHAVEN_VERSION: ${DEEPHAVEN_VERSION}"
 echo "DEEPHAVEN_CORE_WHEEL: ${DEEPHAVEN_CORE_WHEEL}"
 docker buildx bake -f server.hcl
 
-title "-- Starting Deephaven and Redpanda --"
-cd ${DEEPHAVEN_DIR}
-echo "DOCKER_IMG=deephaven/server:benchmark-local" > .env
-docker compose up -d
-
-
-
diff --git a/.github/scripts/build-server-distribution-remote.sh b/.github/scripts/build-server-distribution-remote.sh
index a8b4c9a3..7a49e76b 100755
--- a/.github/scripts/build-server-distribution-remote.sh
+++ b/.github/scripts/build-server-distribution-remote.sh
@@ -26,17 +26,6 @@ fi
 
 title () { echo; echo $1; }
 
-title "- Setting up Remote Docker Image on ${HOST} -"
-
-if [[ ${DOCKER_IMG} != *"${BRANCH_DELIM}"* ]]; then
-  cd ${DEEPHAVEN_DIR}
-  echo "DOCKER_IMG=ghcr.io/deephaven/server:${DOCKER_IMG}" > .env
-  docker compose pull
-  title "-- Starting Deephaven and Redpanda --"
-  docker compose up -d
-  exit 0
-fi
-
 readarray -d "${BRANCH_DELIM}" -t splitarr <<< "${DOCKER_IMG}"
 OWNER=${splitarr[0]}
 BRANCH_NAME=${splitarr[1]}
diff --git a/.github/scripts/manage-deephaven-remote.sh b/.github/scripts/manage-deephaven-remote.sh
new file mode 100755
index 00000000..4aced933
--- /dev/null
+++ b/.github/scripts/manage-deephaven-remote.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+
+# Start or Stop a Deephaven image based on the given directive and image/branch name
+# The directives argument can be start or stop
+# The supplied image argument can be an image name or <owner>::<branch>
+
+HOST=`hostname`
+DEEPHAVEN_DIR=/root/deephaven
+DOCKER_IMG=$1
+DIRECTIVE=$2
+BRANCH_DELIM="::"
+
+if [ ! -d "${DEEPHAVEN_DIR}" ]; then
+  echo "$0: Missing one or more Benchmark setup directories"
+  exit 1
+fi
+
+if [[ $# != 2 ]]; then
+  echo "$0: Missing docker directive or image/branch argument"
+  exit 1
+fi
+
+title () { echo; echo $1; }
+
+title "- Setting up Remote Docker Image on ${HOST} -"
+
+cd ${DEEPHAVEN_DIR}
+
+if [[ ${DOCKER_IMG} != *"${BRANCH_DELIM}"* ]]; then
+  echo "DOCKER_IMG=ghcr.io/deephaven/server:${DOCKER_IMG}" > .env
+  docker compose pull
+else 
+  echo "DOCKER_IMG=deephaven/server:benchmark-local" > .env
+fi
+
+if [[ ${DIRECTIVE} == 'start' ]]; then
+  docker compose up -d
+fi
+
+if [[ ${DIRECTIVE} == 'stop' ]]; then
+  docker compose down
+fi
+
diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml
index 36ef53b6..144db67e 100644
--- a/.github/workflows/remote-benchmarks.yml
+++ b/.github/workflows/remote-benchmarks.yml
@@ -73,6 +73,10 @@ jobs:
       run: |
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-docker-image-remote
 
+    - name: Start Remote Remote Deephaven Server
+      run: | 
+        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} manage-deephaven-remote start "${DOCKER_IMG}"
+
     - name: Run Remote Benchmark Artifact Build
       run: |
         ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-benchmark-artifact-remote

From b0a2ebfcb4bc90144eea4cc77f76c4391327b65e Mon Sep 17 00:00:00 2001
From: stanbrub <stanbrubaker@illumon.com>
Date: Mon, 25 Mar 2024 16:57:31 -0600
Subject: [PATCH 28/28] Fixed argument order managing docker up down

---
 .github/scripts/manage-deephaven-remote.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/manage-deephaven-remote.sh b/.github/scripts/manage-deephaven-remote.sh
index 4aced933..dd39c399 100755
--- a/.github/scripts/manage-deephaven-remote.sh
+++ b/.github/scripts/manage-deephaven-remote.sh
@@ -9,8 +9,8 @@ set -o pipefail
 
 HOST=`hostname`
 DEEPHAVEN_DIR=/root/deephaven
-DOCKER_IMG=$1
-DIRECTIVE=$2
+DIRECTIVE=$1
+DOCKER_IMG=$2
 BRANCH_DELIM="::"
 
 if [ ! -d "${DEEPHAVEN_DIR}" ]; then