diff --git a/.github/resources/adhoc-benchmark-docker-compose.yml b/.github/resources/adhoc-benchmark-docker-compose.yml index 4dcf6a37..68a85ff8 100644 --- a/.github/resources/adhoc-benchmark-docker-compose.yml +++ b/.github/resources/adhoc-benchmark-docker-compose.yml @@ -2,7 +2,7 @@ version: "3.4" services: deephaven: - image: ghcr.io/deephaven/server:${DOCKER_IMG} + image: ${DOCKER_IMG} ports: - "${DEEPHAVEN_PORT:-10000}:10000" volumes: diff --git a/.github/resources/compare-benchmark-docker-compose.yml b/.github/resources/compare-benchmark-docker-compose.yml index 4dcf6a37..68a85ff8 100644 --- a/.github/resources/compare-benchmark-docker-compose.yml +++ b/.github/resources/compare-benchmark-docker-compose.yml @@ -2,7 +2,7 @@ version: "3.4" services: deephaven: - image: ghcr.io/deephaven/server:${DOCKER_IMG} + image: ${DOCKER_IMG} ports: - "${DEEPHAVEN_PORT:-10000}:10000" volumes: diff --git a/.github/resources/release-benchmark-docker-compose.yml b/.github/resources/release-benchmark-docker-compose.yml index 4dcf6a37..68a85ff8 100644 --- a/.github/resources/release-benchmark-docker-compose.yml +++ b/.github/resources/release-benchmark-docker-compose.yml @@ -2,7 +2,7 @@ version: "3.4" services: deephaven: - image: ghcr.io/deephaven/server:${DOCKER_IMG} + image: ${DOCKER_IMG} ports: - "${DEEPHAVEN_PORT:-10000}:10000" volumes: diff --git a/.github/scripts/build-docker-image-remote.sh b/.github/scripts/build-docker-image-remote.sh new file mode 100755 index 00000000..341ea2ca --- /dev/null +++ b/.github/scripts/build-docker-image-remote.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +# Build a local docker image on the remote side +# Ensure the docker image is running in the Deephaven directory + +HOST=`hostname` +GIT_DIR=/root/git +DEEPHAVEN_DIR=/root/deephaven +DEEPHAVEN_VERSION_FILE=${GIT_DIR}/deephaven-core/build/version + +if [ ! -d "${DEEPHAVEN_DIR}" ]; then + echo "$0: Missing one or more Benchmark setup directories" + exit 1 +fi + +if [ ! -f "${DEEPHAVEN_VERSION_FILE}" ]; then + echo "$0: Missing Deephaven version file. Was the project built first?" + exit 1 +fi + +title () { echo; echo $1; } + +title "- Setting up Remote Docker Image on ${HOST} -" + +title "-- Building Deephaven Docker Image --" +export DEEPHAVEN_VERSION=$(cat ${DEEPHAVEN_VERSION_FILE}) +cd ${GIT_DIR}/deephaven-server-docker +cp ${GIT_DIR}/deephaven-core/server/jetty-app/build/distributions/server-jetty-*.tar contexts/server/ +cp ${GIT_DIR}/deephaven-core/server/jetty-app/build/distributions/server-jetty-*.tar contexts/server-slim/ +cp ${GIT_DIR}/deephaven-core/py/server/build/wheel/deephaven_core-*-py3-none-any.whl contexts/server/ + +export DEEPHAVEN_SOURCES=custom +export DEEPHAVEN_CORE_WHEEL=$(find . -type f -name "*.whl" | xargs -n 1 basename) +export TAG=benchmark-local + +echo "DEEPHAVEN_VERSION: ${DEEPHAVEN_VERSION}" +echo "DEEPHAVEN_CORE_WHEEL: ${DEEPHAVEN_CORE_WHEEL}" +docker buildx bake -f server.hcl + diff --git a/.github/scripts/build-server-distribution-remote.sh b/.github/scripts/build-server-distribution-remote.sh new file mode 100755 index 00000000..7a49e76b --- /dev/null +++ b/.github/scripts/build-server-distribution-remote.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +# Assemble the Deephaven server artifacts on the remote side if needed +# The supplied argument can be an image name or :: +# Ensure that the artifacts and Deephaven version are available in standard directories + +HOST=`hostname` +GIT_DIR=/root/git +DEEPHAVEN_DIR=/root/deephaven +DOCKER_IMG=$1 +BRANCH_DELIM="::" +BUILD_JAVA=temurin-11-jdk-amd64 + +if [ ! -d "${DEEPHAVEN_DIR}" ]; then + echo "$0: Missing one or more Benchmark setup directories" + exit 1 +fi + +if [[ $# != 1 ]]; then + echo "$0: Missing docker image/branch argument" + exit 1 +fi + +title () { echo; echo $1; } + +readarray -d "${BRANCH_DELIM}" -t splitarr <<< "${DOCKER_IMG}" +OWNER=${splitarr[0]} +BRANCH_NAME=${splitarr[1]} + +title "-- Cloning deephaven-core --" +cd ${GIT_DIR} +rm -rf deephaven-core +git clone https://github.com/${OWNER}/deephaven-core.git +cd deephaven-core +git checkout ${BRANCH_NAME} + +title "-- Cloning deephaven-server-docker --" +cd ${GIT_DIR} +rm -rf deephaven-server-docker +git clone https://github.com/deephaven/deephaven-server-docker.git +cd deephaven-server-docker +git checkout main + +title "-- Assembling Python Deephaven Core Server --" +cd ${GIT_DIR}/deephaven-core +OLD_JAVA_HOME="${JAVA_HOME}" +export JAVA_HOME=/usr/lib/jvm/${BUILD_JAVA} + +echo "org.gradle.daemon=false" >> gradle.properties +./gradlew outputVersion server-jetty-app:assemble py-server:assemble + + diff --git a/.github/scripts/fetch-results-local.sh b/.github/scripts/fetch-results-local.sh index 0269e433..b7e1fdda 100755 --- a/.github/scripts/fetch-results-local.sh +++ b/.github/scripts/fetch-results-local.sh @@ -22,6 +22,7 @@ fi # Pull results from the benchmark server scp -r ${USER}@${HOST}:${RUN_DIR}/results . scp -r ${USER}@${HOST}:${RUN_DIR}/logs . +scp -r ${USER}@${HOST}:${RUN_DIR}/*.jar . # If the RUN_TYPE is adhoc, userfy the destination directory DEST_DIR=${RUN_TYPE} diff --git a/.github/scripts/manage-deephaven-remote.sh b/.github/scripts/manage-deephaven-remote.sh new file mode 100755 index 00000000..dd39c399 --- /dev/null +++ b/.github/scripts/manage-deephaven-remote.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +# Start or Stop a Deephaven image based on the given directive and image/branch name +# The directives argument can be start or stop +# The supplied image argument can be an image name or :: + +HOST=`hostname` +DEEPHAVEN_DIR=/root/deephaven +DIRECTIVE=$1 +DOCKER_IMG=$2 +BRANCH_DELIM="::" + +if [ ! -d "${DEEPHAVEN_DIR}" ]; then + echo "$0: Missing one or more Benchmark setup directories" + exit 1 +fi + +if [[ $# != 2 ]]; then + echo "$0: Missing docker directive or image/branch argument" + exit 1 +fi + +title () { echo; echo $1; } + +title "- Setting up Remote Docker Image on ${HOST} -" + +cd ${DEEPHAVEN_DIR} + +if [[ ${DOCKER_IMG} != *"${BRANCH_DELIM}"* ]]; then + echo "DOCKER_IMG=ghcr.io/deephaven/server:${DOCKER_IMG}" > .env + docker compose pull +else + echo "DOCKER_IMG=deephaven/server:benchmark-local" > .env +fi + +if [[ ${DIRECTIVE} == 'start' ]]; then + docker compose up -d +fi + +if [[ ${DIRECTIVE} == 'stop' ]]; then + docker compose down +fi + diff --git a/.github/scripts/run-publish-local.sh b/.github/scripts/run-publish-local.sh index 15cda446..289ad658 100755 --- a/.github/scripts/run-publish-local.sh +++ b/.github/scripts/run-publish-local.sh @@ -22,11 +22,13 @@ BENCH_PROPS_NAME=${RUN_TYPE}-scale-benchmark.properties BENCH_PROPS_PATH=${GIT_DIR}/.github/resources/${BENCH_PROPS_NAME} mkdir -p ${RUN_DIR} -cp ${GIT_DIR}/target/deephaven-benchmark-*.jar ${RUN_DIR}/ +cp ./deephaven-benchmark-*.jar ${RUN_DIR}/ rm -f ${RUN_DIR}/deephaven-benchmark*-tests.jar cat ${BENCH_PROPS_PATH} | sed 's|${slackToken}|'"${SLACK_TOKEN}|g" | sed 's|${slackChannel}'"|${SLACK_CHANNEL}|g" > ${RUN_DIR}/${BENCH_PROPS_NAME} cd ${DEEPHAVEN_DIR} +cp ${GIT_DIR}/.github/resources/integration-docker-compose.yml docker-compose.yml +docker compose pull sudo docker compose down sudo docker compose up -d sleep 10 diff --git a/.github/scripts/setup-test-server-remote.sh b/.github/scripts/setup-test-server-remote.sh index 62b005fe..0331b3a6 100755 --- a/.github/scripts/setup-test-server-remote.sh +++ b/.github/scripts/setup-test-server-remote.sh @@ -29,13 +29,23 @@ title () { echo; echo $1; } title "- Setting Up Remote Benchmark Testing on ${HOST} -" title "-- Adding OS Applications --" -apt update +UPDATED=$(update-alternatives --list java | grep -i temurin; echo $?) +if [[ ${UPDATED} != 0 ]]; then + title "-- Adding Adoptium to APT registry --" + apt install -y wget apt-transport-https gpg + wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - + echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list + apt update +fi -title "-- Installing JDK 21 --" -apt install openjdk-21-jre-headless +title "-- Installing JVMs --" +apt -y install temurin-11-jdk +apt -y install temurin-21-jdk +# Look at installed packages: dpkg --list | grep jdk +# Configure default java: update-alternatives --config java title "-- Installing Maven --" -apt install maven +apt -y install maven title "-- Installing Docker --" command_exists() { @@ -58,9 +68,9 @@ fi title "-- Removing Git Benchmark Repositories --" rm -rf ${GIT_DIR} +mkdir -p ${GIT_DIR} title "-- Clone Git Benchmark Repository ${GIT_REPO} --" -mkdir -p ${GIT_DIR} cd ${GIT_DIR} git clone https://github.com/${GIT_REPO}.git cd benchmark @@ -68,23 +78,23 @@ cd benchmark title "-- Clone Git Benchmark Branch ${GIT_BRANCH} --" git checkout ${GIT_BRANCH} -title "-- Stopping and Removing Docker Installations --" -docker ps -a -q | xargs --no-run-if-empty -n 1 docker stop -docker ps -a -q | xargs --no-run-if-empty -n 1 docker rm -docker images -a -q | xargs --no-run-if-empty -n 1 docker rmi +title "-- Stopping Docker Containers --" +docker ps -a -q | xargs --no-run-if-empty -n 1 docker kill + +title "-- Removing Docker Containers --" +docker ps -a -q | xargs --no-run-if-empty -n 1 docker rm --force + +title "-- Removing Docker Images --" +docker images -a -q | xargs --no-run-if-empty -n 1 docker rmi --force + +title "-- Pruning Docker Volumes --" docker system prune --volumes --force rm -rf ${DEEPHAVEN_DIR} -title "-- Installing Deephaven and Redpanda --" +title "-- Staging Docker Resources --" mkdir -p ${DEEPHAVEN_DIR} cd ${DEEPHAVEN_DIR} cp ${GIT_DIR}/benchmark/.github/resources/${RUN_TYPE}-benchmark-docker-compose.yml docker-compose.yml -echo "DOCKER_IMG=${DOCKER_IMG}" > .env -docker compose pull - -title "-- Starting Deephaven and Redpanda --" -docker compose up -d - diff --git a/.github/workflows/adhoc-remote-benchmarks.yml b/.github/workflows/adhoc-remote-benchmarks.yml index 2e9dd6c7..bf9724fe 100644 --- a/.github/workflows/adhoc-remote-benchmarks.yml +++ b/.github/workflows/adhoc-remote-benchmarks.yml @@ -11,7 +11,7 @@ on: workflow_dispatch: inputs: docker_image: - description: 'Docker Image Name' + description: 'Docker Image Name or DH Core Branch' required: true default: 'edge' type: string diff --git a/.github/workflows/remote-benchmarks.yml b/.github/workflows/remote-benchmarks.yml index 47c5ade4..144db67e 100644 --- a/.github/workflows/remote-benchmarks.yml +++ b/.github/workflows/remote-benchmarks.yml @@ -53,22 +53,8 @@ jobs: java-version: '21' distribution: 'temurin' cache: maven - - - name: Docker Pull Deephaven and Redpanda - run: | - cp .github/resources/${RUN_TYPE}-benchmark-docker-compose.yml docker-compose.yml - echo "DOCKER_IMG=${DOCKER_IMG}" > .env - docker compose pull - - - name: Docker Up Deephaven and Redpanda - run: docker compose up -d - - - name: Build with Maven - run: | - mvn -B verify --file pom.xml - rm -rf results - - - name: Setup Local and Remote Scripts + + - name: Setup Local Scripts run: | sudo chmod +x ${SD}/* ${SD}/setup-ssh-local.sh ${HOST} "${{secrets.BENCHMARK_KEY}}" @@ -77,6 +63,20 @@ jobs: run: | ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} setup-test-server-remote ${REPO} ${BRANCH} ${RUN_TYPE} "${DOCKER_IMG}" + - name: Run Remote Server Distribution Build + if: ${{ contains(env.DOCKER_IMG, '::') }} + run: | + ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-server-distribution-remote "${DOCKER_IMG}" + + - name: Run Remote Docker Image Build + if: ${{ contains(env.DOCKER_IMG, '::') }} + run: | + ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-docker-image-remote + + - name: Start Remote Remote Deephaven Server + run: | + ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} manage-deephaven-remote start "${DOCKER_IMG}" + - name: Run Remote Benchmark Artifact Build run: | ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-benchmark-artifact-remote