diff --git a/.github/scripts/make-release-distro.sh b/.github/scripts/make-release-distro.sh
index 73328424..9891802b 100755
--- a/.github/scripts/make-release-distro.sh
+++ b/.github/scripts/make-release-distro.sh
@@ -7,36 +7,43 @@ set -o nounset
 # Copyright (c) 2024-2024 Deephaven Data Labs and Patent Pending
 
 # Create a tar file with the given version using the git project located in the 
-# working directory 
+# working directory. Also, make a release-notes.md files compared between the given
+# release commit and previous version
 
-if [[ $# != 3 ]]; then
-    echo "$0: Missing release version, release commit or previous version/commit argument"
+if [[ $# != 4 ]]; then
+    echo "$0: Missing release version, release commit, previous version, or distro source argument"
     exit 1
 fi
 
 RELEASE_VERSION=$1
 RELEASE_COMMIT=$2
 PREVIOUS_VERSION=$3
+DISTRO_SOURCE=$4
 RELEASE_TAG="v${RELEASE_VERSION}"
 PREVIOUS_TAG="v${PREVIOUS_VERSION}"
 ARTIFACT=deephaven-benchmark-${RELEASE_VERSION}
-DISTRO=target/distro
+DISTRO_DEST=target/distro
 THIS=$(basename "$0")
 RELEASE_NOTES=target/release-notes.md
 
+PREVIOUS_REF=${PREVIOUS_TAG}
+if [[ ${PREVIOUS_VERSION} != *"."*"."* ]]; then
+  PREVIOUS_REF=${PREVIOUS_VERSION}
+fi
+
 # Make the Release Notes File
 echo "**What's Changed**" > ${RELEASE_NOTES}
-git log --oneline ${PREVIOUS_TAG}...${RELEASE_COMMIT} | sed -e 's/^/- /' >> ${RELEASE_NOTES}
+git log --oneline ${PREVIOUS_REF}...${RELEASE_COMMIT} | sed -e 's/^/- /' >> ${RELEASE_NOTES}
 echo "**Full Changelog**: https://github.com/deephaven/benchmark/compare/${PREVIOUS_TAG}...${RELEASE_TAG}" >> ${RELEASE_NOTES}
 
 # Build the Distro for running standard benchmarks
-mkdir -p ${DISTRO}/libs/
-cp .github/distro/* ${DISTRO}
-cp target/dependencies/* ${DISTRO}/libs
-cp target/deephaven-benchmark-1.0-SNAPSHOT.jar ${DISTRO}/libs/${ARTIFACT}.jar
-cp target/deephaven-benchmark-1.0-SNAPSHOT-tests.jar ${DISTRO}/libs/${ARTIFACT}-tests.jar
-echo "VERSION=${RELEASE_VERSION}" > ${DISTRO}/.env
-
-cd ${DISTRO}
+mkdir -p ${DISTRO_DEST}/libs/
+cp ${DISTRO_SOURCE}/* ${DISTRO_DEST}
+cp target/dependencies/* ${DISTRO_DEST}/libs
+cp target/deephaven-benchmark-1.0-SNAPSHOT.jar ${DISTRO_DEST}/libs/${ARTIFACT}.jar
+cp target/deephaven-benchmark-1.0-SNAPSHOT-tests.jar ${DISTRO_DEST}/libs/${ARTIFACT}-tests.jar
+echo "VERSION=${RELEASE_VERSION}" > ${DISTRO_DEST}/.env
+
+cd ${DISTRO_DEST}
 tar cvzf ../${ARTIFACT}.tar * .env
 
diff --git a/.github/workflows/publish-benchmarks.yml b/.github/workflows/publish-benchmarks.yml
index 55ed401c..402f7332 100644
--- a/.github/workflows/publish-benchmarks.yml
+++ b/.github/workflows/publish-benchmarks.yml
@@ -81,7 +81,7 @@ jobs:
       
     - name: Build Release Distro and Notes
       working-directory: ${{env.RELEASE_DIR}}
-      run: ${LATEST_DIR}/.github/scripts/make-release-distro.sh ${VERSION} ${COMMIT} ${PREV_VERSION}
+      run: ${LATEST_DIR}/.github/scripts/make-release-distro.sh ${VERSION} ${COMMIT} ${PREV_VERSION} ${LATEST_DIR}/.github/distro
       
     - name: Test Release Distro
       working-directory: ${{env.RELEASE_DIR}}