From ca9b70c522054e0354f75de11a6c0dbdc0d4699b Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 25 Sep 2024 20:52:43 +0800 Subject: [PATCH 1/3] Add canary for openjdk regression test The canary will be run alongside the newest revision in regression test to identify unnoticed environment changes. --- configs/running-openjdk-base.yml | 4 ++++ configs/running-openjdk-canary-complete.yml | 6 +++++ scripts/history_report.py | 2 +- scripts/openjdk-history-run.sh | 25 ++++++++++++++------- 4 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 configs/running-openjdk-canary-complete.yml diff --git a/configs/running-openjdk-base.yml b/configs/running-openjdk-base.yml index e834f7c..c072d15 100644 --- a/configs/running-openjdk-base.yml +++ b/configs/running-openjdk-base.yml @@ -97,3 +97,7 @@ runtimes: type: OpenJDK release: 11 home: "./build/jdk-mmtk-branch/images/jdk" + jdk-mmtk-canary: + type: OpenJDK + release: 11 + home: "./build/jdk-mmtk-canary/images/jdk" diff --git a/configs/running-openjdk-canary-complete.yml b/configs/running-openjdk-canary-complete.yml new file mode 100644 index 0000000..e67b6cd --- /dev/null +++ b/configs/running-openjdk-canary-complete.yml @@ -0,0 +1,6 @@ +includes: + - "./running-openjdk-base.yml" + +configs: + # Run GenImmix using the canary version. + - "jdk-mmtk-canary|genimmix|common_mmtk" diff --git a/scripts/history_report.py b/scripts/history_report.py index 20844d4..5839ca4 100644 --- a/scripts/history_report.py +++ b/scripts/history_report.py @@ -26,7 +26,7 @@ prefix = config['name'] -# all subfolders are plan names +# all subfolders are plan names, or "canary" for the canary version plans = os.listdir(result_repo_vm_root) # check from date and to date diff --git a/scripts/openjdk-history-run.sh b/scripts/openjdk-history-run.sh index 1f1e7c3..e66ddff 100755 --- a/scripts/openjdk-history-run.sh +++ b/scripts/openjdk-history-run.sh @@ -3,12 +3,14 @@ set -ex # include common.sh . $(dirname "$0")/common.sh -openjdk_binding=$(realpath $1) -output_dir=$(realpath -m $2) -openjdk_rev=$(git -C $openjdk_binding rev-parse HEAD) +openjdk_binding_latest=$(realpath $1) +openjdk_binding_canary=$(realpath $2) +output_dir=$(realpath -m $3) +# openjdk_rev is used for the commit message. We use the revision ID of the latest version. +openjdk_rev=$(git -C $openjdk_binding_latest rev-parse HEAD) # OpenJDK root -openjdk=$openjdk_binding/repos/openjdk +openjdk=$openjdk_binding_latest/repos/openjdk ensure_empty_dir $kit_build ensure_empty_dir $kit_upload @@ -16,18 +18,20 @@ ensure_empty_dir $log_dir checkout_result_repo # Build -build_openjdk_with_mmtk $openjdk_binding release jdk-mmtk +build_openjdk_with_mmtk $openjdk_binding_latest release jdk-mmtk +build_openjdk_with_mmtk $openjdk_binding_canary release jdk-mmtk-canary run_exp() { - plan=$1 + dir_name=$1 config=$2 heap_modifier=$3 # Run run_id=$(run_benchmarks $log_dir $kit_root/configs/$config $heap_modifier $history_invocations) # Save result - mkdir -p $result_repo_dir/openjdk/$plan - cp -r $log_dir/$run_id $result_repo_dir/openjdk/$plan + RESULT_DIR=$result_repo_dir/openjdk/$dir_name + mkdir -p $RESULT_DIR + cp -r $log_dir/$run_id $RESULT_DIR } # Build @@ -54,6 +58,11 @@ run_exp stickyimmix running-openjdk-stickyimmix-complete.yml 6 # MarkSweep run_exp marksweep running-openjdk-marksweep-complete.yml 6 +# GenImmix using the canary version. +# If the performance of the canary version changed, +# it means there is an environment change that impacts the performance. +run_exp canary running-openjdk-canary-complete.yml 6 + # Commit result commit_result_repo 'OpenJDK Binding: '$openjdk_rev From 3e5e58bd884a8c6065ba18051eac506dc8fbb1c3 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 16 Oct 2024 09:36:03 +0800 Subject: [PATCH 2/3] Add "canary" as a "plan" --- configs/openjdk-plot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configs/openjdk-plot.yml b/configs/openjdk-plot.yml index 77accde..3f81d5a 100644 --- a/configs/openjdk-plot.yml +++ b/configs/openjdk-plot.yml @@ -14,6 +14,11 @@ plans: baseline: ['jdk-g1', 'jdk-zgc'] - plan: marksweep baseline: ['jdk-g1', 'jdk-zgc'] + # "canary" is not a real plan. It is for the canary build. + # We add it here to make history_report.py recognize its result + # and plot it in a separate block in parallel to other real plans in OpenJDK. + - plan: canary + baseline: ['jdk-g1', 'jdk-zgc'] notes: - date: "20230116" time: "0000" From 011a0379e6367f5ac341ae608f3ba40de79efd8a Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 16 Oct 2024 11:06:49 +0800 Subject: [PATCH 3/3] Add a test feature to run only 1 invocation --- scripts/openjdk-history-run.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/openjdk-history-run.sh b/scripts/openjdk-history-run.sh index e66ddff..52f5664 100755 --- a/scripts/openjdk-history-run.sh +++ b/scripts/openjdk-history-run.sh @@ -25,9 +25,15 @@ run_exp() { dir_name=$1 config=$2 heap_modifier=$3 + invocations=$history_invocations + + # Use this when testing the scripts so that tests run faster, albeit producing less accurate results. + if [ "$OPENJDK_HISTORY_RUN_TEST_FAST" = "1" ]; then + invocations=1 + fi # Run - run_id=$(run_benchmarks $log_dir $kit_root/configs/$config $heap_modifier $history_invocations) + run_id=$(run_benchmarks $log_dir $kit_root/configs/$config $heap_modifier $invocations) # Save result RESULT_DIR=$result_repo_dir/openjdk/$dir_name mkdir -p $RESULT_DIR