diff --git a/bench-common.libsonnet b/bench-common.libsonnet index cf90ed2c82bd..f6760bd80fe2 100644 --- a/bench-common.libsonnet +++ b/bench-common.libsonnet @@ -9,7 +9,9 @@ generated_name:: utils.hyphenize([self.job_prefix, self.suite, self.platform, utils.prefixed_jdk(self.jdk_version), self.os, self.arch, self.job_suffix]), job_prefix:: null, job_suffix:: null, - name: self.generated_name, + name: + if self.is_jdk_supported(self.jdk_version) then self.generated_name + else error "JDK" + self.jdk_version + " is not supported for " + self.generated_name + "! Suite is explicitly marked as working for JDK versions "+ self.min_jdk_version + " until " + self.max_jdk_version, suite:: error "'suite' must be set to generate job name", timelimit: error "build 'timelimit' is not set for "+ self.name +"!", local ol8_image = self.ci_resources.infra.ol8_bench_image, @@ -18,6 +20,12 @@ "mount_modules": true }, should_use_hwloc:: std.objectHasAll(self, "is_numa") && self.is_numa && std.length(std.find("bench", self.targets)) > 0, + min_jdk_version:: null, + max_jdk_version:: null, + is_jdk_supported(jdk_version):: + if self.min_jdk_version != null && jdk_version < self.min_jdk_version then false + else if self.max_jdk_version != null && jdk_version > self.max_jdk_version then false + else true }, bench_hw:: { diff --git a/compiler/ci_common/benchmark-builders.jsonnet b/compiler/ci_common/benchmark-builders.jsonnet index d72e99f60d74..1371ff16b1f8 100644 --- a/compiler/ci_common/benchmark-builders.jsonnet +++ b/compiler/ci_common/benchmark-builders.jsonnet @@ -7,31 +7,39 @@ local jdk8 = c.oraclejdk8, local jdk11 = c.labsjdk11, + local jdk17 = c.labsjdk17, + + local amd64_jdks = [jdk8, jdk11, jdk17], + local aarch64_jdks = [jdk11, jdk17], local main_builds = [ - c.post_merge + hw.x52 + jdk8 + cc.libgraal + bench.dacapo, + c.daily + hw.x52 + jdk8 + cc.libgraal + bench.dacapo, c.daily + hw.x52 + jdk8 + cc.jargraal + bench.dacapo, + c.weekly + hw.x52 + jdk8 + cc.libgraal + bench.dacapo_size_variants, + c.on_demand + hw.x52 + jdk8 + cc.jargraal + bench.dacapo_size_variants, c.weekly + hw.x52 + jdk8 + cc.libgraal + bench.dacapo_timing, c.weekly + hw.x52 + jdk8 + cc.jargraal + bench.dacapo_timing, - c.post_merge + hw.x52 + jdk8 + cc.libgraal + bench.scala_dacapo, + c.daily + hw.x52 + jdk8 + cc.libgraal + bench.scala_dacapo, c.daily + hw.x52 + jdk8 + cc.jargraal + bench.scala_dacapo, + c.weekly + hw.x52 + jdk8 + cc.libgraal + bench.scala_dacapo_size_variants, + c.on_demand + hw.x52 + jdk8 + cc.jargraal + bench.scala_dacapo_size_variants, c.weekly + hw.x52 + jdk8 + cc.libgraal + bench.scala_dacapo_timing, c.weekly + hw.x52 + jdk8 + cc.jargraal + bench.scala_dacapo_timing, c.post_merge + hw.x52 + jdk8 + cc.libgraal + bench.renaissance, c.daily + hw.x52 + jdk8 + cc.jargraal + bench.renaissance, c.daily + hw.x52 + jdk8 + cc.libgraal + bench.specjvm2008, - c.daily + hw.x52 + jdk8 + cc.jargraal + bench.specjvm2008, + c.weekly + hw.x52 + jdk8 + cc.jargraal + bench.specjvm2008, c.daily + hw.x52 + jdk8 + cc.libgraal + bench.specjbb2005, - c.daily + hw.x52 + jdk8 + cc.jargraal + bench.specjbb2005, - c.daily + hw.x52 + jdk8 + cc.libgraal + bench.specjbb2015, - c.daily + hw.x52 + jdk8 + cc.jargraal + bench.specjbb2015, + c.weekly + hw.x52 + jdk8 + cc.jargraal + bench.specjbb2005, + c.weekly + hw.x52 + jdk8 + cc.libgraal + bench.specjbb2015, + c.weekly + hw.x52 + jdk8 + cc.jargraal + bench.specjbb2015, c.weekly + hw.x52 + jdk8 + cc.libgraal + bench.specjbb2015_full_machine, c.on_demand + hw.x52 + jdk8 + cc.jargraal + bench.specjbb2015_full_machine, c.weekly + hw.x52 + jdk8 + cc.libgraal + bench.renaissance_0_10, c.on_demand + hw.x52 + jdk8 + cc.jargraal + bench.renaissance_0_10, c.daily + hw.x52 + jdk8 + cc.libgraal + bench.awfy, c.daily + hw.x52 + jdk8 + cc.jargraal + bench.awfy, - c.post_merge + hw.x52 + jdk8 + cc.libgraal + bench.renaissance_legacy, + c.daily + hw.x52 + jdk8 + cc.libgraal + bench.renaissance_legacy, c.daily + hw.x52 + jdk8 + cc.jargraal + bench.renaissance_legacy, c.daily + hw.x52 + jdk8 + cc.libgraal + bench.micros_graal_whitebox, c.weekly + hw.x52 + jdk8 + cc.jargraal + bench.micros_graal_whitebox, @@ -41,12 +49,17 @@ c.weekly + hw.x52 + jdk8 + cc.jargraal + bench.micros_misc_graal_dist, c.daily + hw.x52 + jdk8 + cc.libgraal + bench.micros_shootout_graal_dist, c.weekly + hw.x52 + jdk8 + cc.jargraal + bench.micros_shootout_graal_dist, - c.daily + hw.x52 + jdk11 + cc.libgraal + bench.dacapo, + + c.post_merge + hw.x52 + jdk11 + cc.libgraal + bench.dacapo, c.daily + hw.x52 + jdk11 + cc.jargraal + bench.dacapo, + c.weekly + hw.x52 + jdk11 + cc.libgraal + bench.dacapo_size_variants, + c.weekly + hw.x52 + jdk11 + cc.jargraal + bench.dacapo_size_variants, c.weekly + hw.x52 + jdk11 + cc.libgraal + bench.dacapo_timing, c.weekly + hw.x52 + jdk11 + cc.jargraal + bench.dacapo_timing, - c.daily + hw.x52 + jdk11 + cc.libgraal + bench.scala_dacapo, + c.post_merge + hw.x52 + jdk11 + cc.libgraal + bench.scala_dacapo, c.daily + hw.x52 + jdk11 + cc.jargraal + bench.scala_dacapo, + c.weekly + hw.x52 + jdk11 + cc.libgraal + bench.scala_dacapo_size_variants, + c.weekly + hw.x52 + jdk11 + cc.jargraal + bench.scala_dacapo_size_variants, c.weekly + hw.x52 + jdk11 + cc.libgraal + bench.scala_dacapo_timing, c.weekly + hw.x52 + jdk11 + cc.jargraal + bench.scala_dacapo_timing, c.post_merge + hw.x52 + jdk11 + cc.libgraal + bench.renaissance, @@ -56,7 +69,7 @@ c.daily + hw.x52 + jdk11 + cc.libgraal + bench.specjbb2005, c.daily + hw.x52 + jdk11 + cc.jargraal + bench.specjbb2005, c.daily + hw.x52 + jdk11 + cc.libgraal + bench.specjbb2015, - c.daily + hw.x52 + jdk11 + cc.jargraal + bench.specjbb2015, + c.weekly + hw.x52 + jdk11 + cc.jargraal + bench.specjbb2015, c.weekly + hw.x52 + jdk11 + cc.libgraal + bench.specjbb2015_full_machine, c.on_demand + hw.x52 + jdk11 + cc.jargraal + bench.specjbb2015_full_machine, c.weekly + hw.x52 + jdk11 + cc.libgraal + bench.renaissance_0_10, @@ -73,6 +86,41 @@ c.weekly + hw.x52 + jdk11 + cc.jargraal + bench.micros_misc_graal_dist, c.daily + hw.x52 + jdk11 + cc.libgraal + bench.micros_shootout_graal_dist, c.weekly + hw.x52 + jdk11 + cc.jargraal + bench.micros_shootout_graal_dist, + + c.daily + hw.x52 + jdk17 + cc.libgraal + bench.dacapo, + c.daily + hw.x52 + jdk17 + cc.jargraal + bench.dacapo, + c.daily + hw.x52 + jdk17 + cc.libgraal + bench.dacapo_size_variants, + c.weekly + hw.x52 + jdk17 + cc.jargraal + bench.dacapo_size_variants, + c.weekly + hw.x52 + jdk17 + cc.libgraal + bench.dacapo_timing, + c.weekly + hw.x52 + jdk17 + cc.jargraal + bench.dacapo_timing, + c.daily + hw.x52 + jdk17 + cc.libgraal + bench.scala_dacapo, + c.daily + hw.x52 + jdk17 + cc.jargraal + bench.scala_dacapo, + c.daily + hw.x52 + jdk17 + cc.libgraal + bench.scala_dacapo_size_variants, + c.weekly + hw.x52 + jdk17 + cc.jargraal + bench.scala_dacapo_size_variants, + c.weekly + hw.x52 + jdk17 + cc.libgraal + bench.scala_dacapo_timing, + c.weekly + hw.x52 + jdk17 + cc.jargraal + bench.scala_dacapo_timing, + #c.post_merge + hw.x52 + jdk17 + cc.libgraal + bench.renaissance, + #c.daily + hw.x52 + jdk17 + cc.jargraal + bench.renaissance, + c.daily + hw.x52 + jdk17 + cc.libgraal + bench.specjvm2008, + c.daily + hw.x52 + jdk17 + cc.jargraal + bench.specjvm2008, + c.daily + hw.x52 + jdk17 + cc.libgraal + bench.specjbb2005, + c.daily + hw.x52 + jdk17 + cc.jargraal + bench.specjbb2005, + c.daily + hw.x52 + jdk17 + cc.libgraal + bench.specjbb2015, + c.weekly + hw.x52 + jdk17 + cc.jargraal + bench.specjbb2015, + c.weekly + hw.x52 + jdk17 + cc.libgraal + bench.specjbb2015_full_machine, + c.on_demand + hw.x52 + jdk17 + cc.jargraal + bench.specjbb2015_full_machine, + c.daily + hw.x52 + jdk17 + cc.libgraal + bench.awfy, + c.daily + hw.x52 + jdk17 + cc.jargraal + bench.awfy, + #c.post_merge + hw.x52 + jdk17 + cc.libgraal + bench.renaissance_legacy, + #c.daily + hw.x52 + jdk17 + cc.jargraal + bench.renaissance_legacy, + c.daily + hw.x52 + jdk17 + cc.libgraal + bench.micros_graal_whitebox, + c.weekly + hw.x52 + jdk17 + cc.jargraal + bench.micros_graal_whitebox, + c.daily + hw.x52 + jdk17 + cc.libgraal + bench.micros_graal_dist, + c.weekly + hw.x52 + jdk17 + cc.jargraal + bench.micros_graal_dist, + c.daily + hw.x52 + jdk17 + cc.libgraal + bench.micros_misc_graal_dist, + c.weekly + hw.x52 + jdk17 + cc.jargraal + bench.micros_misc_graal_dist, + c.daily + hw.x52 + jdk17 + cc.libgraal + bench.micros_shootout_graal_dist, + c.weekly + hw.x52 + jdk17 + cc.jargraal + bench.micros_shootout_graal_dist ], // JFR and async-profiler jobs @@ -81,8 +129,9 @@ c.weekly + hw.x52 + jdk + cc.libgraal + cc.enable_profiling + suite + { job_prefix:: "bench-profiling" }, c.weekly + hw.x52 + jdk + cc.jargraal + cc.enable_profiling + suite + { job_prefix:: "bench-profiling" } ] - for jdk in [jdk8, jdk11] + for jdk in amd64_jdks for suite in bench.groups.profiled_suites + if suite.is_jdk_supported(jdk.jdk_version) ]), // Microservices @@ -91,8 +140,9 @@ c.daily + hw.x52 + jdk + cc.libgraal + suite, c.daily + hw.x52 + jdk + cc.jargraal + suite ] - for jdk in [jdk8, jdk11] + for jdk in amd64_jdks for suite in bench.groups.microservice_suites + if suite.is_jdk_supported(jdk.jdk_version) ]), // intensive weekly benchmarking @@ -101,16 +151,19 @@ cc.generate_fork_builds(c.weekly + hw.x52 + jdk + cc.libgraal + suite), cc.generate_fork_builds(c.weekly + hw.x52 + jdk + cc.jargraal + suite) ]) - for jdk in [jdk8, jdk11] + for jdk in amd64_jdks for suite in bench.groups.weekly_forks_suites + if suite.is_jdk_supported(jdk.jdk_version) ]), local aarch64_builds = std.flattenArrays([ [ - c.weekly + hw.xgene3 + jdk11 + cc.libgraal + suite, - c.weekly + hw.xgene3 + jdk11 + cc.jargraal + suite + c.weekly + hw.xgene3 + jdk + cc.libgraal + suite, + c.weekly + hw.xgene3 + jdk + cc.jargraal + suite ] + for jdk in aarch64_jdks for suite in bench.groups.main_suites + if suite.is_jdk_supported(jdk.jdk_version) ]), local all_builds = main_builds + weekly_forks_builds + profiling_builds + microservice_builds + aarch64_builds, diff --git a/compiler/ci_common/benchmark-suites.libsonnet b/compiler/ci_common/benchmark-suites.libsonnet index a3d530d88e67..27cde8cf4168 100644 --- a/compiler/ci_common/benchmark-suites.libsonnet +++ b/compiler/ci_common/benchmark-suites.libsonnet @@ -8,13 +8,13 @@ groups:: { open_suites:: std.set([$.awfy, $.dacapo, $.scala_dacapo, $.renaissance], keyF=uniq_key), spec_suites:: std.set([$.specjvm2008, $.specjbb2005, $.specjbb2015], keyF=uniq_key), - legacy_suites:: std.set([$.renaissance_legacy], keyF=uniq_key), + legacy_and_secondary_suites:: std.set([$.renaissance_legacy, $.dacapo_size_variants, $.scala_dacapo_size_variants], keyF=uniq_key), jmh_micros_suites:: std.set([$.micros_graal_dist, $.micros_misc_graal_dist , $.micros_shootout_graal_dist], keyF=uniq_key), graal_internals_suites:: std.set([$.micros_graal_whitebox], keyF=uniq_key), special_suites:: std.set([$.renaissance_0_10, $.specjbb2015_full_machine], keyF=uniq_key), microservice_suites:: std.set([$.microservice_benchmarks], keyF=uniq_key), - main_suites:: std.set(self.open_suites + self.spec_suites + self.legacy_suites, keyF=uniq_key), + main_suites:: std.set(self.open_suites + self.spec_suites + self.legacy_and_secondary_suites, keyF=uniq_key), all_suites:: std.set(self.main_suites + self.jmh_micros_suites + self.special_suites + self.microservice_suites, keyF=uniq_key), weekly_forks_suites:: self.main_suites, @@ -30,7 +30,9 @@ ], timelimit: "30:00", forks_batches:: null, // disables it for now (GR-30956) - forks_timelimit:: "3:00:00" + forks_timelimit:: "3:00:00", + min_jdk_version:: 8, + max_jdk_version:: null }, dacapo: cc.compiler_benchmark + c.heap.default + { @@ -40,7 +42,25 @@ ], timelimit: "45:00", forks_batches:: 1, - forks_timelimit:: "02:45:00" + forks_timelimit:: "02:45:00", + min_jdk_version:: 8, + max_jdk_version:: null + }, + + dacapo_size_variants: cc.compiler_benchmark + c.heap.default + { + suite:: "dacapo-size-variants", + run+: [ + self.benchmark_cmd + ["dacapo-small:*", "--"] + self.extra_vm_args, + self._bench_upload(), + self.benchmark_cmd + ["dacapo-large:*", "--"] + self.extra_vm_args, + self._bench_upload(), + self.benchmark_cmd + ["dacapo-huge:*", "--"] + self.extra_vm_args + ], + timelimit: "05:30:00", + forks_batches:: null, # weekly forks disabled + forks_timelimit:: null, + min_jdk_version:: 8, + max_jdk_version:: null }, dacapo_timing: cc.compiler_benchmark + c.heap.default + { @@ -48,7 +68,9 @@ run+: [ self.benchmark_cmd + ["dacapo-timing:*", "--"] + self.extra_vm_args ], - timelimit: "45:00" + timelimit: "45:00", + min_jdk_version:: 8, + max_jdk_version:: null }, scala_dacapo: cc.compiler_benchmark + c.heap.default + { @@ -58,7 +80,29 @@ ], timelimit: "45:00", forks_batches:: 1, - forks_timelimit:: "03:30:00" + forks_timelimit:: "03:30:00", + min_jdk_version:: 8, + max_jdk_version:: null + }, + + scala_dacapo_size_variants: cc.compiler_benchmark + c.heap.default + { + suite:: "scala-dacapo-size-variants", + run+: [ + self.benchmark_cmd + ["scala-dacapo-tiny:*", "--"] + self.extra_vm_args, + self._bench_upload(), + self.benchmark_cmd + ["scala-dacapo-small:*", "--"] + self.extra_vm_args, + self._bench_upload(), + self.benchmark_cmd + ["scala-dacapo-large:*", "--"] + self.extra_vm_args, + self._bench_upload(), + self.benchmark_cmd + ["scala-dacapo-huge:*", "--"] + self.extra_vm_args, + self._bench_upload(), + self.benchmark_cmd + ["scala-dacapo-gargantuan:*", "--"] + self.extra_vm_args + ], + timelimit: "05:00:00", + forks_batches:: null, # weekly forks disabled + forks_timelimit:: null, + min_jdk_version:: 8, + max_jdk_version:: null }, scala_dacapo_timing: cc.compiler_benchmark + c.heap.default + { @@ -66,7 +110,9 @@ run+: [ self.benchmark_cmd + ["scala-dacapo-timing:*", "--"] + self.extra_vm_args ], - timelimit: "45:00" + timelimit: "45:00", + min_jdk_version:: 8, + max_jdk_version:: null }, renaissance: cc.compiler_benchmark + c.heap.default + { @@ -82,14 +128,18 @@ ], timelimit: "3:00:00", forks_batches:: 4, - forks_timelimit:: "06:30:00" + forks_timelimit:: "06:30:00", + min_jdk_version:: 8, + max_jdk_version:: 11 }, renaissance_0_10: self.renaissance + { suite:: "renaissance-0-10", environment+: { "RENAISSANCE_VERSION": "0.10.0" - } + }, + min_jdk_version:: 8, + max_jdk_version:: 11 }, renaissance_legacy: cc.compiler_benchmark + c.heap.default + { @@ -105,7 +155,9 @@ ], timelimit: "2:45:00", forks_batches:: 4, - forks_timelimit:: "06:30:00" + forks_timelimit:: "06:30:00", + min_jdk_version:: 8, + max_jdk_version:: 11 }, specjbb2005: cc.compiler_benchmark + c.heap.large_with_large_young_gen + { @@ -118,7 +170,9 @@ ], timelimit: "4:00:00", forks_batches:: 1, - forks_timelimit:: "20:00:00" + forks_timelimit:: "20:00:00", + min_jdk_version:: 8, + max_jdk_version:: null }, specjbb2015: cc.compiler_benchmark + c.heap.large_with_large_young_gen + { @@ -131,7 +185,9 @@ ], timelimit: "3:00:00", forks_batches:: 1, - forks_timelimit:: "20:00:00" + forks_timelimit:: "20:00:00", + min_jdk_version:: 8, + max_jdk_version:: null }, specjbb2015_full_machine: cc.compiler_benchmark + c.heap.large_with_large_young_gen + { @@ -142,10 +198,12 @@ run+: [ self.plain_benchmark_cmd + ["specjbb2015", "--"] + self.extra_vm_args ], - timelimit: "3:00:00" + timelimit: "3:00:00", + min_jdk_version:: 8, + max_jdk_version:: null }, - specjvm2008: cc.compiler_benchmark + c.heap.large + { + specjvm2008: cc.compiler_benchmark + c.heap.default + { suite:: "specjvm2008", downloads+: { "SPECJVM2008": { name: "specjvm2008", version: "1.01" } @@ -158,7 +216,9 @@ ], timelimit: "3:00:00", forks_batches:: 5, - forks_timelimit:: "06:00:00" + forks_timelimit:: "06:00:00", + min_jdk_version:: 8, + max_jdk_version:: null }, // Microservice benchmarks @@ -224,7 +284,9 @@ self.benchmark_cmd + ["spring-helloworld-wrk:helloworld"] + hwlocBind_1C_1T + ["--"] + self.extra_vm_args + ["-Xms8m", "-Xmx64m", "-XX:ActiveProcessorCount=1", "-XX:MaxDirectMemorySize=256m"], bench_upload ], - timelimit: "7:00:00" + timelimit: "7:00:00", + min_jdk_version:: 11, # GR-32793: disabled JDK8 + max_jdk_version:: null }, // JMH microbenchmarks @@ -233,7 +295,9 @@ run+: [ self.benchmark_cmd + ["jmh-whitebox:*", "--"] + self.extra_vm_args ], - timelimit: "3:00:00" + timelimit: "3:00:00", + min_jdk_version:: 8, + max_jdk_version:: null }, micros_graal_dist: cc.compiler_benchmark + c.heap.default + { @@ -241,7 +305,9 @@ run+: [ self.benchmark_cmd + ["jmh-dist:GRAAL_COMPILER_MICRO_BENCHMARKS", "--"] + self.extra_vm_args ], - timelimit: "3:00:00" + timelimit: "3:00:00", + min_jdk_version:: 8, + max_jdk_version:: null }, micros_misc_graal_dist: cc.compiler_benchmark + c.heap.default + { @@ -249,7 +315,9 @@ run+: [ self.benchmark_cmd + ["jmh-dist:GRAAL_BENCH_MISC", "--"] + self.extra_vm_args ], - timelimit: "3:00:00" + timelimit: "3:00:00", + min_jdk_version:: 8, + max_jdk_version:: null }, micros_shootout_graal_dist: cc.compiler_benchmark + c.heap.default { @@ -257,6 +325,8 @@ run+: [ self.benchmark_cmd + ["jmh-dist:GRAAL_BENCH_SHOOTOUT", "--"] + self.extra_vm_args ], - timelimit: "3:00:00" + timelimit: "3:00:00", + min_jdk_version:: 8, + max_jdk_version:: null } } diff --git a/compiler/ci_common/compiler-common.libsonnet b/compiler/ci_common/compiler-common.libsonnet index a1dd2502fea1..1a3d73cff09d 100644 --- a/compiler/ci_common/compiler-common.libsonnet +++ b/compiler/ci_common/compiler-common.libsonnet @@ -46,8 +46,9 @@ }, compiler_benchmark:: self.compiler_bench_base + self.compiler_benchmarks_notifications + { + _bench_upload(filename="${BENCH_RESULTS_FILE_PATH}"):: ["bench-uploader.py", filename], teardown+: [ - ["bench-uploader.py", "${BENCH_RESULTS_FILE_PATH}"] + self._bench_upload() ] }, diff --git a/compiler/ci_common/gate_tasks.hocon b/compiler/ci_common/gate_tasks.hocon index a9b90af55a69..24c3d028996c 100644 --- a/compiler/ci_common/gate_tasks.hocon +++ b/compiler/ci_common/gate_tasks.hocon @@ -28,6 +28,8 @@ builds += [ ${gateTestCTWWeekly} ${labsjdk-ee-11} ${gateLinuxAMD64} {name: "weekly-test-compiler-ctw-labsjdk-ee-11-linux-amd64"} ${graalWeekly} ${gateTest} ${oraclejdk8Debug} ${gateLinuxAMD64} {name: "weekly-test-compiler-test-8-linux-amd64-fastdebug"} ${graalWeekly} {timelimit: "3:00:00"} ${gateTestBenchmark} ${oraclejdk8} ${gateLinuxAMD64} {name: "gate-compiler-benchmarktest-8-linux-amd64"} + ${gateTestBenchmark} ${labsjdk-ee-11} ${gateLinuxAMD64} {name: "gate-compiler-benchmarktest-11-linux-amd64"} + ${gateTestBenchmark} ${labsjdk-ee-17} ${gateLinuxAMD64} {name: "gate-compiler-benchmarktest-17-linux-amd64"} ${gateTestBenchmark} ${oraclejdk8Debug} ${gateLinuxAMD64} {name: "weekly-test-compiler-benchmarktest-8-linux-amd64-fastdebug"} ${graalWeekly} {timelimit: "1:00:00"} ${gateStyle} ${oraclejdk8} ${gateLinuxAMD64} {name: "gate-compiler-style-linux-amd64", timelimit: "45:00"} ${gateCoverage} ${oraclejdk8} ${gateLinuxAMD64} {name: "weekly-compiler-coverage-8-linux-amd64"} ${graalWeekly} {timelimit: "1:50:00"} diff --git a/compiler/ci_includes/baseline-benchmarks.jsonnet b/compiler/ci_includes/baseline-benchmarks.jsonnet index f78b793e7a2d..857c556c9ba5 100644 --- a/compiler/ci_includes/baseline-benchmarks.jsonnet +++ b/compiler/ci_includes/baseline-benchmarks.jsonnet @@ -5,13 +5,18 @@ local bench = (import '../ci_common/benchmark-suites.libsonnet'), local hw = bc.bench_hw, - local jdk8 = c.oraclejdk8, + local jdk8 = c.oraclejdk8, local jdk11 = c.labsjdk11, + local jdk17 = c.labsjdk17, + + local amd64_jdks = [jdk8, jdk11, jdk17], + local aarch64_jdks = [jdk11, jdk17], local hotspot_main_builds = [ - c.on_demand + hw.x52 + jdk + cc.c2 + suite - for jdk in [jdk8, jdk11] + c.weekly + hw.x52 + jdk + cc.c2 + suite + for jdk in amd64_jdks for suite in bench.groups.all_suites + if suite.is_jdk_supported(jdk.jdk_version) ], @@ -19,20 +24,26 @@ [ c.weekly + hw.x52 + jdk + cc.c2 + cc.enable_profiling + suite + { job_prefix:: "bench-profiling" } ] - for jdk in [jdk8, jdk11] + for jdk in amd64_jdks for suite in bench.groups.profiled_suites + if suite.is_jdk_supported(jdk.jdk_version) ]), local weekly_forks_builds = std.flattenArrays([ cc.generate_fork_builds(c.weekly + hw.x52 + jdk + cc.c2 + suite) - for jdk in [jdk8, jdk11] + for jdk in amd64_jdks for suite in bench.groups.weekly_forks_suites + if suite.is_jdk_supported(jdk.jdk_version) ]), - local aarch64_builds = [ - c.weekly + hw.xgene3 + jdk11 + cc.c2 + suite + local aarch64_builds = std.flattenArrays([ + [ + c.weekly + hw.xgene3 + jdk + cc.c2 + suite + ] + for jdk in aarch64_jdks for suite in bench.groups.main_suites - ], + if suite.is_jdk_supported(jdk.jdk_version) + ]), local all_builds = hotspot_main_builds + hotspot_profiling_builds + weekly_forks_builds + aarch64_builds, // adds a "defined_in" field to all builds mentioning the location of this current file diff --git a/compiler/mx.compiler/mx_compiler.py b/compiler/mx.compiler/mx_compiler.py index 5feed69d9ddc..46f748a8afe9 100644 --- a/compiler/mx.compiler/mx_compiler.py +++ b/compiler/mx.compiler/mx_compiler.py @@ -58,7 +58,8 @@ from mx_sdk_vm import jlink_new_jdk import mx_sdk_vm_impl -import mx_graal_benchmark # pylint: disable=unused-import +import mx_benchmark +import mx_graal_benchmark import mx_graal_tools #pylint: disable=unused-import import argparse @@ -361,29 +362,6 @@ def _remove_empty_entries(a): return [] return [x for x in a if x] -def _gate_java_benchmark(args, successRe, cwd=None): - """ - Runs a Java benchmark and aborts if the benchmark process exits with a non-zero - exit code or the `successRe` pattern is not in the output of the benchmark process. - - :param list args: the arguments to pass to the VM - :param str successRe: a regular expression - """ - out = mx.OutputCapture() - try: - run_java(args, out=mx.TeeOutputCapture(out), err=subprocess.STDOUT, cwd=cwd) - finally: - jvmErrorFile = re.search(r'(([A-Z]:|/).*[/\\]hs_err_pid[0-9]+\.log)', out.data) - if jvmErrorFile: - jvmErrorFile = jvmErrorFile.group() - mx.log('Dumping ' + jvmErrorFile) - with open(jvmErrorFile) as fp: - mx.log(fp.read()) - os.unlink(jvmErrorFile) - - if not re.search(successRe, out.data, re.MULTILINE): - mx.abort('Could not find benchmark success pattern: ' + successRe) - def _is_batik_supported(jdk): """ Determines if Batik runs on the given jdk. Batik's JPEGRegistryEntry contains a reference @@ -397,68 +375,29 @@ def _is_batik_supported(jdk): mx.warn('Batik uses Sun internal class com.sun.image.codec.jpeg.TruncatedFileException which is not present in ' + jdk.home) return False -class DaCapoWrapper(object): - """ - A context manager that dumps the stdout and stderr logs of DaCapo and ScalaDacapo - executions return with a non-zero exit code. - """ - - def __init__(self, scratch_dir): - """ - :param str scratch_dir: the directory in which the [Scala]DaCapo logs will be written - """ - self.scratch_dir = scratch_dir - - def __enter__(self): - self._logs = self._gather_logs() - return self - - def __exit__(self, exc_type, exc_value, traceback): - if exc_value is not None: - for before, after in zip(self._logs, self._gather_logs()): - if after.isNewerThan(before): - mx.log('Dumping ' + after.path) - mx.log('```') - with open(after.path) as fp: - mx.log(fp.read()) - mx.log('```') - - def _gather_logs(self): - return [mx.TimeStampFile(join(self.scratch_dir, name + '.log')) for name in ('stdout', 'stderr')] - def _gate_dacapo(name, iterations, extraVMarguments=None, force_serial_gc=True, set_start_heap_size=True, threads=None): + if iterations == -1: + return vmargs = ['-XX:+UseSerialGC'] if force_serial_gc else [] if set_start_heap_size: vmargs += ['-Xms2g'] vmargs += ['-XX:-UseCompressedOops', '-Djava.net.preferIPv4Stack=true', '-Dgraal.CompilationFailureAction=ExitVM'] + _remove_empty_entries(extraVMarguments) - dacapoJar = mx.library('DACAPO').get_path(True) - if name == 'batik' and not _is_batik_supported(jdk): - return - scratch_dir = join(os.getcwd(), 'scratch') - args = ['-n', str(iterations), '--preserve', '--scratch-directory', scratch_dir] + args = ['-n', str(iterations), '--preserve'] if threads is not None: args += ['-t', str(threads)] - with DaCapoWrapper(scratch_dir): - _gate_java_benchmark(vmargs + ['-jar', dacapoJar, name] + args, r'^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====') + return mx_benchmark.gate_mx_benchmark(["dacapo:{}".format(name), "--tracker=none", "--"] + vmargs + ["--"] + args) def jdk_includes_corba(jdk): # corba has been removed since JDK11 (http://openjdk.java.net/jeps/320) return jdk.javaCompliance < '11' def _gate_scala_dacapo(name, iterations, extraVMarguments=None): + if iterations == -1: + return vmargs = ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-Dgraal.CompilationFailureAction=ExitVM'] + _remove_empty_entries(extraVMarguments) - if name == 'actors' and jdk.javaCompliance >= '9' and jdk_includes_corba(jdk): - vmargs += ['--add-modules', 'java.corba'] - scalaDacapoJar = mx.library('DACAPO_SCALA').get_path(True) - scratch_dir = join(os.getcwd(), 'scratch') - with DaCapoWrapper(scratch_dir): - args = ['-jar', scalaDacapoJar, name, '-n', str(iterations), '--preserve', '--scratch-directory', scratch_dir] - cwd = None - if name == 'scaladoc': - # GR-32428: scaladoc scans $PWD for package.class files which can cause problems - # so run it in a directory most likely not containing extraneous package.class files. - cwd = dirname(scalaDacapoJar) - _gate_java_benchmark(vmargs + args, r'^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====', cwd=cwd) + + args = ['-n', str(iterations), '--preserve'] + return mx_benchmark.gate_mx_benchmark(["scala-dacapo:{}".format(name), "--tracker=none", "--"] + vmargs + ["--"] + args) def compiler_gate_runner(suites, unit_test_runs, bootstrap_tests, tasks, extraVMarguments=None, extraUnitTestArguments=None): if jdk.javaCompliance >= '9': @@ -516,63 +455,48 @@ def compiler_gate_runner(suites, unit_test_runs, bootstrap_tests, tasks, extraVM def compiler_gate_benchmark_runner(tasks, extraVMarguments=None, prefix=''): - # run selected DaCapo benchmarks + # run DaCapo benchmarks # + ######################### # DaCapo benchmarks that can run with system assertions enabled but # java.util.Logging assertions disabled because the the DaCapo harness - # misuses the API. - dacapos = { - 'avrora': 1, - 'h2': 1, - 'jython': 2, - 'luindex': 1, - 'lusearch': 4, - 'xalan': 1, - 'batik': 1, - 'fop': 8, - 'pmd': 1, - 'sunflow': 2, + # misuses the API. The same harness is used by Scala DaCapo. + dacapo_esa = ['-esa', '-da:java.util.logging...'] + + # a few iterations to get the chance to catch miscompilations in the gate + default_iterations = 1 + + dacapo_suite = mx_graal_benchmark.DaCapoBenchmarkSuite() + dacapo_gate_iterations = { + k: default_iterations for k, v in dacapo_suite.daCapoIterations().items() if v > 0 } - for name, iterations in sorted(dacapos.items()): + dacapo_gate_iterations.update({'fop': 8}) + mx.warn("Disabling gate for dacapo:tradebeans and dacapo:tradesoap because of AssertionError (GR-33115)") + dacapo_gate_iterations.update({'tradebeans': -1, 'tradesoap': -1}) + for name, iterations in sorted(dacapo_gate_iterations.items()): + if name == "batik" and not _is_batik_supported(jdk): + continue with Task(prefix + 'DaCapo:' + name, tasks, tags=GraalTags.benchmarktest) as t: if t: _gate_dacapo(name, iterations, _remove_empty_entries(extraVMarguments) + - ['-XX:+UseJVMCICompiler', '-Dgraal.TrackNodeSourcePosition=true', '-esa', '-da:java.util.logging...']) + ['-Dgraal.TrackNodeSourcePosition=true'] + dacapo_esa) - # run selected Scala DaCapo benchmarks - # Scala DaCapo benchmarks that can run with system assertions enabled but - # java.util.Logging assertions disabled because the the DaCapo harness - # misuses the API. - scala_dacapos = { - 'apparat': 1, - 'factorie': 1, - 'kiama': 4, - 'scalac': 1, - 'scaladoc': 1, - 'scalap': 1, - 'scalariform':1, - 'scalatest': 1, - 'scalaxb': 1, - 'specs': 1, - 'tmt': 1, - 'actors': 1, + # run Scala DaCapo benchmarks # + ############################### + scala_dacapo_suite = mx_graal_benchmark.ScalaDaCapoBenchmarkSuite() + scala_dacapo_gate_iterations = { + k: default_iterations for k, v in scala_dacapo_suite.daCapoIterations().items() if v > 0 } - if not jdk_includes_corba(jdk): - mx.warn('Removing scaladacapo:actors from benchmarks because corba has been removed since JDK11 (http://openjdk.java.net/jeps/320)') - del scala_dacapos['actors'] - - if jdk.javaCompliance >= "16": - # See GR-29222 for details. - mx.warn('Removing scaladacapo:specs from benchmarks because it uses a library that violates module permissions which is no longer allowed in JDK 16 (JDK-8255363)') - del scala_dacapos['specs'] - for name, iterations in sorted(scala_dacapos.items()): + for name, iterations in sorted(scala_dacapo_gate_iterations.items()): with Task(prefix + 'ScalaDaCapo:' + name, tasks, tags=GraalTags.benchmarktest) as t: if t: _gate_scala_dacapo(name, iterations, _remove_empty_entries(extraVMarguments) + - ['-XX:+UseJVMCICompiler', '-Dgraal.TrackNodeSourcePosition=true', '-esa', '-da:java.util.logging...']) + ['-Dgraal.TrackNodeSourcePosition=true'] + dacapo_esa) + # run benchmark with non default setup # + ######################################## # ensure -Xbatch still works with Task(prefix + 'DaCapo_pmd:BatchMode', tasks, tags=GraalTags.test) as t: - if t: _gate_dacapo('pmd', 1, _remove_empty_entries(extraVMarguments) + ['-XX:+UseJVMCICompiler', '-Xbatch']) + if t: _gate_dacapo('pmd', 1, _remove_empty_entries(extraVMarguments) + ['-Xbatch']) # ensure benchmark counters still work if mx.get_arch() != 'aarch64': # GR-8364 Exclude benchmark counters on AArch64 @@ -581,7 +505,7 @@ def compiler_gate_benchmark_runner(tasks, extraVMarguments=None, prefix=''): fd, logFile = tempfile.mkstemp() os.close(fd) # Don't leak file descriptors try: - _gate_dacapo('pmd', 3, _remove_empty_entries(extraVMarguments) + ['-Dgraal.LogFile=' + logFile, '-XX:+UseJVMCICompiler', '-Dgraal.LIRProfileMoves=true', '-Dgraal.GenericDynamicCounters=true', '-Dgraal.TimedDynamicCounters=1000', '-XX:JVMCICounterSize=10']) + _gate_dacapo('pmd', default_iterations, _remove_empty_entries(extraVMarguments) + ['-Dgraal.LogFile=' + logFile, '-Dgraal.LIRProfileMoves=true', '-Dgraal.GenericDynamicCounters=true', '-Dgraal.TimedDynamicCounters=1000', '-XX:JVMCICounterSize=10']) with open(logFile) as fp: haystack = fp.read() needle = 'MoveOperations (dynamic counters)' @@ -596,24 +520,24 @@ def compiler_gate_benchmark_runner(tasks, extraVMarguments=None, prefix=''): # ensure -XX:+PreserveFramePointer still works with Task(prefix + 'DaCapo_pmd:PreserveFramePointer', tasks, tags=GraalTags.test) as t: - if t: _gate_dacapo('pmd', 4, _remove_empty_entries(extraVMarguments) + ['-XX:+UseJVMCICompiler', '-Xmx256M', '-XX:+PreserveFramePointer'], threads=4, force_serial_gc=False, set_start_heap_size=False) + if t: _gate_dacapo('pmd', default_iterations, _remove_empty_entries(extraVMarguments) + ['-Xmx256M', '-XX:+PreserveFramePointer'], threads=4, force_serial_gc=False, set_start_heap_size=False) if isJDK8: # temporarily isolate those test (GR-10990) cms = ['cms'] # ensure CMS still works with Task(prefix + 'DaCapo_pmd:CMS', tasks, tags=cms) as t: - if t: _gate_dacapo('pmd', 4, _remove_empty_entries(extraVMarguments) + ['-XX:+UseJVMCICompiler', '-Xmx256M', '-XX:+UseConcMarkSweepGC'], threads=4, force_serial_gc=False, set_start_heap_size=False) + if t: _gate_dacapo('pmd', default_iterations, _remove_empty_entries(extraVMarguments) + ['-Xmx256M', '-XX:+UseConcMarkSweepGC'], threads=4, force_serial_gc=False, set_start_heap_size=False) # ensure CMSIncrementalMode still works with Task(prefix + 'DaCapo_pmd:CMSIncrementalMode', tasks, tags=cms) as t: - if t: _gate_dacapo('pmd', 4, _remove_empty_entries(extraVMarguments) + ['-XX:+UseJVMCICompiler', '-Xmx256M', '-XX:+UseConcMarkSweepGC', '-XX:+CMSIncrementalMode'], threads=4, force_serial_gc=False, set_start_heap_size=False) + if t: _gate_dacapo('pmd', default_iterations, _remove_empty_entries(extraVMarguments) + ['-Xmx256M', '-XX:+UseConcMarkSweepGC', '-XX:+CMSIncrementalMode'], threads=4, force_serial_gc=False, set_start_heap_size=False) if prefix != '': # ensure G1 still works with libgraal with Task(prefix + 'DaCapo_pmd:G1', tasks, tags=cms) as t: - if t: _gate_dacapo('pmd', 4, _remove_empty_entries(extraVMarguments) + ['-XX:+UseJVMCICompiler', '-Xmx256M', '-XX:+UseG1GC'], threads=4, force_serial_gc=False, set_start_heap_size=False) + if t: _gate_dacapo('pmd', default_iterations, _remove_empty_entries(extraVMarguments) + ['-Xmx256M', '-XX:+UseG1GC'], threads=4, force_serial_gc=False, set_start_heap_size=False) diff --git a/compiler/mx.compiler/suite.py b/compiler/mx.compiler/suite.py index f57d2784e224..31d75f79542e 100644 --- a/compiler/mx.compiler/suite.py +++ b/compiler/mx.compiler/suite.py @@ -1,5 +1,5 @@ suite = { - "mxversion" : "5.302.1", + "mxversion" : "5.309.2", "name" : "compiler", "sourceinprojectwhitelist" : [], diff --git a/java-benchmarks/mx.java-benchmarks/mx_java_benchmarks.py b/java-benchmarks/mx.java-benchmarks/mx_java_benchmarks.py index 4d21c4fad51d..3b5918f11e80 100644 --- a/java-benchmarks/mx.java-benchmarks/mx_java_benchmarks.py +++ b/java-benchmarks/mx.java-benchmarks/mx_java_benchmarks.py @@ -547,6 +547,9 @@ def group(self): def subgroup(self): return "graal-compiler" + def name(self): + raise NotImplementedError() + def daCapoClasspathEnvVarName(self): raise NotImplementedError() @@ -565,6 +568,15 @@ def daCapoPath(self): def daCapoIterations(self): raise NotImplementedError() + def daCapoSizes(self): + raise NotImplementedError() + + def existingSizes(self): + return list(dict.fromkeys([s for bench, sizes in self.daCapoSizes().items() for s in sizes])) + + def workloadSize(self): + raise NotImplementedError() + def validateEnvironment(self): if not self.daCapoPath(): raise RuntimeError( @@ -577,10 +589,23 @@ def validateReturnCode(self, retcode): def postprocessRunArgs(self, benchname, runArgs): parser = argparse.ArgumentParser(add_help=False) parser.add_argument("-n", default=None) + parser.add_argument("-s", "--size", default=None) args, remaining = parser.parse_known_args(runArgs) + + if args.size: + if args.size not in self.existingSizes(): + mx.abort("Unknown workload size '{}'. " + "Existing benchmark sizes are: {}".format(args.size, ','.join(self.existingSizes()))) + + if args.size != self.workloadSize(): + mx.abort("Mismatch between suite-defined workload size ('{}') " + "and user-provided one ('{}')!".format(self.workloadSize(), args.size)) + + otherArgs = ["-s", self.workloadSize()] + remaining + if args.n: if args.n.isdigit(): - return ["-n", args.n] + remaining + return ["-n", args.n] + otherArgs if args.n == "-1": return None else: @@ -589,7 +614,7 @@ def postprocessRunArgs(self, benchname, runArgs): return None else: iterations = iterations + self.getExtraIterationCount(iterations) - return ["-n", str(iterations)] + remaining + return ["-n", str(iterations)] + otherArgs def createCommandLineArgs(self, benchmarks, bmSuiteArgs): if benchmarks is None: @@ -648,11 +673,11 @@ def repairDatapoints(self, benchmarks, bmSuiteArgs, partialResults): partialResults.append(datapoint) def benchmarkList(self, bmSuiteArgs): - bench_list = [key for key, value in self.daCapoIterations().items() if value != -1] - if java_home_jdk().javaCompliance >= '9' and "batik" in bench_list: - # batik crashes on JDK9+. This is fixed in the upcoming dacapo chopin release - bench_list.remove("batik") - return bench_list + missing_sizes = set(self.daCapoIterations().keys()).difference(set(self.daCapoSizes().keys())) + if len(missing_sizes) > 0: + mx.abort("Missing size definitions for benchmark(s): {}".format(missing_sizes)) + return [b for b, it in self.daCapoIterations().items() + if self.workloadSize() in self.daCapoSizes().get(b, []) and it != -1] def daCapoSuiteTitle(self): """Title string used in the output next to the performance result.""" @@ -744,43 +769,111 @@ def run(self, benchmarks, bmSuiteArgs): _daCapoIterations = { - "avrora" : 20, - "batik" : 40, - "eclipse" : -1, - "fop" : 40, - "h2" : 25, - "jython" : 40, - "luindex" : 15, - "lusearch" : 40, - "pmd" : 30, - "sunflow" : 35, - "tomcat" : -1, # Stopped working as of 8u92 - "tradebeans" : -1, - "tradesoap" : -1, - "xalan" : 30, + "avrora" : 20, + "batik" : 40, + "eclipse" : -1, + "fop" : 40, + "h2" : 25, + "jython" : 50, + "luindex" : 20, + "lusearch" : 40, + "lusearch-fix": -1, + "pmd" : 30, + "sunflow" : 35, + "tomcat" : -1, + "tradebeans" : 20, + "tradesoap" : 20, + "xalan" : 30, +} + + +_daCapoSizes = { + "avrora": ["default", "small", "large"], + "batik": ["default", "small", "large"], + "eclipse": ["default", "small", "large"], + "fop": ["default", "small"], + "h2": ["default", "small", "large", "huge"], + "jython": ["default", "small", "large"], + "luindex": ["default", "small"], + "lusearch": ["default", "small", "large"], + "lusearch-fix": ["default", "small", "large"], + "pmd": ["default", "small", "large"], + "sunflow": ["default", "small", "large"], + "tomcat": ["default", "small", "large", "huge"], + "tradebeans": ["default", "small", "large", "huge"], + "tradesoap": ["default", "small", "large", "huge"], + "xalan": ["default", "small", "large"] } class DaCapoBenchmarkSuite(BaseDaCapoBenchmarkSuite): #pylint: disable=too-many-ancestors - """DaCapo 9.12 (Bach) benchmark suite implementation.""" + """DaCapo benchmark suite implementation.""" def name(self): - return "dacapo" + if self.workloadSize() == "default": + return "dacapo" + else: + return "dacapo-{}".format(self.workloadSize()) def version(self): - return '9.12-bach' + return super(DaCapoBenchmarkSuite, self).version() + + def defaultSuiteVersion(self): + return "9.12-MR1-bach" + + def availableSuiteVersions(self): + return ["9.12-bach", "9.12-MR1-bach"] + + def workloadSize(self): + return "default" def daCapoSuiteTitle(self): - return "DaCapo 9.12" + title = None + if self.version() == "9.12-bach": + title = "DaCapo 9.12" + elif self.version() == "9.12-MR1-bach": + title = "DaCapo 9.12-MR1" + return title def daCapoClasspathEnvVarName(self): return "DACAPO_CP" def daCapoLibraryName(self): - return "DACAPO" + if self.version() == "9.12-bach": # 2009 release + return "DACAPO" + elif self.version() == "9.12-MR1-bach": # 2018 maintenance release + return "DACAPO_MR1_BACH" + else: + return None def daCapoIterations(self): - return _daCapoIterations + iterations = _daCapoIterations.copy() + if self.version() == "9.12-bach": + del iterations["eclipse"] + del iterations["tradebeans"] + del iterations["tradesoap"] + del iterations["lusearch-fix"] + # Stopped working as of 8u92 on the initial release + del iterations["tomcat"] + + if mx.get_jdk().javaCompliance >= '9' and self.version() in ["9.12-bach", "9.12-MR1-bach"]: + if "batik" in iterations: + # batik crashes on JDK9+. This is fixed in the dacapo chopin release only + del iterations["batik"] + if "tradesoap" in iterations: + # validation fails transiently but frequently in the first iteration in JDK9+ + del iterations["tradesoap"] + + if self.workloadSize() == "small": + # Ensure sufficient warmup by doubling the number of default iterations for the small configuration + iterations = {k: (2 * int(v)) if v != -1 else v for k, v in iterations.items()} + if self.workloadSize() in {"huge", "gargantuan"}: + # Reduce the default number of iterations for very large workloads to keep the runtime reasonable + iterations = {k: max(int((int(v)/2)), 5) if v != -1 else v for k, v in iterations.items()} + return iterations + + def daCapoSizes(self): + return _daCapoSizes def flakySuccessPatterns(self): return [ @@ -792,8 +885,38 @@ def flakySuccessPatterns(self): re.MULTILINE), ] + def vmArgs(self, bmSuiteArgs): + vmArgs = super(DaCapoBenchmarkSuite, self).vmArgs(bmSuiteArgs) + if java_home_jdk().javaCompliance >= '16': + vmArgs += ["--add-opens", "java.base/java.lang=ALL-UNNAMED"] + return vmArgs + + +class DacapoSmallBenchmarkSuite(DaCapoBenchmarkSuite): + """The subset of DaCapo benchmarks supporting the 'small' configuration.""" + + def workloadSize(self): + return "small" + + +class DacapoLargeBenchmarkSuite(DaCapoBenchmarkSuite): + """The subset of DaCapo benchmarks supporting the 'large' configuration.""" + + def workloadSize(self): + return "large" + + +class DacapoHugeBenchmarkSuite(DaCapoBenchmarkSuite): + """The subset of DaCapo benchmarks supporting the 'huge' configuration.""" + + def workloadSize(self): + return "huge" + mx_benchmark.add_bm_suite(DaCapoBenchmarkSuite()) +mx_benchmark.add_bm_suite(DacapoSmallBenchmarkSuite()) +mx_benchmark.add_bm_suite(DacapoLargeBenchmarkSuite()) +mx_benchmark.add_bm_suite(DacapoHugeBenchmarkSuite()) class DaCapoD3SBenchmarkSuite(DaCapoBenchmarkSuite): # pylint: disable=too-many-ancestors @@ -973,12 +1096,30 @@ def run(self, benchmarks, bmSuiteArgs): "tmt" : 12 } +_daCapoScalaSizes = { + "actors": ["default", "tiny", "small", "large", "huge", "gargantuan"], + "apparat": ["default", "tiny", "small", "large", "huge", "gargantuan"], + "factorie": ["default", "tiny", "small", "large", "huge", "gargantuan"], + "kiama": ["default", "small"], + "scalac": ["default", "small", "large"], + "scaladoc": ["default", "small", "large"], + "scalap": ["default", "small", "large"], + "scalariform": ["default", "tiny", "small", "large", "huge"], + "scalatest": ["default", "small"], # 'large' and 'huge' sizes fail validation + "scalaxb": ["default", "tiny", "small", "large", "huge"], + "specs": ["default", "small", "large"], + "tmt": ["default", "tiny", "small", "large", "huge"] +} + class ScalaDaCapoBenchmarkSuite(BaseDaCapoBenchmarkSuite): #pylint: disable=too-many-ancestors """Scala DaCapo benchmark suite implementation.""" def name(self): - return "scala-dacapo" + if self.workloadSize() == "default": + return "scala-dacapo" + else: + return "scala-dacapo-{}".format(self.workloadSize()) def version(self): return "0.1.0" @@ -992,13 +1133,23 @@ def daCapoClasspathEnvVarName(self): def daCapoLibraryName(self): return "DACAPO_SCALA" + def workloadSize(self): + return "default" + def daCapoIterations(self): result = _daCapoScalaConfig.copy() if not java_home_jdk().javaCompliance < '11': - mx.warn('Removing scaladacapo:actors from benchmarks because corba has been removed since JDK11 (http://openjdk.java.net/jeps/320)') + mx.logv('Removing scala-dacapo:actors from benchmarks because corba has been removed since JDK11 (http://openjdk.java.net/jeps/320)') del result['actors'] + if java_home_jdk().javaCompliance >= '16': + # See GR-29222 for details. + mx.logv('Removing scala-dacapo:specs from benchmarks because it uses a library that violates module permissions which is no longer allowed in JDK 16 (JDK-8255363)') + del result['specs'] return result + def daCapoSizes(self): + return _daCapoScalaSizes + def flakySkipPatterns(self, benchmarks, bmSuiteArgs): skip_patterns = super(ScalaDaCapoBenchmarkSuite, self).flakySuccessPatterns() if "specs" in benchmarks: @@ -1015,8 +1166,47 @@ def vmArgs(self, bmSuiteArgs): return vmArgs -mx_benchmark.add_bm_suite(ScalaDaCapoBenchmarkSuite()) +class ScalaDacapoTinyBenchmarkSuite(ScalaDaCapoBenchmarkSuite): + """The subset of Scala DaCapo benchmarks supporting the 'small' configuration.""" + + def workloadSize(self): + return "tiny" + + +class ScalaDacapoSmallBenchmarkSuite(ScalaDaCapoBenchmarkSuite): + """The subset of Scala DaCapo benchmarks supporting the 'small' configuration.""" + + def workloadSize(self): + return "small" + + +class ScalaDacapoLargeBenchmarkSuite(ScalaDaCapoBenchmarkSuite): + """The subset of Scala DaCapo benchmarks supporting the 'large' configuration.""" + + def workloadSize(self): + return "large" + +class ScalaDacapoHugeBenchmarkSuite(ScalaDaCapoBenchmarkSuite): + """The subset of Scala DaCapo benchmarks supporting the 'huge' configuration.""" + + def workloadSize(self): + return "huge" + + +class ScalaDacapoGargantuanBenchmarkSuite(ScalaDaCapoBenchmarkSuite): + """The subset of Scala DaCapo benchmarks supporting the 'gargantuan' configuration.""" + + def workloadSize(self): + return "gargantuan" + + +mx_benchmark.add_bm_suite(ScalaDaCapoBenchmarkSuite()) +mx_benchmark.add_bm_suite(ScalaDacapoTinyBenchmarkSuite()) +mx_benchmark.add_bm_suite(ScalaDacapoSmallBenchmarkSuite()) +mx_benchmark.add_bm_suite(ScalaDacapoLargeBenchmarkSuite()) +mx_benchmark.add_bm_suite(ScalaDacapoHugeBenchmarkSuite()) +mx_benchmark.add_bm_suite(ScalaDacapoGargantuanBenchmarkSuite()) _allSpecJVM2008Benches = [ @@ -1107,7 +1297,14 @@ def createCommandLineArgs(self, benchmarks, bmSuiteArgs): vmArgs = self.vmArgs(bmSuiteArgs) runArgs = self.runArgs(bmSuiteArgs) - return vmArgs + ["-jar"] + [self.specJvmPath()] + runArgs + benchmarks + + # The startup benchmarks are executed by spawning a new JVM. However, this new VM doesn't + # inherit the flags passed to the main process. + # According to the SpecJVM jar help message, one must use the '--jvmArgs' option to specify + # options to pass to the startup benchmarks. It has no effect on the non startup benchmarks. + startupJVMArgs = ["--jvmArgs", " ".join(vmArgs)] if "startup" in ' '.join(benchmarks) else [] + + return vmArgs + ["-jar"] + [self.specJvmPath()] + runArgs + benchmarks + startupJVMArgs def runArgs(self, bmSuiteArgs): runArgs = super(SpecJvm2008BenchmarkSuite, self).runArgs(bmSuiteArgs) @@ -1116,6 +1313,15 @@ def runArgs(self, bmSuiteArgs): # Skips initial check benchmark which tests for javac.jar on classpath. runArgs += ["-pja", "-Dspecjvm.run.initial.check=false"] return runArgs + + def vmArgs(self, bmSuiteArgs): + vmArgs = super(SpecJvm2008BenchmarkSuite, self).vmArgs(bmSuiteArgs) + if java_home_jdk().javaCompliance >= '16' and \ + ("xml.transform" in self.benchmarkList(bmSuiteArgs) or + "startup.xml.transform" in self.benchmarkList(bmSuiteArgs)): + vmArgs += ["--add-exports=java.xml/com.sun.org.apache.xerces.internal.parsers=ALL-UNNAMED", + "--add-exports=java.xml/com.sun.org.apache.xerces.internal.util=ALL-UNNAMED"] + return vmArgs def benchmarkList(self, bmSuiteArgs): if java_home_jdk().javaCompliance >= '9': @@ -1168,17 +1374,23 @@ def rules(self, out, benchmarks, bmSuiteArgs): mx_benchmark.add_bm_suite(SpecJvm2008BenchmarkSuite()) -_SpecJbb_specific_vmArgs = [ - "-XX:+UseNUMA", - "-XX:+AlwaysPreTouch", - "-XX:+UseLargePagesInMetaspace", - "-XX:-UseAdaptiveSizePolicy", - "-XX:-UseAdaptiveNUMAChunkSizing", - "-XX:+PrintGCDetails" -] +def _get_specjbb_vmArgs(java_compliance): + args = [ + "-XX:+UseNUMA", + "-XX:+AlwaysPreTouch", + "-XX:-UseAdaptiveSizePolicy", + "-XX:-UseAdaptiveNUMAChunkSizing", + "-XX:+PrintGCDetails" + ] -if mx.is_linux(): - _SpecJbb_specific_vmArgs.append("-XX:+UseTransparentHugePages") + if java_compliance < '16': + # JDK-8243161: Deprecated in JDK15 and marked obsolete in JDK16 + args.append("-XX:+UseLargePagesInMetaspace") + + if mx.is_linux(): + args.append("-XX:+UseTransparentHugePages") + + return args class HeapSettingsMixin(object): @@ -1230,7 +1442,7 @@ def subgroup(self): def vmArgs(self, bmSuiteArgs): vmArgs = self.vmArgshHeapFromEnv(super(SpecJbb2005BenchmarkSuite, self).vmArgs(bmSuiteArgs)) - return _SpecJbb_specific_vmArgs + vmArgs + return _get_specjbb_vmArgs(mx.get_jdk().javaCompliance) + vmArgs def specJbbClassPath(self): specjbb2005 = mx.get_env("SPECJBB2005") @@ -1371,7 +1583,7 @@ def subgroup(self): def vmArgs(self, bmSuiteArgs): vmArgs = self.vmArgshHeapFromEnv(super(SpecJbb2013BenchmarkSuite, self).vmArgs(bmSuiteArgs)) - return _SpecJbb_specific_vmArgs + vmArgs + return _get_specjbb_vmArgs(mx.get_jdk().javaCompliance) + vmArgs def specJbbClassPath(self): specjbb2013 = mx.get_env("SPECJBB2013") @@ -1481,7 +1693,7 @@ def version(self): def vmArgs(self, bmSuiteArgs): vmArgs = self.vmArgshHeapFromEnv(super(SpecJbb2015BenchmarkSuite, self).vmArgs(bmSuiteArgs)) - return _SpecJbb_specific_vmArgs + vmArgs + return _get_specjbb_vmArgs(mx.get_jdk().javaCompliance) + vmArgs def specJbbClassPath(self): specjbb2015 = mx.get_env("SPECJBB2015") @@ -1621,25 +1833,21 @@ def subgroup(self): return "graal-compiler" def renaissanceLibraryName(self): - return "RENAISSANCE_{}".format(self.renaissanceVersionToRun()) + return "RENAISSANCE_{}".format(self.version()) def renaissanceIterations(self): benchmarks = _renaissanceConfig.copy() - if self.renaissanceVersionToRun() == "0.9.0": + if self.version() == "0.9.0": del benchmarks["scala-doku"] # was introduced in 0.10.0 return benchmarks - def renaissanceVersionToRun(self): - current_version = self.defaultRenaissanceVersion() - version_to_run = self.desiredVersion() if self.desiredVersion() else current_version - if version_to_run not in self.availableRenaissanceVersions(): - mx.abort("Available Renaissance versions are : {}".format(self.availableRenaissanceVersions())) - return version_to_run + def version(self): + return super(RenaissanceBenchmarkSuite, self).version() - def defaultRenaissanceVersion(self): + def defaultSuiteVersion(self): return "0.11.0" - def availableRenaissanceVersions(self): + def availableSuiteVersions(self): return ["0.9.0", "0.10.0", "0.11.0"] def renaissancePath(self): @@ -1696,7 +1904,7 @@ def rules(self, out, benchmarks, bmSuiteArgs): "benchmark": ("", str), "benchmark-configuration": ("", str), "bench-suite": self.benchSuiteName(), - "bench-suite-version": self.renaissanceVersionToRun(), + "bench-suite-version": self.version(), "vm": "jvmci", "config.name": "default", "metric.name": "warmup", diff --git a/java-benchmarks/mx.java-benchmarks/suite.py b/java-benchmarks/mx.java-benchmarks/suite.py index 18cbd4d2cb75..5cd6b2e9c18c 100644 --- a/java-benchmarks/mx.java-benchmarks/suite.py +++ b/java-benchmarks/mx.java-benchmarks/suite.py @@ -1,5 +1,5 @@ suite = { - "mxversion" : "5.301.0", + "mxversion" : "5.309.2", "name": "java-benchmarks", "javac.lint.overrides": "none", @@ -46,6 +46,11 @@ "sha1" : "e39957904b7e79caf4fa54f30e8e4ee74d4e9e37", }, + "DACAPO_MR1_BACH" : { + "urls" : ["https://lafo.ssw.uni-linz.ac.at/pub/graal-external-deps/dacapo-9.12-MR1-bach.jar"], + "sha1" : "9cf63ef9620032b47f1a4897fe910755596b371a", + }, + "DACAPO_SCALA" : { "urls" : ["https://lafo.ssw.uni-linz.ac.at/pub/graal-external-deps/dacapo-scala-0.1.0-20120216.jar"], "sha1" : "59b64c974662b5cf9dbd3cf9045d293853dd7a51", diff --git a/sdk/mx.sdk/mx_sdk_benchmark.py b/sdk/mx.sdk/mx_sdk_benchmark.py index 4039aecb4dab..f93e90696ad8 100644 --- a/sdk/mx.sdk/mx_sdk_benchmark.py +++ b/sdk/mx.sdk/mx_sdk_benchmark.py @@ -664,7 +664,22 @@ def testStartupPerformance(self): def testPeakPerformance(self, warmup): jmeterDirectory = mx.library("APACHE_JMETER_" + self.jmeterVersion(), True).get_path(True) jmeterPath = os.path.join(jmeterDirectory, "apache-jmeter-" + self.jmeterVersion(), "bin/ApacheJMeter.jar") - jmeterCmd = [mx.get_jdk().java, "-jar", jmeterPath, "-n", "-t", self.workloadConfigurationPath(), "-j", "/dev/stdout"] # pylint: disable=line-too-long + extraVMArgs = [] + if mx.get_jdk().javaCompliance >= '9': + extraVMArgs += ["--add-opens=java.desktop/sun.awt=ALL-UNNAMED", + "--add-opens=java.desktop/sun.swing=ALL-UNNAMED", + "--add-opens=java.desktop/javax.swing.text.html=ALL-UNNAMED", + "--add-opens=java.desktop/java.awt=ALL-UNNAMED", + "--add-opens=java.desktop/java.awt.font=ALL-UNNAMED", + "--add-opens=java.desktop/sun.awt.X11=ALL-UNNAMED", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.text=ALL-UNNAMED"] + jmeterCmd = [mx.get_jdk().java] + extraVMArgs + ["-jar", jmeterPath, + "-t", self.workloadConfigurationPath(), + "-n", "-j", "/dev/stdout"] mx.log("Running JMeter: {0}".format(jmeterCmd)) output = mx.TeeOutputCapture(mx.OutputCapture()) mx.run(jmeterCmd, out=output, err=output) diff --git a/substratevm/mx.substratevm/mx_substratevm_benchmark.py b/substratevm/mx.substratevm/mx_substratevm_benchmark.py index 42cc46699789..0db79674e2a4 100644 --- a/substratevm/mx.substratevm/mx_substratevm_benchmark.py +++ b/substratevm/mx.substratevm/mx_substratevm_benchmark.py @@ -153,7 +153,7 @@ def benchSuiteName(self, bmSuiteArgs=None): return 'renaissance' def renaissance_harness_lib_name(self): - version_to_run = super(RenaissanceNativeImageBenchmarkSuite, self).renaissanceVersionToRun() + version_to_run = super(RenaissanceNativeImageBenchmarkSuite, self).version() version_end_index = str(version_to_run).rindex('.') return 'RENAISSANCE_HARNESS_v' + str(version_to_run)[0:version_end_index] @@ -165,7 +165,7 @@ def harness_path(self): # Before supporting new Renaissance versions, we must cross-compile Renaissance harness project # with scala 11 for benchmarks compiled with this version of Scala. - def availableRenaissanceVersions(self): + def availableSuiteVersions(self): return ["0.9.0", "0.10.0", "0.11.0"] def renaissance_unpacked(self): diff --git a/vm/mx.vm/mx_vm_gate.py b/vm/mx.vm/mx_vm_gate.py index 8b5f39232694..8f40d9227e61 100644 --- a/vm/mx.vm/mx_vm_gate.py +++ b/vm/mx.vm/mx_vm_gate.py @@ -26,6 +26,7 @@ # ---------------------------------------------------------------------------------------------------- import mx +import mx_benchmark import mx_subst import mx_unittest import mx_sdk_vm @@ -92,36 +93,36 @@ def gate_body(args, tasks): mx.run([java_exe, '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', - '-jar', mx.library('DACAPO').get_path(True), 'avrora']) + '-jar', mx.library('DACAPO').get_path(True), 'avrora', '-n', '1']) # Ensure that fatal errors in libgraal route back to HotSpot - testdir = mkdtemp() try: - scratch_dir = join(testdir, 'scratch') - cmd = [java_exe, - '-XX:+UseJVMCICompiler', - '-XX:+UseJVMCINativeLibrary', - '-XX:+PrintFlagsFinal', - '-Dlibgraal.CrashAt=length,hashCode', - '-Dlibgraal.CrashAtIsFatal=true', - '-jar', mx.library('DACAPO').get_path(True), 'avrora', '--preserve', '--scratch-directory', scratch_dir] + vmargs = ['-XX:+UseJVMCICompiler', + '-XX:+UseJVMCINativeLibrary', + '-XX:+PrintFlagsFinal', + '-Dlibgraal.CrashAt=length,hashCode', + '-Dlibgraal.CrashAtIsFatal=true'] + cmd = ["dacapo:avrora", "--tracker=none", "--"] + vmargs + ["--", "--preserve"] out = mx.OutputCapture() - with mx_compiler.DaCapoWrapper(scratch_dir): - exitcode = mx.run(cmd, cwd=testdir, nonZeroIsFatal=False, out=out) + exitcode, bench_suite, _ = mx_benchmark.gate_mx_benchmark(cmd, nonZeroIsFatal=False, out=out, err=out) if exitcode == 0: if 'CrashAtIsFatal: no fatalError function pointer installed' in out.data: # Executing a VM that does not configure fatal errors handling # in libgraal to route back through the VM. pass else: - mx.abort('Expected following command to result in non-zero exit code: ' + ' '.join(cmd)) + mx.abort('Expected following benchmark to result in non-zero exit code: ' + ' '.join(cmd)) else: + if len(bench_suite.scratchDirs()) == 0: + mx.abort("No scratch dir found despite error being expected!") + latest_scratch_dir = bench_suite.scratchDirs()[-1] seen_libjvmci_log = False - hs_errs = glob.glob(join(testdir, 'hs_err_pid*.log')) + hs_errs = glob.glob(join(latest_scratch_dir, 'hs_err_pid*.log')) if not hs_errs: - mx.abort('Expected a file starting with "hs_err_pid" in test directory. Entries found=' + str(listdir(testdir))) + mx.abort('Expected a file starting with "hs_err_pid" in test directory. Entries found=' + str(listdir(latest_scratch_dir))) for hs_err in hs_errs: - with open(join(testdir, hs_err)) as fp: + mx.log("Verifying content of {}".format(join(latest_scratch_dir, hs_err))) + with open(join(latest_scratch_dir, hs_err)) as fp: contents = fp.read() if 'libjvmci' in hs_err: seen_libjvmci_log = True @@ -130,11 +131,14 @@ def gate_body(args, tasks): else: if 'Fatal error in JVMCI' not in contents: mx.abort('Expected "Fatal error in JVMCI" to be in contents of ' + hs_err + ':' + linesep + contents) + if 'JVMCINativeLibraryErrorFile' in out.data and not seen_libjvmci_log: - mx.abort('Expected a file matching "hs_err_pid*_libjvmci.log" in test directory. Entries found=' + str(listdir(testdir))) + mx.abort('Expected a file matching "hs_err_pid*_libjvmci.log" in test directory. Entries found=' + str(listdir(latest_scratch_dir))) finally: - mx.rmtree(testdir) + for scratch_dir in bench_suite.scratchDirs(): + mx.log("Cleaning up scratch dir after gate task completion: {}".format(scratch_dir)) + mx.rmtree(scratch_dir) with Task('LibGraal Compiler:CTW', tasks, tags=[VmGateTasks.libgraal]) as t: if t: diff --git a/vm/mx.vm/suite.py b/vm/mx.vm/suite.py index b746fa69dad8..93931603e88f 100644 --- a/vm/mx.vm/suite.py +++ b/vm/mx.vm/suite.py @@ -1,7 +1,7 @@ suite = { "name": "vm", "version" : "21.3.0", - "mxversion" : "5.307.0", + "mxversion" : "5.309.2", "release" : False, "groupId" : "org.graalvm",