From 42232b4ab7299b2fcfb21cb7941bc16d14ff0144 Mon Sep 17 00:00:00 2001 From: dataisland Date: Tue, 11 Jun 2024 10:58:15 -0500 Subject: [PATCH] Upload SPEC test script --- artifact/README.md | 37 +++++++ artifact/spec2017/.gitignore | 1 + artifact/spec2017/Dockerfile | 7 ++ artifact/spec2017/build.sh | 16 +++ artifact/spec2017/config/native.cfg | 76 +++++++++++++ artifact/spec2017/config/shadowbound.cfg | 113 ++++++++++++++++++++ artifact/spec2017/config/whitelist.txt | 53 ++++++++++ artifact/spec2017/scripts/spectest.c | 72 +++++++++++++ artifact/spec2017/scripts/spectest.py | 129 +++++++++++++++++++++++ artifact/spec2017/scripts/start.sh | 31 ++++++ docker-compose.yml | 6 ++ 11 files changed, 541 insertions(+) create mode 100644 artifact/spec2017/.gitignore create mode 100644 artifact/spec2017/Dockerfile create mode 100755 artifact/spec2017/build.sh create mode 100644 artifact/spec2017/config/native.cfg create mode 100644 artifact/spec2017/config/shadowbound.cfg create mode 100644 artifact/spec2017/config/whitelist.txt create mode 100644 artifact/spec2017/scripts/spectest.c create mode 100644 artifact/spec2017/scripts/spectest.py create mode 100755 artifact/spec2017/scripts/start.sh diff --git a/artifact/README.md b/artifact/README.md index d4fb2af7e..9e1736f7c 100644 --- a/artifact/README.md +++ b/artifact/README.md @@ -63,6 +63,23 @@ docker run -it dataisland/shadowbound-seceval /bin/bash /root/test.sh ``` +After running the command, you should see the following output: + +```shell +[+] 2017-9164-9166 +[+] 2017-9167-9173 +[+] CVE-2006-6563 +[+] CVE-2009-2285 +[+] CVE-2013-4243 +[+] CVE-2013-7443 +[+] CVE-2014-1912 +[+] CVE-2015-8668 +[+] CVE-2015-9101 +[+] CVE-2016-10270 +[+] CVE-2016-10271 +... +``` + #### 🔨 Build Image by Yourself > [!WARNING] @@ -83,3 +100,23 @@ docker run -it sec-eval /bin/bash ``` ### ⚡ (Step 5) Performance Test + +In this step, we demonstrate the performance of ShadowBound. You can use the following command to run the test: + +#### 🖼️ Use Pre-built Image + +```bash +docker pull dataisland/shadowbound-perfeval:latest +docker run -privileged -it dataisland/shadowbound-perfeval /bin/bash +python3 /root/scripts/spectest.py | tee /root/spectest.log +``` + +After running the command, you can the result at `/root/spectest.log`. + +#### 🔨 Build Image by Yoursel + +If you want to build the image by yourself, you need to download the SPEC CPU 2017 benchmark suite (`cpu2017.iso`) from the official website. Then, use the following command to build the image: + +```bash +CPU2017_PATH=/path/to/cpu2017.iso artifact/spec2017/build.sh +``` diff --git a/artifact/spec2017/.gitignore b/artifact/spec2017/.gitignore new file mode 100644 index 000000000..7a8f7aad0 --- /dev/null +++ b/artifact/spec2017/.gitignore @@ -0,0 +1 @@ +*.iso \ No newline at end of file diff --git a/artifact/spec2017/Dockerfile b/artifact/spec2017/Dockerfile new file mode 100644 index 000000000..d8f01c54f --- /dev/null +++ b/artifact/spec2017/Dockerfile @@ -0,0 +1,7 @@ +FROM shadowbound + +COPY ./config /shadowbound/config +COPY ./artifact/spec2017/config /root/config +COPY ./artifact/spec2017/scripts /root/scripts + +RUN cd /root/scripts && gcc -o spectest spectest.c && pip3 install prettytable numpy \ No newline at end of file diff --git a/artifact/spec2017/build.sh b/artifact/spec2017/build.sh new file mode 100755 index 000000000..e1f03ed2f --- /dev/null +++ b/artifact/spec2017/build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +CPU2017_PATH=${CPU2017_PATH:-$HOME/shadowbound/artifact/spec2017/cpu2017.iso} + +if [ ! -f $CPU2017_PATH ]; then + echo "Please set CPU2017_PATH to the path of the SPEC CPU2017 ISO" + echo "If you do not have the ISO, please use our pre-built image" + exit 1 +fi + +CPU2017_DIR=`dirname $CPU2017_PATH` + +docker compose up --build spec2017-eval +docker run --privileged -v $CPU2017_DIR:/cpu2017 -it spec2017-eval /root/scripts/start.sh \ No newline at end of file diff --git a/artifact/spec2017/config/native.cfg b/artifact/spec2017/config/native.cfg new file mode 100644 index 000000000..c54895227 --- /dev/null +++ b/artifact/spec2017/config/native.cfg @@ -0,0 +1,76 @@ +%ifndef %{label} +% define label "native" +%endif + +%ifndef %{build_ncpus} +% define build_ncpus 8 +%endif + +command_add_redirect = 1 +flagsurl = $[top]/config/flags/clang.xml +ignore_errors = 1 +iterations = 1 +label = %{label} +line_width = 1020 +log_line_width = 1020 +makeflags = --jobs=%{build_ncpus} +mean_anyway = 1 +output_format = txt,csv +preenv = 1 +reportable = 0 +tune = peak + +intrate,fprate: + copies = 1 +intspeed,fpspeed: + threads = 1 + +default: + CC = /shadowbound/llvm-project/build/bin/clang -std=c99 + CXX = /shadowbound/llvm-project/build/bin/clang++ -std=c++03 + + CC_VERSION_OPTION = --version + CXX_VERSION_OPTION = --version + +default: + sw_base_ptrsize = 64-bit + sw_peak_ptrsize = 64-bit + +any_fortran: + fail_build = 1 + +default: + EXTRA_PORTABILITY = -DSPEC_LP64 + +500.perlbench_r,600.perlbench_s: +%if %{bits} == 32 +% define suffix IA32 +%else +% define suffix X64 +%endif + PORTABILITY = -DSPEC_LINUX_%{suffix} + +502.gcc_r,602.gcc_s: + EXTRA_CFLAGS = -fno-strict-aliasing -fgnu89-inline + +510.parest_r: + CXXPORTABILITY = -DSPEC_PAREST_STD_FLUSH_WORKAROUND=1 + +523.xalancbmk_r,623.xalancbmk_s: + CXXPORTABILITY = -DSPEC_LINUX + +525.x264_r,625.x264_s: + PORTABILITY = -fcommon + +526.blender_r: + PORTABILITY = -funsigned-char -DSPEC_LINUX + CXXPORTABILITY = -D__BOOL_DEFINED + +default=base: + OPTIMIZE = -g -O1 -fno-slp-vectorize + +default=peak: + OPTIMIZE = -g -O1 -fno-slp-vectorize + +default: + sw_compiler001 = Clang 15 diff --git a/artifact/spec2017/config/shadowbound.cfg b/artifact/spec2017/config/shadowbound.cfg new file mode 100644 index 000000000..fb1a96c79 --- /dev/null +++ b/artifact/spec2017/config/shadowbound.cfg @@ -0,0 +1,113 @@ +%ifndef %{label} +% define label "shadowbound" +%endif + +%ifndef %{build_ncpus} +% define build_ncpus 8 +%endif + +command_add_redirect = 1 +flagsurl = $[top]/config/flags/clang.xml +ignore_errors = 1 +iterations = 1 +label = %{label} +line_width = 1020 +log_line_width = 1020 +makeflags = --jobs=%{build_ncpus} +mean_anyway = 1 +output_format = txt,csv +preenv = 1 +reportable = 0 +tune = peak + +intrate,fprate: + copies = 1 +intspeed,fpspeed: + threads = 1 + +default: + CC = /shadowbound/llvm-project/build/bin/clang -std=c99 -fsanitize=overflow-defense -mllvm -odef-whitelist=/root/cpu2017/whitelist.txt + CXX = /shadowbound/llvm-project/build/bin/clang++ -std=c++03 -fsanitize=overflow-defense -mllvm -odef-whitelist=/root/cpu2017/whitelist.txt + + CC_VERSION_OPTION = --version + CXX_VERSION_OPTION = --version + +default: + sw_base_ptrsize = 64-bit + sw_peak_ptrsize = 64-bit + +any_fortran: + fail_build = 1 + +default: + EXTRA_PORTABILITY = -DSPEC_LP64 + +500.perlbench_r,600.perlbench_s: +%if %{bits} == 32 +% define suffix IA32 +%else +% define suffix X64 +%endif + PORTABILITY = -DSPEC_LINUX_%{suffix} + EXTRA_CFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/500.perlbench_r.json -mllvm -odef-perf-test=1 + +502.gcc_r,602.gcc_s: + EXTRA_CFLAGS = -fno-strict-aliasing -fgnu89-inline -mllvm -odef-pattern-opt-file=/shadowbound/config/502.gcc_r.json -mllvm -odef-perf-test=1 + +505.mcf_r,605.mcf_s: + EXTRA_CFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/505.mcf_r.json + +508.namd_r: + EXTRA_CXXFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/508.namd_r.json -mllvm -odef-perf-test=1 + +510.parest_r: + CXXPORTABILITY = -DSPEC_PAREST_STD_FLUSH_WORKAROUND=1 + EXTRA_CXXFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/510.parest_r.json -mllvm -odef-perf-test=1 + +511.povray_r: + EXTRA_CFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/511.povray_r.json + EXTRA_CXXFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/511.povray_r.json + +519.lbm_r,619.lbm_s: + EXTRA_CFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/519.lbm_r.json + +520.omnetpp_r,620.omnetpp_s: + EXTRA_CXXFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/520.omnetpp_r.json + +523.xalancbmk_r,623.xalancbmk_s: + CXXPORTABILITY = -DSPEC_LINUX + EXTRA_CXXFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/523.xalancbmk_r.json -mllvm -odef-perf-test=1 + +525.x264_r,625.x264_s: + PORTABILITY = -fcommon + EXTRA_CFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/525.x264_r.json + +526.blender_r: + PORTABILITY = -funsigned-char -DSPEC_LINUX + CXXPORTABILITY = -D__BOOL_DEFINED + EXTRA_CFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/526.blender_r.json + EXTRA_CXXFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/526.blender_r.json + +531.deepsjeng_r,631.deepsjeng_s: + EXTRA_CXXFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/531.deepsjeng_r.json + +538.imagick_r,638.imagick_s: + EXTRA_CFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/538.imagick_r.json + +541.leela_r,641.leela_s: + EXTRA_CXXFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/541.leela_r.json + +544.nab_r,644.nab_s: + EXTRA_CFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/544.nab_r.json + +557.xz_r,657.xz_s: + EXTRA_CFLAGS = -mllvm -odef-pattern-opt-file=/shadowbound/config/557.xz_r.json + +default=base: + OPTIMIZE = -g -O1 -fno-slp-vectorize + +default=peak: + OPTIMIZE = -g -O1 -fno-slp-vectorize + +default: + sw_compiler001 = Clang 15 diff --git a/artifact/spec2017/config/whitelist.txt b/artifact/spec2017/config/whitelist.txt new file mode 100644 index 000000000..60821c4d8 --- /dev/null +++ b/artifact/spec2017/config/whitelist.txt @@ -0,0 +1,53 @@ +# 500.perlbench_r +S_nextchar +Perl_regexec_flags + +# 503.gcc_r +lex_string +bitmap_obstack_release +c_parse_file +sra_deinitialize +free_scc_vn +free_vn_table +execute_rtl_pre +execute_rtl_cprop +rest_of_handle_gcse2 +run_scc_vn +free_aux_for_blocks +free_aux_for_edges + +# 526.blender_r +polyfill_prepare + +# 508.namd_r +_ZN20ComputeNonbondedUtil9calc_selfEP9nonbonded +_ZN20ComputeNonbondedUtil38calc_self_energy_merge_fullelect_pprofEP9nonbonded +_ZN20ComputeNonbondedUtil37calc_self_energy_slow_fullelect_pprofEP9nonbonded +_ZN20ComputeNonbondedUtil36calc_self_energy_merge_fullelect_lesEP9nonbonded +_ZN20ComputeNonbondedUtil36calc_self_energy_merge_fullelect_intEP9nonbonded +_ZN20ComputeNonbondedUtil36calc_self_energy_merge_fullelect_fepEP9nonbonded +_ZN20ComputeNonbondedUtil35calc_self_energy_slow_fullelect_lesEP9nonbonded +_ZN20ComputeNonbondedUtil35calc_self_energy_slow_fullelect_fepEP9nonbonded +_ZN20ComputeNonbondedUtil32calc_self_energy_merge_fullelectEP9nonbonded +_ZN20ComputeNonbondedUtil32calc_self_energy_fullelect_pprofEP9nonbonded +_ZN20ComputeNonbondedUtil31calc_self_merge_fullelect_pprofEP9nonbonded +_ZN20ComputeNonbondedUtil31calc_self_energy_slow_fullelectEP9nonbonded +_ZN20ComputeNonbondedUtil30calc_self_slow_fullelect_pprofEP9nonbonded +_ZN20ComputeNonbondedUtil30calc_self_energy_fullelect_lesEP9nonbonded +_ZN20ComputeNonbondedUtil30calc_self_energy_fullelect_intEP9nonbonded +_ZN20ComputeNonbondedUtil30calc_self_energy_fullelect_fepEP9nonbonded +_ZN20ComputeNonbondedUtil29calc_self_merge_fullelect_lesEP9nonbonded +_ZN20ComputeNonbondedUtil28calc_self_slow_fullelect_lesEP9nonbonded +_ZN20ComputeNonbondedUtil26calc_self_energy_fullelectEP9nonbonded +_ZN20ComputeNonbondedUtil25calc_self_merge_fullelectEP9nonbonded +_ZN20ComputeNonbondedUtil25calc_self_fullelect_pprofEP9nonbonded +_ZN20ComputeNonbondedUtil24calc_self_slow_fullelectEP9nonbonded +_ZN20ComputeNonbondedUtil23calc_self_fullelect_lesEP9nonbonded +_ZN20ComputeNonbondedUtil22calc_self_energy_pprofEP9nonbonded +_ZN20ComputeNonbondedUtil20calc_self_energy_lesEP9nonbonded +_ZN20ComputeNonbondedUtil20calc_self_energy_intEP9nonbonded +_ZN20ComputeNonbondedUtil20calc_self_energy_fepEP9nonbonded +_ZN20ComputeNonbondedUtil19calc_self_fullelectEP9nonbonded +_ZN20ComputeNonbondedUtil16calc_self_energyEP9nonbonded +_ZN20ComputeNonbondedUtil15calc_self_pprofEP9nonbonded +_ZN20ComputeNonbondedUtil13calc_self_lesEP9nonbonded \ No newline at end of file diff --git a/artifact/spec2017/scripts/spectest.c b/artifact/spec2017/scripts/spectest.c new file mode 100644 index 000000000..95efd0182 --- /dev/null +++ b/artifact/spec2017/scripts/spectest.c @@ -0,0 +1,72 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct rusage rusage; +struct timeval start, end; +pid_t pid; + +static inline int run(const char *file, char *const *argv) +{ + pid_t pid = fork(); + if (pid == 0) + { + freopen("/dev/null", "w", stdout); + freopen("/dev/null", "w", stderr); + execvp(file, argv); + exit(-1); + } + else if (pid > 0) + { + int status; + wait4(pid, &status, 0, &rusage); + gettimeofday(&end, NULL); + return status; + } + else + { + perror("fork"); + return 1; + } +} + +int main(int argc, char *argv[]) +{ + if (argc < 2) + { + printf("Usage: exe_file args\n"); + printf("e.g: perlbench_s -I./lib checkspam.pl 2500 5 25 11 150 1 1 1 1\n"); + return 1; + } + + int time_used, cpu_time_used, memory_used, signum; + + gettimeofday(&start, NULL); + + int ret = run(argv[1], (char *const *)&argv[1]); + signum = WEXITSTATUS(ret); + + time_used = (int)(end.tv_sec * 1000 + end.tv_usec / 1000 - start.tv_sec * 1000 - + start.tv_usec / 1000); + cpu_time_used = + rusage.ru_utime.tv_sec * 1000 + rusage.ru_utime.tv_usec / 1000 + + rusage.ru_stime.tv_sec * 1000 + rusage.ru_stime.tv_usec / 1000; + + memory_used = rusage.ru_maxrss; + + printf("time: %6d ms\n", time_used); + printf("memory: %6d kb\n", memory_used); + printf("exit: %6d\n", signum); + fflush(stdout); + + return 0; +} \ No newline at end of file diff --git a/artifact/spec2017/scripts/spectest.py b/artifact/spec2017/scripts/spectest.py new file mode 100644 index 000000000..fdd10b37f --- /dev/null +++ b/artifact/spec2017/scripts/spectest.py @@ -0,0 +1,129 @@ +import os +import subprocess +import numpy as np +from prettytable import PrettyTable + +def get_command_mapper(config): + return { + '500.perlbench_r': [ + f'../run_peak_refrate_{config}.0000/perlbench_r_peak.{config} -I./lib checkspam.pl 2500 5 25 11 150 1 1 1 1', + f'../run_peak_refrate_{config}.0000/perlbench_r_peak.{config} -I./lib diffmail.pl 4 800 10 17 19 300', + f'../run_peak_refrate_{config}.0000/perlbench_r_peak.{config} -I./lib splitmail.pl 6400 12 26 16 100 0', + ], + '502.gcc_r': [ + f'../run_peak_refrate_{config}.0000/cpugcc_r_peak.{config} gcc-pp.c -O3 -finline-limit=0 -fif-conversion -fif-conversion2 -o gcc-pp.opts-O3_-finline-limit_0_-fif-conversion_-fif-conversion2.s', + f'../run_peak_refrate_{config}.0000/cpugcc_r_peak.{config} gcc-pp.c -O2 -finline-limit=36000 -fpic -o gcc-pp.opts-O2_-finline-limit_36000_-fpic.s', + f'../run_peak_refrate_{config}.0000/cpugcc_r_peak.{config} gcc-smaller.c -O3 -fipa-pta -o gcc-smaller.opts-O3_-fipa-pta.s', + f'../run_peak_refrate_{config}.0000/cpugcc_r_peak.{config} ref32.c -O5 -o ref32.opts-O5.s', + f'../run_peak_refrate_{config}.0000/cpugcc_r_peak.{config} ref32.c -O3 -fselective-scheduling -fselective-scheduling2 -o ref32.opts-O3_-fselective-scheduling_-fselective-scheduling2.s', + ], + '505.mcf_r': [ + f'../run_peak_refrate_{config}.0000/mcf_r_peak.{config} inp.in', + ], + '520.omnetpp_r': [ + f'../run_peak_refrate_{config}.0000/omnetpp_r_peak.{config} -c General -r 0', + ], + '523.xalancbmk_r': [ + f'../run_peak_refrate_{config}.0000/cpuxalan_r_peak.{config} -v t5.xml xalanc.xsl', + ], + '525.x264_r': [ + f'../run_peak_refrate_{config}.0000/x264_r_peak.{config} --pass 1 --stats x264_stats.log --bitrate 1000 --frames 1000 -o BuckBunny_New.264 BuckBunny.yuv 1280x720', + f'../run_peak_refrate_{config}.0000/x264_r_peak.{config} --pass 2 --stats x264_stats.log --bitrate 1000 --dumpyuv 200 --frames 1000 -o BuckBunny_New.264 BuckBunny.yuv 1280x720', + f'../run_peak_refrate_{config}.0000/x264_r_peak.{config} --seek 500 --dumpyuv 200 --frames 1250 -o BuckBunny_New.264 BuckBunny.yuv 1280x720', + ], + '531.deepsjeng_r': [ + f'../run_peak_refrate_{config}.0000/deepsjeng_r_peak.{config} ref.txt', + ], + '541.leela_r': [ + f'../run_peak_refrate_{config}.0000/leela_r_peak.{config} ref.sgf', + ], + '557.xz_r': [ + f'../run_peak_refrate_{config}.0000/xz_r_peak.{config} cld.tar.xz 160 19cf30ae51eddcbefda78dd06014b4b96281456e078ca7c13e1c0c9e6aaea8dff3efb4ad6b0456697718cede6bd5454852652806a657bb56e07d61128434b474 59796407 61004416 6', + f'../run_peak_refrate_{config}.0000/xz_r_peak.{config} cpu2006docs.tar.xz 250 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 23047774 23513385 6e', + f'../run_peak_refrate_{config}.0000/xz_r_peak.{config} input.combined.xz 250 a841f68f38572a49d86226b7ff5baeb31bd19dc637a922a972b2e6d1257a890f6a544ecab967c313e370478c74f760eb229d4eef8a8d2836d233d3e9dd1430bf 40401484 41217675 7', + ], + '508.namd_r': [ + f'../run_peak_refrate_{config}.0000/namd_r_peak.{config} --input apoa1.input --output apoa1.ref.output --iterations 65', + ], + '510.parest_r': [ + f'../run_peak_refrate_{config}.0000/parest_r_peak.{config} ref.prm', + ], + '511.povray_r': [ + f'../run_peak_refrate_{config}.0000/povray_r_peak.{config} SPEC-benchmark-ref.ini', + ], + '519.lbm_r': [ + f'../run_peak_refrate_{config}.0000/lbm_r_peak.{config} 3000 reference.dat 0 0 100_100_130_ldc.of', + ], + '526.blender_r': [ + f'../run_peak_refrate_{config}.0000/blender_r_peak.{config} sh3_no_char.blend --render-output sh3_no_char_ --threads 1 -b -F RAWTGA -s 849 -e 849 -a', + ], + '538.imagick_r': [ + f'../run_peak_refrate_{config}.0000/imagick_r_peak.{config} -limit disk 0 refrate_input.tga -edge 41 -resample 181% -emboss 31 -colorspace YUV -mean-shift 19x19+15% -resize 30% refrate_output.tga', + ], + '544.nab_r': [ + f'../run_peak_refrate_{config}.0000/nab_r_peak.{config} 1am0 1122214447 122', + ], + } + +def safe_run_command(command, cwd): + while True: + output = subprocess.check_output( + [command], shell=True, stderr=subprocess.DEVNULL, cwd=cwd) + lines = output.strip().splitlines()[-3:] + time = int(lines[0].split()[1].decode()) + memory = int(lines[1].split()[1].decode()) + retval = int(lines[2].split()[1].decode()) + if retval == 0 and time > 1000: ## HACK: Avoid any potential errors + return time, memory + +SPECTEST = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'spectest') + +table = PrettyTable() +native_command_mapper = get_command_mapper('native') +shadowbound_command_mapper = get_command_mapper('shadowbound') + +table.field_names = ['Benchmark', 'Time', 'Memory'] +time_overheads = [] +memory_overheads = [] + +for bench in get_command_mapper('').keys(): + print(f'Testing {bench}') + native_time, native_memory = 0, 0 + for idx, command in enumerate(native_command_mapper[bench]): + print(f'[native] Running "{command}"') + time, memory = safe_run_command( + f"{SPECTEST} {command}", + f"/root/cpu2017/benchspec/CPU/{bench}/run/run_peak_refrate_native.0000" + ) + print('Time: ', time, 'ms') + print('Memory: ', memory, 'KB') + + native_time += time + native_memory += memory + + shadowbound_time, shadowbound_memory = 0, 0 + for idx, command in enumerate(shadowbound_command_mapper[bench]): + print(f'[shadowbound] Running "{command}"') + time, memory = safe_run_command( + f"{SPECTEST} {command}", + f"/root/cpu2017/benchspec/CPU/{bench}/run/run_peak_refrate_shadowbound.0000" + ) + + print('Time: ', time, 'ms') + print('Memory: ', memory, 'KB') + + shadowbound_time += time + shadowbound_memory += memory + + time_overhead = shadowbound_time / native_time + memory_overhead = shadowbound_memory / native_memory + table.add_row([bench, f"{time_overhead:.2f}x", f"{memory_overhead:.2f}x"]) + + time_overheads.append(max(time_overhead - 1, 1000 / native_time)) + memory_overheads.append(max(memory_overhead - 1, 1024 / native_memory)) + +geometric_mean_time_overhead = np.prod(time_overheads) ** (1 / len(time_overheads)) * 100 +geometric_mean_memory_overhead = np.prod(memory_overheads) ** (1 / len(memory_overheads)) * 100 +table.add_row(['Geometric Mean', f"{geometric_mean_time_overhead:.2f}%", f"{geometric_mean_memory_overhead:.2f}%"]) + +print(table) diff --git a/artifact/spec2017/scripts/start.sh b/artifact/spec2017/scripts/start.sh new file mode 100755 index 000000000..35c9a7c67 --- /dev/null +++ b/artifact/spec2017/scripts/start.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e + +## Install SPEC CPU2017 +mkdir /root/cpu2017 +mount -t iso9660 -o ro,exec,loop /cpu2017/cpu2017.iso /mnt +cd /mnt +echo "yes" | ./install.sh -d /root/cpu2017 +cp /root/config/whitelist.txt /root/cpu2017 +cp /root/config/native.cfg /root/cpu2017/config +cp /root/config/shadowbound.cfg /root/cpu2017/config +cd /root/cpu2017 +umount /mnt + +## Set up environment +source shrc +runcpu -c native -a runsetup intrate_no_fortran fprate_no_fortran +runcpu -c shadowbound -a runsetup intrate_no_fortran fprate_no_fortran + +## Clean SPEC CPU2017 +find /root/cpu2017 -mindepth 1 -maxdepth 1 ! -name 'benchspec' -exec rm -rf {} + +rm /root/cpu2017/benchspec/CPU/*.bset +rm -rf /root/cpu2017/benchspec/CPU/6* /root/cpu2017/benchspec/CPU/9* +rm -rf /root/cpu2017/benchspec/CPU/5*/src /root/cpu2017/benchspec/CPU/5*/Docs /root/cpu2017/benchspec/CPU/5*/Spec + +## Run SPEC CPU2017 +python3 -u /root/scripts/spectest.py | tee /root/spectest.log + +## Interactive shell +/bin/bash \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 69c4a1c2c..f2f9b9b9e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,13 @@ services: context: artifact/security dockerfile: Dockerfile image: sec-eval + spec2017-eval: + build: + context: . + dockerfile: artifact/spec2017/Dockerfile + image: spec2017-eval # docker compose up --build shadowbound # docker compose up --build nginx-eval # docker compose up --build sec-eval +# docker compose up --build spec2017-eval