From beb069655612ad6eff1cb1132fc723432afa55d6 Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Sat, 26 Oct 2024 00:35:38 +0200 Subject: [PATCH] Add a TAG Client/Server system for ZTS Signed-off-by: Tino Reichardt --- .github/workflows/scripts/qemu-3-deps.sh | 11 +- .github/workflows/scripts/qemu-6-tests.sh | 36 +++- scripts/zfs-tests.sh | 241 +++++++++++++++++----- tests/test-runner/bin/test-runner.py.in | 34 ++- 4 files changed, 253 insertions(+), 69 deletions(-) diff --git a/.github/workflows/scripts/qemu-3-deps.sh b/.github/workflows/scripts/qemu-3-deps.sh index bf06c1892ceb..e91598a529b5 100755 --- a/.github/workflows/scripts/qemu-3-deps.sh +++ b/.github/workflows/scripts/qemu-3-deps.sh @@ -16,7 +16,7 @@ function archlinux() { sudo pacman -Sy --noconfirm base-devel bc cpio cryptsetup dhclient dkms \ fakeroot fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils \ parted pax perf python-packaging python-setuptools qemu-guest-agent ksh \ - samba sysstat rng-tools rsync wget xxhash + samba socat sysstat rng-tools rsync wget xxhash echo "##[endgroup]" } @@ -38,7 +38,8 @@ function debian() { lsscsi nfs-kernel-server pamtester parted python3 python3-all-dev \ python3-cffi python3-dev python3-distlib python3-packaging \ python3-setuptools python3-sphinx qemu-guest-agent rng-tools rpm2cpio \ - rsync samba sysstat uuid-dev watchdog wget xfslibs-dev xxhash zlib1g-dev + rsync samba socat sysstat uuid-dev watchdog wget xfslibs-dev xxhash \ + zlib1g-dev echo "##[endgroup]" } @@ -48,7 +49,8 @@ function freebsd() { echo "##[group]Install Development Tools" sudo pkg install -y autoconf automake autotools base64 checkbashisms fio \ gdb gettext gettext-runtime git gmake gsed jq ksh93 lcov libtool lscpu \ - pkgconf python python3 pamtester pamtester qemu-guest-agent rsync xxhash + pkgconf python python3 pamtester pamtester qemu-guest-agent rsync socat \ + xxhash sudo pkg install -xy \ '^samba4[[:digit:]]+$' \ '^py3[[:digit:]]+-cffi$' \ @@ -75,7 +77,8 @@ function rhel() { libuuid-devel lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester \ parted perf python3 python3-cffi python3-devel python3-packaging \ kernel-devel python3-setuptools qemu-guest-agent rng-tools rpcgen \ - rpm-build rsync samba sysstat systemd watchdog wget xfsprogs-devel xxhash \ + rpm-build rsync samba socat sysstat systemd watchdog wget \ + xfsprogs-devel xxhash \ zlib-devel echo "##[endgroup]" } diff --git a/.github/workflows/scripts/qemu-6-tests.sh b/.github/workflows/scripts/qemu-6-tests.sh index 2f023198bbf6..afff063f1e9d 100755 --- a/.github/workflows/scripts/qemu-6-tests.sh +++ b/.github/workflows/scripts/qemu-6-tests.sh @@ -9,16 +9,19 @@ set -eu -function prefix() { - ID="$1" - LINE="$2" +function get_time_diff() { CURRENT=$(date +%s) - TSSTART=$(cat /tmp/tsstart) DIFF=$((CURRENT-TSSTART)) H=$((DIFF/3600)) DIFF=$((DIFF-(H*3600))) M=$((DIFF/60)) S=$((DIFF-(M*60))) + printf "%02d:%02d:%02d" "$H" "$M" "$S" +} + +function prefix() { + ID="$1" + LINE="$2" CTR=$(cat /tmp/ctr) echo $LINE| grep -q "^Test[: ]" && CTR=$((CTR+1)) && echo $CTR > /tmp/ctr @@ -27,12 +30,13 @@ function prefix() { COLOR="$BASE/scripts/zfs-tests-color.sh" CLINE=$(echo $LINE| grep "^Test[ :]" | sed -e 's|/usr/local|/usr|g' \ | sed -e 's| /usr/share/zfs/zfs-tests/tests/| |g' | $COLOR) + if [ -z "$CLINE" ]; then printf "vm${ID}: %s\n" "$LINE" else # [vm2: 00:15:54 256] Test: functional/checksum/setup (run as root) [00:00] [PASS] - printf "[vm${ID}: %02d:%02d:%02d %4d] %s\n" \ - "$H" "$M" "$S" "$CTR" "$CLINE" + t=$(get_time_diff) + printf "[vm${ID}: %s %4d] %s\n" "$t" "$CTR" "$CLINE" fi } @@ -42,9 +46,9 @@ if [ -z ${1:-} ]; then source env.txt SSH=$(which ssh) TESTS='$HOME/zfs/.github/workflows/scripts/qemu-6-tests.sh' - echo 0 > /tmp/ctr - date "+%s" > /tmp/tsstart + TSSTART=$(date +%s) + echo 0 > /tmp/ctr for i in $(seq 1 $VMs); do IP="192.168.122.1$i" daemonize -c /var/tmp -p vm${i}.pid -o vm${i}log.txt -- \ @@ -54,7 +58,7 @@ if [ -z ${1:-} ]; then | while read -r line; do prefix "$i" "$line"; done & echo $! > vm${i}log.pid # don't mix up the initial --- Configuration --- part - sleep 0.13 + sleep 0.2 done # wait for all vm's to finish @@ -97,8 +101,20 @@ fi sudo dmesg -c > dmesg-prerun.txt mount > mount.txt df -h > df-prerun.txt -$TDIR/zfs-tests.sh -vK -s 3GB -T $TAGS +# start tag-server on the first vm: +if [ "$2" = "1" ]; then + $TDIR/zfs-tests.sh -vK -s 3GB -T 192.168.122.11:2323/server +fi + +# wait for tag-server on first VM +sleep 1 + +# run all tags, provided by tag-server +$TDIR/zfs-tests.sh -vK -s 3GB -T 192.168.122.11:2323/vm$2 +#$TDIR/zfs-tests.sh -vK -s 3GB -T $TAGS RV=$? + +# get some stats df -h > df-postrun.txt echo $RV > tests-exitcode.txt sync diff --git a/scripts/zfs-tests.sh b/scripts/zfs-tests.sh index 2906d73442c2..cf9e39fb00d1 100755 --- a/scripts/zfs-tests.sh +++ b/scripts/zfs-tests.sh @@ -53,6 +53,7 @@ ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh" UNAME=$(uname) RERUN="" KMEMLEAK="" +TAG_CLIENT="" # Override some defaults if on FreeBSD if [ "$UNAME" = "FreeBSD" ] ; then @@ -210,10 +211,9 @@ find_runfile() { } # Given a TAGS with a format like "1/3" or "2/3" then divide up the test list -# into portions and print that portion. So "1/3" for "the first third of the +# into portions and print that portion. So "1/3" for "the first third of the # test tags". # -# split_tags() { # Get numerator and denominator NUM=$(echo "$TAGS" | cut -d/ -f1) @@ -252,6 +252,78 @@ split_tags() { sed -E 's/,$//' } +# Test for old instance of server and stop it + delete old tmpfiles +tag_server_stop() { + test -s $PIDFILE && kill `cat $PIDFILE` || true 2>/dev/null + rm -rf $RDIR +} + +# Given a TAGS with a format like "192.168.122.10:12345/server", then start +# in TAG-Server mode, which multiple clients can ask for the next TAG. +# - the server replys with next TAG or EOF on the end +# - ip and port are variable +# - the "/server" string in the end is fixed for server mode +# +RDIR="/tmp/tag-server" +RESPONSE="$RDIR/response.sh" +PIDFILE="$RDIR/tag-server.pid" +tag_server() { + IP=$1 + PORT=$2 + NAME=$3 + + tag_server_stop + mkdir -p $RDIR + + # generate a response script + i=0; x="" + _RUNFILES=${RUNFILES//","/" "} + cat $_RUNFILES | tr -d "[],\'" | awk '/tags = /{print $NF}' | sort | \ + uniq | grep -v functional > TAGS + while read line; do + i=$((i+1)) + x="$x\nTAGS[$i]=$line" + done < ./TAGS + rm -f TAGS + TAGS=$(echo -e "$x") + cat < $RESPONSE +#!/usr/bin/env bash +set -eu +declare -A TAGS +$TAGS +MAX=$i +todo=\$(cat \$0.current) +next=\$((todo+1)) +if [ \$todo -gt \$MAX ]; then + echo EOF + kill \$(cat $PIDFILE) +else + echo \${TAGS[\$todo]} + echo \$next > \$0.current +fi +EOF + chmod +x $RESPONSE + echo 1 > $RESPONSE.current + # start server socket and bind to given IP:PORT + socat TCP-LISTEN:$PORT,bind=$IP,reuseaddr,fork SYSTEM:"'${SHELL}' $RESPONSE" & + echo $! > $PIDFILE + echo "TAG-Server ($IP:$PORT) started with PID $(cat $PIDFILE)" + exit +} + +# Given a TAGS with a format like "192.168.122.10:12345/vmN", then start +# as a TAG Client, which asks a server for the tests that should be run. +# +tag_client() { + IP=$1 + PORT=$2 + NAME=$3 + exec 3<>/dev/tcp/$IP/$PORT + echo "$NAME needs work ;)" >&3 + next=$(cat <&3) + echo $next +} + # # Symlink file if it appears under any of the given paths. # @@ -349,37 +421,43 @@ constrain_path() { usage() { cat << EOF USAGE: -$0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER] +$0 [-hvqDxkfScRm] [-n NFSFILE] [-I NUM] [-d DIR] [-s SIZE] ... +... [-r RUNFILES] [-t PATH|NAME] [-T TAGS] [-u USER] DESCRIPTION: ZFS Test Suite launch script OPTIONS: - -h Show this message - -v Verbose zfs-tests.sh output - -q Quiet test-runner output - -D Debug; show all test output immediately (noisy) - -x Remove all testpools, dm, lo, and files (unsafe) - -k Disable cleanup after test failure - -K Log test names to /dev/kmsg - -f Use files only, disables block device tests - -S Enable stack tracer (negative performance impact) - -c Only create and populate constrained path - -R Automatically rerun failing tests - -m Enable kmemleak reporting (Linux only) - -n NFSFILE Use the nfsfile to determine the NFS configuration - -I NUM Number of iterations - -d DIR Use world-writable DIR for files and loopback devices - -s SIZE Use vdevs of SIZE (default: 4G) - -r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES}) - -t PATH|NAME Run single test at PATH relative to test suite, - or search for test by NAME - -T TAGS Comma separated list of tags (default: 'functional') - Alternately, specify a fraction like "1/3" or "2/3" to - run the first third of tests or 2nd third of the tests. This - is useful for splitting up the test amongst different + -h Show this message + -v Verbose zfs-tests.sh output + -q Quiet test-runner output + -D Debug; show all test output immediately (noisy) + -x Remove all testpools, dm, lo, and files (unsafe) + -k Disable cleanup after test failure + -K Log test names to /dev/kmsg + -f Use files only, disables block device tests + -S Enable stack tracer (negative performance impact) + -c Only create and populate constrained path + -R Automatically rerun failing tests + -m Enable kmemleak reporting (Linux only) + -n NFSFILE Use the nfsfile to determine the NFS configuration + -I NUM Number of iterations + -d DIR Use world-writable DIR for files and loopback devices + -s SIZE Use vdevs of SIZE (default: 4G) + -r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES}) + -t PATH|NAME Run single test at PATH relative to test suite, or search + for test by NAME + -T TAGS Comma separated list of tags (default: 'functional') + Alternately #1: specify a fraction like "1/3" or "2/3" to + run the first third of tests or 2nd third of the tests. + This is useful for splitting up the test amongst different runners. - -u USER Run single test as USER (default: root) + Alternately #2: specify a client/server IP:PORT definition + like this for server: -T 192.168.100.2:2323/server and + this for client: -T 192.168.100.2:2323/client1. This is + also useful for dynamic splitting up the test amongst + different runners. + -u USER Run single test as USER (default: root) EXAMPLES: # Run the default ${DEFAULT_RUNFILES//\.run/} suite of tests and output the configuration used. @@ -537,8 +615,6 @@ fi # TAGS=${TAGS:='functional'} - - # # Attempt to locate the runfiles describing the test workload. # @@ -574,6 +650,11 @@ RUNFILES=${R#,} # if echo "$TAGS" | grep -Eq '^[0-9]+/[0-9]+$' ; then TAGS=$(split_tags) +elif echo "$TAGS" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+/server$' ; then + TAG_ARGS=$(echo "$TAGS"|tr ":/" " ") + tag_server $TAG_ARGS +elif echo "$TAGS" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+/.*$' ; then + TAG_CLIENT="$TAGS" fi # @@ -758,29 +839,90 @@ mkdir -p "$FILEDIR" || : RESULTS_FILE=$(mktemp_file zts-results) REPORT_FILE=$(mktemp_file zts-report) +function get_time_diff() { + CURRENT=$(date +%s) + DIFF=$((CURRENT-TSSTART)) + H=$((DIFF/3600)) + DIFF=$((DIFF-(H*3600))) + M=$((DIFF/60)) + S=$((DIFF-(M*60))) + printf "%02d:%02d:%02d" "$H" "$M" "$S" +} + +# +# Run only one test, so no summary and so on +# +run_tests_tag_client() { + PATH=$STF_PATH \ + ${TEST_RUNNER} \ + ${QUIET:+-q} -Q \ + ${DEBUG:+-D} \ + ${KMEMLEAK:+-m} \ + ${KMSG:+-K} \ + -c "${RUNFILES}" \ + -T "${TAGS}" \ + -i "${STF_SUITE}" \ + -I "${ITERATIONS}" \ + 2>&1 | tee -a "$RESULTS_FILE" + echo $? >"$REPORT_FILE" +} + # # Run all the tests as specified. # -msg "${TEST_RUNNER}" \ - "${QUIET:+-q}" \ - "${DEBUG:+-D}" \ - "${KMEMLEAK:+-m}" \ - "${KMSG:+-K}" \ - "-c \"${RUNFILES}\"" \ - "-T \"${TAGS}\"" \ - "-i \"${STF_SUITE}\"" \ - "-I \"${ITERATIONS}\"" -{ PATH=$STF_PATH \ - ${TEST_RUNNER} \ - ${QUIET:+-q} \ - ${DEBUG:+-D} \ - ${KMEMLEAK:+-m} \ - ${KMSG:+-K} \ - -c "${RUNFILES}" \ - -T "${TAGS}" \ - -i "${STF_SUITE}" \ - -I "${ITERATIONS}" \ - 2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE" +if [ ! -z "$TAG_CLIENT" ]; then + TSSTART=$(date +%s) + TAG_ARGS=$(echo "$TAG_CLIENT"|tr ":/" " ") + while true; do + TAGS=$(tag_client $TAG_ARGS) + test -z "$TAGS" && break + test "$TAGS" = "EOF" && break + run_tests_tag_client + done + + # Add Summary to $RESULTS_FILE + awk -v T=$(get_time_diff) ' +BEGIN { pass=0; skip=0; fail=0; } +/\[PASS\]/ { pass+=1 } +/\[FAIL\]/ { fail+=1 } +/\[KILLED\]/ { fail+=1 } +/\[SKIP\]/ { skip+=1 } +END { + if (pass+fail+skip > 0) + percent_passed = (pass/(pass+fail+skip) * 100) + else + percent_passed = 0 + print "\nResults Summary" + printf("PASS:\t%4d\n", pass) + printf("FAIL:\t%4d\n", fail) + printf("SKIP:\t%4d\n\n", skip) + printf("Running Time:\t%s\n", T) + printf("Percent passed:\t%.1f%\n", percent_passed) + printf("Log directory:\t/var/tmp/test_results/tag-client\n\n") +}' $RESULTS_FILE | tee -a $RESULTS_FILE +else + msg "${TEST_RUNNER}" \ + "${QUIET:+-q}" \ + "${DEBUG:+-D}" \ + "${KMEMLEAK:+-m}" \ + "${KMSG:+-K}" \ + "-c \"${RUNFILES}\"" \ + "-T \"${TAGS}\"" \ + "-i \"${STF_SUITE}\"" \ + "-I \"${ITERATIONS}\"" + { PATH=$STF_PATH \ + ${TEST_RUNNER} \ + ${QUIET:+-q} \ + ${DEBUG:+-D} \ + ${KMEMLEAK:+-m} \ + ${KMSG:+-K} \ + -c "${RUNFILES}" \ + -T "${TAGS}" \ + -i "${STF_SUITE}" \ + -I "${ITERATIONS}" \ + 2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE" +fi + read -r RUNRESULT <"$REPORT_FILE" # @@ -816,7 +958,6 @@ if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then RESULT=$? fi - cat "$REPORT_FILE" RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE") diff --git a/tests/test-runner/bin/test-runner.py.in b/tests/test-runner/bin/test-runner.py.in index 92fb64feeeef..9739cd3e9b34 100755 --- a/tests/test-runner/bin/test-runner.py.in +++ b/tests/test-runner/bin/test-runner.py.in @@ -642,18 +642,22 @@ Tags: %s class TestRun(object): - props = ['quiet', 'outputdir', 'debug'] + props = ['quiet', 'outputdir', 'debug', 'nosummary'] def __init__(self, options): self.tests = {} self.testgroups = {} self.starttime = time() - self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') + if options.nosummary: + self.timestamp = 'tag-client' + else: + self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') self.outputdir = os.path.join(options.outputdir, self.timestamp) self.setup_logging(options) self.defaults = [ ('outputdir', BASEDIR), ('quiet', False), + ('nosummary', False), ('timeout', 60), ('user', ''), ('pre', ''), @@ -879,10 +883,10 @@ class TestRun(object): if not options.template: try: old = os.umask(0) - os.makedirs(self.outputdir, mode=0o777) + os.makedirs(self.outputdir, mode=0o777, exist_ok=True) os.umask(old) filename = os.path.join(self.outputdir, 'log') - LOG_FILE_OBJ = open(filename, buffering=0, mode='wb') + LOG_FILE_OBJ = open(filename, buffering=0, mode='ab') except OSError as e: fail('%s' % e) @@ -943,6 +947,20 @@ class TestRun(object): return 0 + def nosummary(self): + if Result.total == 0: + return 2 + + if Result.runresults['FAIL'] > 0: + return 1 + + if Result.runresults['KILLED'] > 0: + return 1 + + if Result.runresults['RERAN'] > 0: + return 3 + + return 0 def write_log(msg, target): """ @@ -1113,6 +1131,8 @@ def parse_args(): type='string', help='Specify a post script.') parser.add_option('-q', action='store_true', default=False, dest='quiet', help='Silence on the console during a test run.') + parser.add_option('-Q', action='store_true', default=False, dest='nosummary', + help='Print no summary on the console.') parser.add_option('-s', action='callback', callback=options_cb, default='', dest='failsafe', metavar='script', type='string', help='Specify a failsafe script.') @@ -1170,7 +1190,11 @@ def main(): testrun.complete_outputdirs() testrun.run(options) - exit(testrun.summary()) + + if options.nosummary: + exit(testrun.nosummary()) + else: + exit(testrun.summary()) if __name__ == '__main__':