From 8b3fc57510347fcbc4bf54d16941277bf3da93e7 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Tue, 10 Oct 2023 17:08:40 +0200 Subject: [PATCH] Fix getting output from `sar -A`, moving sar to "system-load" Fix 'sar -A' from timing out: This issue was that when the human-readable output of `sar -A` was added in April to `--entries=xenserver-logs`, the timeout for the capability was not defined. The capability xenserver-logs uses the default timeout of -1 which is racey for commands as they are immediately stopped due to timeout of -1 running out unless the are very very brief. To fix this, it was suggested that sar (a system load report) could be moved to a new capability. Also, it should be noted that sar records are binary system load records, not xenserver-specific log files. Therefore, create a new capability (system_load), repsonsible for collecting the the sar data files (and the humand-readable of output of `sar -A`) with a timeout of 30 seconds and a max_size of 70 MB. Signed-off-by: Bernhard Kaindl --- tests/integration/sar-file-collection.test.sh | 46 ++++++++++++++----- xen-bugtool | 11 +++-- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/tests/integration/sar-file-collection.test.sh b/tests/integration/sar-file-collection.test.sh index a43f3817..f57a23ee 100755 --- a/tests/integration/sar-file-collection.test.sh +++ b/tests/integration/sar-file-collection.test.sh @@ -9,14 +9,22 @@ set -o errexit set -o pipefail if [[ -n "$TRACE" ]]; then set -o xtrace; fi set -o nounset +SRC=$PWD : ${PYTHON:=python2} +# The bugtool capabilitiy this test is testing: +CAP=system-load + +# Test object: A dummy sar files (Unix system activity reporter from the sysstat pkg) +# and the output of a fake `sar -A` command. URL: https://github.com/sysstat/sysstat + # Prepare test container: Mock the files used by this test cp -a tests/integration/dom0-template/* / mkdir -p /var/log/sa echo sa >/var/log/sa/sa01 echo sr >/var/log/sa/sar31 -echo -e '#!/bin/sh\necho $*' >/bin/sar;chmod +x /bin/sar +echo -e '#!/bin/sh\nsleep 2;find xen-bugtool tests/ -type f|xargs cat;echo $*' >/bin/sar +chmod +x /bin/sar # Enter a clean test environment rm -rf .tmp/tests/sar-file-collection @@ -27,10 +35,20 @@ export PYTHONPATH=~-/tests/mocks # Check that mocking xen.lowlevel.xc works for this test $PYTHON -c "from xen.lowlevel.xc import Error, xc;xc().domain_getinfo()" -# Run xen-bugtool --entries=xenserver-logs to capture the dummy SAR files -# and run the mocked sar command: -$PYTHON ~-/xen-bugtool -y --entries=xenserver-logs --output=tar --outfd=1 -s | - tar xvf - --strip-components=1 +# Run xen-bugtool --entries=$CAP to test tar output (to a file descriptor in this case) +tar_basename=tar +export XENRT_BUGTOOL_BASENAME=$tar_basename +# Test creating a tar archive on a file descriptor using --output=tar --outfd=fd +$PYTHON ~-/xen-bugtool -y --entries=$CAP --output=tar --outfd=2 2>tar.bz2 +tar xvf tar.bz2 + +# Run xen-bugtool --entries=$CAP --output=zip to test zip output (output to file only) +zip_basename=zip +export XENRT_BUGTOOL_BASENAME=$zip_basename +$PYTHON ~-/xen-bugtool -y --entries=$CAP --output=zip +unzip -o -d. /var/opt/xen/bug-report/zip.zip + +pushd $zip_basename # Show a detailed file list in the outlog log for human analysis in case of errors find * -type f -print0 | xargs -0 ls -l @@ -39,15 +57,21 @@ find * -type f -print0 | xargs -0 ls -l grep -q '^sa$' var/log/sa/sa01 grep -q '^sr$' var/log/sa/sar31 -# xen-bugtool is expected to call sar -A, and the symlink to echo captures it: -grep -q '^-A$' sar-A.out +# bugtool call sar -A, printing dummy data and the arguments it was given. Check that: +tail -1 sar-A.out | grep -q '^-A$' # There is likely a xml tool to check the file names, in inventory.xml, # but after verfiying the the files were included it should be sufficent # check that they are also mentioned int the inventory.xml: -grep -q 'filename="bug-report-[0-9]*/var/log/sa/sa01"' inventory.xml -grep -q 'filename="bug-report-[0-9]*/var/log/sa/sar31"' inventory.xml -grep -q 'filename="bug-report-[0-9]*/sar-A.out"' inventory.xml +grep -q 'filename=".*/var/log/sa/sa01"' inventory.xml +grep -q 'filename=".*/var/log/sa/sar31"' inventory.xml +grep -q 'filename=".*/sar-A.out"' inventory.xml # Validate the extracted inventory.xml using the XML Schema -xmllint --schema ~-/tests/integration/inventory.xsd inventory.xml +xmllint --schema $SRC/tests/integration/inventory.xsd inventory.xml +popd + +# Check that the tar and zip outputs are identical (except for date and uptime): +sed -i "s/\\<$tar_basename\\>/$zip_basename/" tar/inventory.xml +sed -i 's/date="[^"]*"//;s/uptime="[^"]*"//' {tar,zip}/inventory.xml +diff -rNu tar zip \ No newline at end of file diff --git a/xen-bugtool b/xen-bugtool index cd8e7d9e..c037a345 100755 --- a/xen-bugtool +++ b/xen-bugtool @@ -352,6 +352,7 @@ CAP_PAM = 'pam' CAP_PROCESS_LIST = 'process-list' CAP_PERSISTENT_STATS = 'persistent-stats' CAP_BLOCK_SCHEDULER = 'block-scheduler' +CAP_SYSTEM_LOAD = 'system-load' CAP_SYSTEM_LOGS = 'system-logs' CAP_SYSTEM_SERVICES = 'system-services' CAP_TAPDISK_LOGS = 'tapdisk-logs' @@ -426,6 +427,7 @@ cap(CAP_PERSISTENT_STATS, PII_NO, max_size=50*MB, max_time=60, checked=False, hidden=True) cap(CAP_PROCESS_LIST, PII_YES, max_size=30*KB, max_time=60) +cap(CAP_SYSTEM_LOAD, PII_NO, max_size=70*MB, max_time=30) cap(CAP_SYSTEM_LOGS, PII_MAYBE, max_size=50*MB, max_time=10) cap(CAP_SYSTEM_SERVICES, PII_NO, max_size=128*KB, @@ -1096,13 +1098,14 @@ exclude those logs from the archive. 'telemetry/telemetry.log.%d', 'telemetry/telemetry.log.%d.gz']]] # Collect SAR data (binary and text, and add today's report with sar -A) - sa_logs = get_recent_logs(glob.glob("/var/log/sa/sa*[0-9][0-9]"), caps[CAP_XENSERVER_LOGS][VERBOSITY]) - cmd_output(CAP_XENSERVER_LOGS, ['sar', '-A']) + cmd_output(CAP_SYSTEM_LOAD, ['sar', '-A']) + sar_data = get_recent_logs(glob.glob("/var/log/sa/sa*[0-9][0-9]"), caps[CAP_SYSTEM_LOAD][VERBOSITY]) + update_cap_size(CAP_SYSTEM_LOAD, size_of_all(sar_data)) + file_output(CAP_SYSTEM_LOAD, sar_data) qemu_logs = get_recent_logs(glob.glob('/tmp/qemu.[0-9]*'), caps[CAP_XENSERVER_LOGS][VERBOSITY]) - update_cap_size(CAP_XENSERVER_LOGS, size_of_all(xenserver_logs + sa_logs + qemu_logs)) + update_cap_size(CAP_XENSERVER_LOGS, size_of_all(xenserver_logs + qemu_logs)) file_output(CAP_XENSERVER_LOGS, xenserver_logs) - file_output(CAP_XENSERVER_LOGS, sa_logs) file_output(CAP_XENSERVER_LOGS, qemu_logs) tree_output(CAP_XENSERVER_LOGS, OEM_CONFIG_DIR, OEM_XENSERVER_LOGS_RE)