Fix getting output from sar -A, moving sar to "system-load"

Fix 'sar -A' from timing out:

This issue was that when the human-readable output of `sar -A`
was added in April to `--entries=xenserver-logs`, the timeout
for the capability was not defined.

The capability xenserver-logs uses the default timeout of -1
which is racey for commands as they are immediately stopped
due to timeout of -1 running out unless the are very very brief.

To fix this, it was suggested that sar (a system load report)
could be moved to a new capability. Also, it should be noted that
sar records are binary system load records, not xenserver-specific
log files.

Therefore, create a new capability (system_load), repsonsible for
collecting the the sar data files (and the humand-readable of output
of `sar -A`) with a timeout of 30 seconds and a max_size of 70 MB.

Signed-off-by: Bernhard Kaindl <bernhard.kaindl@cloud.com>

tests/integration/sar-file-collection.test.sh

@@ -9,14 +9,22 @@ set -o errexit
 set -o pipefail
 if [[ -n "$TRACE" ]]; then set -o xtrace; fi
 set -o nounset
+SRC=$PWD
 : ${PYTHON:=python2}
 
+# The bugtool capabilitiy this test is testing:
+CAP=system-load
+
+# Test object: A dummy sar files (Unix system activity reporter from the sysstat pkg)
+# and the output of a fake `sar -A` command. URL: https://github.com/sysstat/sysstat
+
 # Prepare test container: Mock the files used by this test
 cp -a tests/integration/dom0-template/* /
 mkdir -p  /var/log/sa
 echo sa  >/var/log/sa/sa01
 echo sr  >/var/log/sa/sar31
-echo -e '#!/bin/sh\necho $*' >/bin/sar;chmod +x /bin/sar
+echo -e '#!/bin/sh\nsleep 2;find xen-bugtool tests/ -type f|xargs cat;echo $*' >/bin/sar
+chmod +x /bin/sar
 
 # Enter a clean test environment
 rm -rf   .tmp/tests/sar-file-collection
@@ -27,10 +35,20 @@ export PYTHONPATH=~-/tests/mocks
 # Check that mocking xen.lowlevel.xc works for this test
 $PYTHON -c "from xen.lowlevel.xc import Error, xc;xc().domain_getinfo()"
 
-# Run xen-bugtool --entries=xenserver-logs to capture the dummy SAR files
-# and run the mocked sar command:
-$PYTHON ~-/xen-bugtool -y --entries=xenserver-logs --output=tar --outfd=1 -s |
-    tar xvf - --strip-components=1
+# Run xen-bugtool --entries=$CAP to test tar output (to a file descriptor in this case)
+tar_basename=tar
+export XENRT_BUGTOOL_BASENAME=$tar_basename
+# Test creating a tar archive on a file descriptor using --output=tar --outfd=fd
+$PYTHON ~-/xen-bugtool -y --entries=$CAP --output=tar --outfd=2 2>tar.bz2
+tar xvf tar.bz2
+
+# Run xen-bugtool --entries=$CAP --output=zip to test zip output (output to file only)
+zip_basename=zip
+export XENRT_BUGTOOL_BASENAME=$zip_basename
+$PYTHON ~-/xen-bugtool -y --entries=$CAP --output=zip
+unzip -o -d. /var/opt/xen/bug-report/zip.zip
+
+pushd $zip_basename
 
 # Show a detailed file list in the outlog log for human analysis in case of errors
 find * -type f -print0 | xargs -0 ls -l
@@ -39,15 +57,21 @@ find * -type f -print0 | xargs -0 ls -l
 grep -q '^sa$' var/log/sa/sa01
 grep -q '^sr$' var/log/sa/sar31
 
-# xen-bugtool is expected to call sar -A, and the symlink to echo captures it:
-grep -q '^-A$' sar-A.out
+# bugtool call sar -A, printing dummy data and the arguments it was given. Check that:
+tail -1 sar-A.out | grep -q '^-A$'
 
 # There is likely a xml tool to check the file names, in inventory.xml,
 # but after verfiying the the files were included it should be sufficent
 # check that they are also mentioned int the inventory.xml:
-grep -q 'filename="bug-report-[0-9]*/var/log/sa/sa01"'  inventory.xml
-grep -q 'filename="bug-report-[0-9]*/var/log/sa/sar31"' inventory.xml
-grep -q 'filename="bug-report-[0-9]*/sar-A.out"'        inventory.xml
+grep -q 'filename=".*/var/log/sa/sa01"'  inventory.xml
+grep -q 'filename=".*/var/log/sa/sar31"' inventory.xml
+grep -q 'filename=".*/sar-A.out"'        inventory.xml
 
 # Validate the extracted inventory.xml using the XML Schema
-xmllint --schema ~-/tests/integration/inventory.xsd inventory.xml
+xmllint --schema $SRC/tests/integration/inventory.xsd inventory.xml
+popd
+
+# Check that the tar and zip outputs are identical (except for date and uptime):
+sed -i "s/\\<$tar_basename\\>/$zip_basename/" tar/inventory.xml
+sed -i 's/date="[^"]*"//;s/uptime="[^"]*"//' {tar,zip}/inventory.xml
+diff -rNu tar zip

xen-bugtool

@@ -352,6 +352,7 @@ CAP_PAM                  = 'pam'
 CAP_PROCESS_LIST         = 'process-list'
 CAP_PERSISTENT_STATS     = 'persistent-stats'
 CAP_BLOCK_SCHEDULER      = 'block-scheduler'
+CAP_SYSTEM_LOAD          = 'system-load'
 CAP_SYSTEM_LOGS          = 'system-logs'
 CAP_SYSTEM_SERVICES      = 'system-services'
 CAP_TAPDISK_LOGS         = 'tapdisk-logs'
@@ -426,6 +427,7 @@ cap(CAP_PERSISTENT_STATS,    PII_NO,                    max_size=50*MB,
     max_time=60, checked=False, hidden=True)
 cap(CAP_PROCESS_LIST,        PII_YES,                   max_size=30*KB,
     max_time=60)
+cap(CAP_SYSTEM_LOAD,         PII_NO,                    max_size=70*MB, max_time=30)
 cap(CAP_SYSTEM_LOGS,         PII_MAYBE,                 max_size=50*MB,
     max_time=10)
 cap(CAP_SYSTEM_SERVICES,     PII_NO,                    max_size=128*KB,
@@ -1096,13 +1098,14 @@ exclude those logs from the archive.
                                   'telemetry/telemetry.log.%d', 'telemetry/telemetry.log.%d.gz']]]
 
     # Collect SAR data (binary and text, and add today's report with sar -A)
-    sa_logs = get_recent_logs(glob.glob("/var/log/sa/sa*[0-9][0-9]"), caps[CAP_XENSERVER_LOGS][VERBOSITY])
-    cmd_output(CAP_XENSERVER_LOGS, ['sar', '-A'])
+    cmd_output(CAP_SYSTEM_LOAD, ['sar', '-A'])
+    sar_data = get_recent_logs(glob.glob("/var/log/sa/sa*[0-9][0-9]"), caps[CAP_SYSTEM_LOAD][VERBOSITY])
+    update_cap_size(CAP_SYSTEM_LOAD, size_of_all(sar_data))
+    file_output(CAP_SYSTEM_LOAD, sar_data)
 
     qemu_logs = get_recent_logs(glob.glob('/tmp/qemu.[0-9]*'), caps[CAP_XENSERVER_LOGS][VERBOSITY])
-    update_cap_size(CAP_XENSERVER_LOGS, size_of_all(xenserver_logs + sa_logs + qemu_logs))
+    update_cap_size(CAP_XENSERVER_LOGS, size_of_all(xenserver_logs + qemu_logs))
     file_output(CAP_XENSERVER_LOGS, xenserver_logs)
-    file_output(CAP_XENSERVER_LOGS, sa_logs)
     file_output(CAP_XENSERVER_LOGS, qemu_logs)
     tree_output(CAP_XENSERVER_LOGS, OEM_CONFIG_DIR, OEM_XENSERVER_LOGS_RE)