Some progress

Author: Lars T Hansen

manual-tests/sessions/Makefile

@@ -1,7 +1,8 @@
 .PHONY: all clean
 
 PROGRAMS=sonar-session sonar-job sonar-worker
-CXXFLAGS=-O2
+CFLAGS=-Wall
+CXXFLAGS=-O2 -Wall
 
 all: $(PROGRAMS)
 clean:

manual-tests/sessions/check-output.awk

@@ -1,81 +1,106 @@
-# WALLTIME must have the execution time in seconds
-
 BEGIN {
-  delete session[0]
-  delete job[0]
-  delete bash[0]
-  WALLTIME=strtonum(WALLTIME)
+  # Arguably keeping all the times is unnecessary, we just need min
+  # and max, but it's OK for debugging.
+  delete job[0]			# job[pid][...] is cputime for jobs
+  delete bash[0]		# bash[pid][...] is cputime for interactive bash shells
+  delete run[0]			# run[pid][...] is cputime for run_test.sh shells
+  WALLTIME=strtonum(WALLTIME)	# Execution time in seconds of tomost sonar-job
+  NUMJOBS=strtonum(NUMJOBS)     # Number of jobs we should see
 }
 
-# For each line, parse the cputime_sec if present and bucket the output by command
 {
-  time=0
-  ix=index($0, ",cputime_sec=")
-  if (ix > 0) {
-    s=substr($0, ix+13)
-    ix=index(s, ",")
-    if (ix > 0) {
-      s=substr(s, 0, ix-1)
-    }
-    time=strtonum(s)
-  }
-  if (index($0, ",cmd=sonar-session") > 0) {
-    session[length(session)]=time
-  } else if (index($0, ",cmd=sonar-job") > 0) {
-    job[length(job)]=time
-  } else if (index($0, ",cmd=bash") > 0) {
-    ix=index($0, ",pid=")
-    if (ix > 0) {
-      s=substr($0, ix+5)
-      ix=index(s, ",")
-      if (ix > 0) {
-        s=substr(s, 0, ix-1)
-      }
-      if (isarray(bash[s])) {
-        bash[s][length(bash[s])]=time
-      } else {
-        bash[s][0]=time
-      }
-    }
+  time=number_field($0, "cputime_sec")
+  pid=number_field($0, "pid")
+  cmd=string_field($0, "cmd")
+  switch (cmd) {
+    case /.*sonar-job/:
+      if (isarray(job[pid]))
+	job[pid][length(job[pid])]=time
+      else
+	job[pid][0]=time
+      break
+    case /.*bash/:
+      if (isarray(bash[pid]))
+	bash[pid][length(bash[pid])]=time
+      else
+	bash[pid][0]=time
+      break
+    case /.*run_test/:
+      if (isarray(run[pid]))
+	run[pid][length(run[pid])]=time
+      else
+	run[pid][0]=time
+      break
   }
 }
 
-# Check that the bucket values are sane
 END {
-  if (length(session) == 0) {
-    print "No sessions!"
-    exit 1
-  }
-  diff=session[length(session)-1] - session[0]
-  if (diff > 2) {
-    print "Session cost " diff " is too high"
-    for ( x in session ) {
-      print "  " session[x]
-    }
-    exit 1
-  }
-
-  if (length(job) == 0) {
-    print("no jobs!")
-    exit 1
+  fail=0
+  if (length(job) != NUMJOBS) {
+    print "Wrong number of jobs, expected " NUMJOBS " got " length(job)
+    fail=1
   }
-  diff=job[length(job)-1] - job[0]
-  if (diff < WALLTIME/2) {
-    print "Job cost " diff " is too small for wall time " WALLTIME
-    for ( x in job ) {
-      print "  " job[x]
+  for (pid in job) {
+    diff=job[pid][length(job[pid])-1] - job[pid][0]
+    if (diff < WALLTIME/2) {
+      print "Job " pid ": Job cost " diff " is too small for wall time " WALLTIME
+      for ( x in job[pid] ) {
+        print "  " job[pid][x]
+      }
+      fail=1
     }
-    exit 1
   }
 
   for (pid in bash) {
     diff=bash[pid][length(bash[pid])-1] - bash[pid][0]
     if (diff > 2) {
-      print "Bash cost " diff " is too high"
+      print "Bash " pid ": Shell cost " diff " is too high"
       for ( x in bash[pid] ) {
         print "  " bash[pid][x]
       }
-      exit 1
+      fail=1
     }
   }
+
+  if (length(run) == 0) {
+    print "No test runners!"
+    fail=1
+  }
+
+  for (pid in run) {
+    diff=run[pid][length(run[pid])-1] - run[pid][0]
+    if (diff > 2) {
+      print "Run " pid ": Shell cost " diff " is too high"
+      for ( x in run[pid] ) {
+        print "  " run[pid][x]
+      }
+      fail=1
+    }
+  }
+
+  if (fail) {
+    exit 1
+  }
+}
+
+function number_field(input, tag,     s) {
+  s = string_field(input, tag)
+  if (s != "")
+    return strtonum(s)
+  return 0
+}
+
+function string_field(input, tag,    s, pat, ix) {
+  s = ""
+  pat="," tag "="
+  ix=index(input, pat)
+  if (ix > 0) {
+    s=substr($0, ix+length(pat))
+    ix=index(s, ",")
+    if (ix > 0) {
+      s=substr(s, 0, ix-1)
+    }
+  }
+  return s
 }
+

manual-tests/sessions/run_test.sh

@@ -3,8 +3,27 @@
 # Usage:
 #  run_test.sh path-to-sonar-binary
 #
-# This must be run from the directory that has sonar-session.c, sonar-job.c, sonar-worker.cc,
-# Makefile and check-output.awk.
+# This tests the accumulation of cpu time to nested jobs.
+#
+# A job is a Unix "process group".  Within a process group, the cpu time of children accrue to the
+# process group leader as the children terminate (are waited for).  But when a process in a group
+# forks off a new process group - creates a subjob - the cpu time of the subjob should not accrue
+# the the parent job.
+#
+# To test this:
+#
+#  - this script runs `sonar-job N` where N >= 0
+#  - if N > 0, that program forks off `sonar-job N-1`
+#  - each job is its own process group
+#  - each sonar-job then runs some copies of sonar-work, the cpu time of which will accrue to the job
+#  - sonar runs in the background to observe the goings-on
+#  - in the end, when the topmost sonar-job returns, we want to check that each sonar-job has gotten
+#    cpu time accrued to it corresponding to the sum of of its workers, not including subjobs, and
+#    also that this script has not been charged for the work of any of the jobs.
+#  - this script is called run_test.sh and this is known to the awk processing script, q.v.
+#
+# This script must be run from the directory that has sonar-job.c, sonar-worker.cc, Makefile and
+# check-output.awk.
 
 set -e
 if (( $# != 1 )); then
@@ -16,6 +35,7 @@ SONARBIN=$1
 # Config these, if you must
 SLEEPTIME=3
 WAITTIME=10
+NUMJOBS=2
 OUTFILE=sonar-output.txt
 
 # Compile things as necessary
@@ -24,19 +44,26 @@ make --quiet
 # Run sonar in the background, every few seconds, and try to capture no more than necessary.
 rm -f $OUTFILE
 ( while true ; do
-      $SONARBIN ps | grep -E ",user=$LOGNAME," | grep -E ',cmd=(sonar|bash)' >> $OUTFILE
+      $SONARBIN ps --batchless --exclude-system-jobs | grep -E ',cmd=(sonar|run_test|bash)' >> $OUTFILE
       sleep $SLEEPTIME
   done ) &
 SONARPID=$!
 
-# Run the new session root
-echo "The test can take 60-90s on 2024-era hardware"
+# Fork off a new job tree - with parameter N, we should get N+1 levels
+echo "The test will take several minutes"
 then=$(date +%s)
-./sonar-session
+./sonar-job $((NUMJOBS - 1))
 now=$(date +%s)
 
+# Wait for session time to be accrued to this shell
+echo "Waiting $((SLEEPTIME * 2))s in run_test (#1) for things to settle..."
+sleep $((SLEEPTIME * 2))
 kill -TERM $SONARPID
 
-# Now process the output.  See check-output.c for details.
-gawk -vWALLTIME=$((now - then)) -f check-output.awk $OUTFILE
+# New output may arrive late after the script has been killed
+echo "Waiting $((SLEEPTIME * 2))s in run_test (#2) for things to settle..."
+sleep $((SLEEPTIME * 2))
+
+# Now process the output.
+gawk -vNUMJOBS=$NUMJOBS -vWALLTIME=$((now - then)) -f check-output.awk $OUTFILE
 echo "Everything is fine"

manual-tests/sessions/sonar-job.c

@@ -1,6 +1,7 @@
 /* The job of this program is to be the parent process for a bunch of worker processes, which we run
  * serially as that provides the best signal. */
 
+#include <assert.h>
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -11,20 +12,61 @@
 #define ITERATIONS 5
 
 int main(int argc, char **argv) {
+    /* If this is not the top of a process group, make it one */
+    if (getpid() != getpgid(0)) {
+        if (setpgid(0, 0) == -1) {
+            perror("Trying to create a new process group");
+            exit(1);
+        }
+    }
+    assert(getpid() == getpgid(0));
+
+    /* If subjobs are wanted, create them */
+    pid_t subjob = 0;
+    if (argc > 1) {
+        int n = atoi(argv[1]);
+        if (n > 0) {
+            switch (subjob = fork()) {
+              case -1:
+                perror("Trying to fork a subjob");
+                exit(1);
+              case 0: {
+                  char buf[20];
+                  sprintf(buf, "%d", n-1);
+                  execl("sonar-job","sonar-job",buf,NULL);
+                  perror("Trying to exec a subjob");
+                  exit(1);
+              }
+            }
+        }
+    }
+
+    /* Do the work */
     for ( int i=0 ; i < ITERATIONS ; i++ ) {
-        pid_t child = fork();
-        if (child == (pid_t)-1) {
+        pid_t worker;
+        switch (worker = fork()) {
+          case -1:
             perror("Trying to fork a new process for sonar-worker");
             exit(1);
-        }
-        if (child == 0) {
+          case 0:
             execl("sonar-worker", "sonar-worker", NULL);
             perror("Trying to exec sonar-worker");
             exit(1);
-        }
-        if (wait(NULL) == -1) {
-            perror("Waiting for sonar-worker");
-            exit(1);
+          default:
+            if (waitpid(worker, NULL, 0) == -1) {
+                perror("Waiting for sonar-worker");
+                exit(1);
+            }
         }
     }
+
+    /* Wait for the subjob to be done */
+    if (subjob > 0) {
+        waitpid(subjob, NULL, 0);
+    }
+
+    /* Wait a bit, so that information from the terminated job can be accounted to the current
+       process. */
+    printf("Waiting 10s in sonar-job for things to settle...\n");
+    sleep(10);
 }

manual-tests/sessions/sonar-session.c

@@ -8,10 +8,10 @@
 #include <sys/wait.h>
 
 int main(int argc, char **argv) {
-    /* When the shell starts this process it may make it a session leader, otherwise, become one. */
+    /* When the shell starts this process it may make it a process group leader, otherwise, become one. */
     if (getpid() != getpgid(0)) {
-        if (setsid() == (pid_t)-1) {
-            perror("Trying to become session leader");
+        if (setpgid(0, 0) == -1) {
+            perror("Trying to become process group leader");
             exit(1);
         }
     }
@@ -36,6 +36,6 @@ int main(int argc, char **argv) {
 
     /* Wait a bit, so that information from the terminated job can be accounted to the current
        process. */
-    printf("Waiting 10s for things to settle...\n");
+    printf("Waiting 10s in sonar-session for things to settle...\n");
     sleep(10);
 }

manual-tests/sessions/sonar-worker.cc

@@ -20,12 +20,12 @@ void init_matrix(T mat[N][M], int scheme) {
 }
 
 void mmul(float A[N][N], float B[N][N], float C[N][N]) {
-  for ( int kk=0 ; kk < N ; kk+=S ) {
-    for ( int jj=0 ; jj < N ; jj+=S ) {
-      for ( int i=0 ; i < N ; i++ ) {
-	for ( int j=jj ; j < jj + S ; j++ ) {
+  for ( size_t kk=0 ; kk < N ; kk+=S ) {
+    for ( size_t jj=0 ; jj < N ; jj+=S ) {
+      for ( size_t i=0 ; i < N ; i++ ) {
+	for ( size_t j=jj ; j < jj + S ; j++ ) {
 	  float sum = C[i][j];
-	  for ( int k=kk ; k < kk + S ; k++ ) {
+	  for ( size_t k=kk ; k < kk + S ; k++ ) {
 	    sum += A[i][k] * B[k][j];
 	  }
 	  C[i][j] = sum;
@@ -46,8 +46,8 @@ int main(int argc, char** argv) {
 
   // Use the result
   float sum = 0.0f;
-  for ( int j=0 ; j < N ; j++ ) {
-      for ( int i=0 ; i < N ; i++ ) {
+  for ( size_t j=0 ; j < N ; j++ ) {
+      for ( size_t i=0 ; i < N ; i++ ) {
           sum += C[j][i];
       }
   }