Add mixed read/write tests to the benchmark.

As we make changes that might impact mixed read/write workloads it is
useful to have some examples of these in our benchmarks.  This change
adds a 20/80, 50/50, and 80/20 read/write workload to the benchmarks.

The results reporting is updated to support multiple job types in a
single benchmark.  This works by continuing to average over iterations
as before, but averages each job separately then sums the averages to
produce the final throughput number for that benchmark (i.e. (avg(read
throughput) + avg(write throughput)) for the mixed benchmarks).

Signed-off-by: Andrew Peace <adpeace@amazon.com>

mountpoint-s3/scripts/fio/mix/mix_1r4w.fio

@@ -0,0 +1,24 @@
+[global]
+name=fs_bench
+bs=256k
+runtime=30s
+time_based
+group_reporting
+
+[sequential_read]
+size=100G
+rw=read
+ioengine=sync
+fallocate=none
+numjobs=1
+
+[sequential_write_four_threads]
+new_group
+numjobs=4
+size=100G
+rw=write
+ioengine=sync
+fallocate=none
+create_on_open=1
+fsync_on_close=1
+unlink=1

mountpoint-s3/scripts/fio/mix/mix_2r2w.fio

@@ -0,0 +1,24 @@
+[global]
+name=fs_bench
+bs=256k
+runtime=30s
+time_based
+group_reporting
+
+[sequential_read_two_threads]
+size=100G
+rw=read
+ioengine=sync
+fallocate=none
+numjobs=2
+
+[sequential_write_two_threads]
+new_group
+numjobs=2
+size=100G
+rw=write
+ioengine=sync
+fallocate=none
+create_on_open=1
+fsync_on_close=1
+unlink=1

mountpoint-s3/scripts/fio/mix/mix_4r1w.fio

@@ -0,0 +1,24 @@
+[global]
+name=fs_bench
+bs=256k
+runtime=30s
+time_based
+group_reporting
+
+[sequential_read_four_threads]
+size=100G
+rw=read
+ioengine=sync
+fallocate=none
+numjobs=4
+
+[sequential_write]
+new_group
+numjobs=1
+size=100G
+rw=write
+ioengine=sync
+fallocate=none
+create_on_open=1
+fsync_on_close=1
+unlink=1

mountpoint-s3/scripts/fs_bench.sh

@@ -37,7 +37,7 @@ cd ${project_dir}
 results_dir=results
 runtime_seconds=30
 startdelay_seconds=30
-iterations=10
+: ${iterations:=10}
 
 rm -rf ${results_dir}
 mkdir -p ${results_dir}
@@ -74,13 +74,28 @@ run_fio_job() {
   done
   echo "done"
 
-  # combine the results and find an average value
-  jq -n 'reduce inputs.jobs[] as $job (null; .name = $job.jobname | .len += 1 | .value += (if ($job."job options".rw == "read")
-      then $job.read.bw / 1024
-      elif ($job."job options".rw == "randread") then $job.read.bw / 1024
-      elif ($job."job options".rw == "randwrite") then $job.write.bw / 1024
-      elif ($job."job options".rw | startswith("read")) then $job.read.bw / 1024
-      else $job.write.bw / 1024 end)) | {name: .name, value: (.value / .len), unit: "MiB/s"}' ${results_dir}/${job_name}_iter*.json | tee ${results_dir}/${job_name}_parsed.json
+  # combine the results and find an average value.  If the test was a single job, take the
+  # average across all iterations.  If it was multiple jobs, take the average across all
+  # iterations of each job, then sum the averages to get the final results.  For example,
+  # for a mixed read/write test, this would average all read jobs, all write jobs, and
+  # combine the two averages into a throughput.
+  jq -s '[
+      [.[].jobs] | add | group_by(.jobname)[] |
+          {
+              name: .[0].jobname,
+              value: ((map(
+                  if .["job options"].rw | test("^(rand)?read")
+                  then .read.bw
+                  else .write.bw
+                  end
+              ) | add) / (length * 1024))
+          }
+      ] | {
+          name: map(.name) | unique | join(","),
+          value: map(.value) | add,
+          unit: "MiB/s"
+      }' \
+      ${results_dir}/${job_name}_iter*.json | tee ${results_dir}/${job_name}_parsed.json
 }
 
 should_run_job() {
@@ -179,6 +194,7 @@ run_benchmarks() {
 
 run_benchmarks read
 run_benchmarks write
+run_benchmarks mix
 
 # combine all bench results into one json file
 echo "Throughput:"