[testing-on-gke] Support generic fio/dlio workloads (#2268)

* update run_tests scripts based on workload config parsers * Fix argparser and dry-run support in run_tests scripts * address self-review comments * Address review comments * Fix some docstrings
GoogleCloudPlatform · Aug 20, 2024 · 9c9d6fb · 9c9d6fb
1 parent 568e07e
commit 9c9d6fb
Show file tree

Hide file tree

Showing 2 changed files with 132 additions and 60 deletions.
diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/run_tests.py b/perfmetrics/scripts/testing_on_gke/examples/dlio/run_tests.py
@@ -15,43 +15,80 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Generates and deploys helm charts for DLIO workloads.
+
+This program takes in a json test-config file, finds out valid
+DLIO workloads from it and generates and deploys a helm chart for
+each valid DLIO workload.
+"""
+
+import argparse
 import subprocess
+import dlio_workload
 
 
 def run_command(command: str):
-  result = subprocess.run(command.split(" "), capture_output=True, text=True)
+  """Runs the given string command as a subprocess."""
+  result = subprocess.run(command.split(' '), capture_output=True, text=True)
   print(result.stdout)
   print(result.stderr)
 
 
-metadataCacheTtlSecs = 6048000
-bucketName_numFilesTrain_recordLength_batchSize = [
-    ("gke-dlio-unet3d-100kb-500k", 500000, 102400, 800),
-    ("gke-dlio-unet3d-100kb-500k", 500000, 102400, 128),
-    ("gke-dlio-unet3d-500kb-1m", 1000000, 512000, 800),
-    ("gke-dlio-unet3d-500kb-1m", 1000000, 512000, 128),
-    ("gke-dlio-unet3d-3mb-100k", 100000, 3145728, 200),
-    ("gke-dlio-unet3d-150mb-5k", 5000, 157286400, 4),
-]
-
-scenarios = ["gcsfuse-file-cache", "gcsfuse-no-file-cache", "local-ssd"]
-
-for (
-    bucketName,
-    numFilesTrain,
-    recordLength,
-    batchSize,
-) in bucketName_numFilesTrain_recordLength_batchSize:
-  for scenario in scenarios:
-    commands = [
-        f"helm install {bucketName}-{batchSize}-{scenario} unet3d-loading-test",
-        f"--set bucketName={bucketName}",
-        f"--set scenario={scenario}",
-        f"--set dlio.numFilesTrain={numFilesTrain}",
-        f"--set dlio.recordLength={recordLength}",
-        f"--set dlio.batchSize={batchSize}",
-    ]
-
-    helm_command = " ".join(commands)
-
-    run_command(helm_command)
+def createHelmInstallCommands(dlioWorkloads: set) -> list:
+  """Create helm install commands for the given dlioWorkload objects."""
+  helm_commands = []
+  for dlioWorkload in dlioWorkloads:
+    for batchSize in dlioWorkload.batchSizes:
+      commands = [
+          (
+              'helm install'
+              f' {dlioWorkload.bucket}-{batchSize}-{dlioWorkload.scenario} unet3d-loading-test'
+          ),
+          f'--set bucketName={dlioWorkload.bucket}',
+          f'--set scenario={dlioWorkload.scenario}',
+          f'--set dlio.numFilesTrain={dlioWorkload.numFilesTrain}',
+          f'--set dlio.recordLength={dlioWorkload.recordLength}',
+          f'--set dlio.batchSize={batchSize}',
+      ]
+
+      helm_command = ' '.join(commands)
+      helm_commands.append(helm_command)
+  return helm_commands
+
+
+def main(args) -> None:
+  dlioWorkloads = dlio_workload.ParseTestConfigForDlioWorkloads(
+      args.workload_config
+  )
+  helmInstallCommands = createHelmInstallCommands(dlioWorkloads)
+  for helmInstallCommand in helmInstallCommands:
+    print(f'{helmInstallCommand}')
+    if not args.dry_run:
+      run_command(helmInstallCommand)
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(
+      prog='DLIO Unet3d test runner',
+      description=(
+          'This program takes in a json test-config file, finds out valid DLIO'
+          ' workloads from it and generates and deploys a helm chart for each'
+          ' DLIO workload.'
+      ),
+  )
+  parser.add_argument(
+      '--workload-config',
+      help='Runs DLIO Unet3d tests using this JSON workload configuration.',
+      required=True,
+  )
+  parser.add_argument(
+      '-n',
+      '--dry-run',
+      action='store_true',
+      help=(
+          'Only print out the test configurations that will run,'
+          ' not actually run them.'
+      ),
+  )
+  args = parser.parse_args()
+  main(args)
diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/run_tests.py b/perfmetrics/scripts/testing_on_gke/examples/fio/run_tests.py
@@ -15,46 +15,81 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Generates and deploys helm charts for FIO workloads.
+
+This program takes in a json test-config file, finds out valid FIO workloads in
+it and generates and deploys a helm chart for each valid FIO workload.
+"""
+
+import argparse
 import subprocess
+import fio_workload
 
 
 def run_command(command: str):
-  result = subprocess.run(command.split(" "), capture_output=True, text=True)
+  """Runs the given string command as a subprocess."""
+  result = subprocess.run(command.split(' '), capture_output=True, text=True)
   print(result.stdout)
   print(result.stderr)
 
 
-bucketName_fileSize_blockSize = [
-    ("gke-fio-64k-1m", "64K", "64K"),
-    ("gke-fio-128k-1m", "128K", "128K"),
-    ("gke-fio-1mb-1m", "1M", "256K"),
-    ("gke-fio-100mb-50k", "100M", "1M"),
-    ("gke-fio-200gb-1", "200G", "1M"),
-]
-
-scenarios = ["gcsfuse-file-cache", "gcsfuse-no-file-cache", "local-ssd"]
-
-for bucketName, fileSize, blockSize in bucketName_fileSize_blockSize:
-  for readType in ["read", "randread"]:
-    for scenario in scenarios:
-      if readType == "randread" and fileSize in ["64K", "128K"]:
-        continue
-
+def createHelmInstallCommands(fioWorkloads: set) -> list:
+  """Create helm install commands for the given fioWorkload objects."""
+  helm_commands = []
+  for fioWorkload in fioWorkloads:
+    for readType in fioWorkload.readTypes:
       commands = [
           (
-              "helm install"
-              f" fio-loading-test-{fileSize.lower()}-{readType}-{scenario} loading-test"
+              'helm install'
+              f' fio-loading-test-{fioWorkload.fileSize.lower()}-{readType}-{fioWorkload.scenario} loading-test'
           ),
-          f"--set bucketName={bucketName}",
-          f"--set scenario={scenario}",
-          f"--set fio.readType={readType}",
-          f"--set fio.fileSize={fileSize}",
-          f"--set fio.blockSize={blockSize}",
+          f'--set bucketName={fioWorkload.bucket}',
+          f'--set scenario={fioWorkload.scenario}',
+          f'--set fio.readType={readType}',
+          f'--set fio.fileSize={fioWorkload.fileSize}',
+          f'--set fio.blockSize={fioWorkload.blockSize}',
+          f'--set fio.filesPerThread={fioWorkload.filesPerThread}',
+          f'--set fio.numThreads={fioWorkload.numThreads}',
       ]
 
-      if fileSize == "100M":
-        commands.append("--set fio.filesPerThread=1000")
+      helm_command = ' '.join(commands)
+      helm_commands.append(helm_command)
+  return helm_commands
+
+
+def main(args) -> None:
+  fioWorkloads = fio_workload.ParseTestConfigForFioWorkloads(
+      args.workload_config
+  )
+  helmInstallCommands = createHelmInstallCommands(fioWorkloads)
+  for helmInstallCommand in helmInstallCommands:
+    print(f'{helmInstallCommand}')
+    if not args.dry_run:
+      run_command(helmInstallCommand)
 
-      helm_command = " ".join(commands)
 
-      run_command(helm_command)
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(
+      prog='FIO test runner',
+      description=(
+          'This program takes in a json test-config file, finds out valid FIO'
+          ' workloads from it and generates and deploys a helm chart for each'
+          ' FIO workload.'
+      ),
+  )
+  parser.add_argument(
+      '--workload-config',
+      help='Runs FIO tests using this JSON workload configuration',
+      required=True,
+  )
+  parser.add_argument(
+      '-n',
+      '--dry-run',
+      action='store_true',
+      help=(
+          'Only print out the test configurations that will run,'
+          ' not actually run them.'
+      ),
+  )
+  args = parser.parse_args()
+  main(args)