From 8d5a9464e1c6fca6d5c682acba57000e0248ec9b Mon Sep 17 00:00:00 2001
From: Federico Mastellone <federico.pascual-mastellone@iohk.io>
Date: Tue, 20 Aug 2024 17:03:30 +0000
Subject: [PATCH 1/7] wb | remove nomad podman backend

---
 Makefile                               |   2 +-
 lib.mk                                 |   2 -
 nix/pkgs.nix                           |   2 -
 nix/workbench/backend/nomad-job.nix    | 240 ++++++-----------
 nix/workbench/backend/nomad.nix        |  39 ---
 nix/workbench/backend/nomad.sh         | 310 +++++-----------------
 nix/workbench/backend/nomad/cloud.nix  |   6 +-
 nix/workbench/backend/nomad/cloud.sh   |   2 -
 nix/workbench/backend/nomad/exec.nix   |   6 +-
 nix/workbench/backend/nomad/exec.sh    |   6 +-
 nix/workbench/backend/nomad/podman.nix | 101 --------
 nix/workbench/backend/nomad/podman.sh  | 340 -------------------------
 nix/workbench/backend/oci-images.nix   |  49 ----
 nix/workbench/nomad.sh                 | 270 +++-----------------
 nix/workbench/wb                       |   1 -
 15 files changed, 180 insertions(+), 1196 deletions(-)
 delete mode 100644 nix/workbench/backend/nomad/podman.nix
 delete mode 100644 nix/workbench/backend/nomad/podman.sh
 delete mode 100644 nix/workbench/backend/oci-images.nix

diff --git a/Makefile b/Makefile
index 7e0dc77a729..e833f85422e 100644
--- a/Makefile
+++ b/Makefile
@@ -41,7 +41,7 @@ trace-documentation:
 ###
 ### Workbench
 ###
-workbench-ci: workbench-ci-test ci-test-auto ci-test-autonix ci-test-autonomadpodman
+workbench-ci: workbench-ci-test ci-test-auto ci-test-autonix
 CI_TARGETS := hlint workbench-ci haddock-hoogle
 ci:  ci-report ci-targets
 ci-report:
diff --git a/lib.mk b/lib.mk
index 9241a42a92e..f0fe8572888 100644
--- a/lib.mk
+++ b/lib.mk
@@ -42,8 +42,6 @@ $$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nix,              $$(prof)
 $$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-autonix,          $$(prof), true,false, true,false, false, supervisor)))
 $$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nomadexec,        $$(prof), true,false,false,false, false, nomadexec)))
 $$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nomadexec-auto,   $$(prof), true,false, true,false, false, nomadexec)))
-$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nomadpodman,      $$(prof), true,false,false,false, false, nomadpodman)))
-$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nomadpodman-auto, $$(prof), true,false, true,false, false, nomadpodman)))
 endef
 
 define define_profile_targets_nomadcloud
diff --git a/nix/pkgs.nix b/nix/pkgs.nix
index 7605f1312b9..c0e16143cff 100644
--- a/nix/pkgs.nix
+++ b/nix/pkgs.nix
@@ -23,8 +23,6 @@ let
           import ./workbench/backend/nomad/cloud.nix  params;
         nomadexec       = params:
           import ./workbench/backend/nomad/exec.nix   params;
-        nomadpodman     = params:
-          import ./workbench/backend/nomad/podman.nix params;
         supervisor      = params:
           import ./workbench/backend/supervisor.nix   params;
       }
diff --git a/nix/workbench/backend/nomad-job.nix b/nix/workbench/backend/nomad-job.nix
index 11ff8b1684d..7d2a0d57968 100644
--- a/nix/workbench/backend/nomad-job.nix
+++ b/nix/workbench/backend/nomad-job.nix
@@ -8,7 +8,6 @@
 , stateDir
 , profileData
 , containerSpecs
-, execTaskDriver
 , generatorTaskName
 , oneTracerPerNode ? false
 , withSsh ? false
@@ -21,8 +20,8 @@ let
   # Nomad creates a working directory for each allocation on a client. This
   # directory can be found in the Nomad data_dir at ./alloc/«alloc_id». The
   # allocation working directory is where Nomad creates task directories and
-  # directories shared between tasks, write logs for tasks, and downloads
-  # artifacts or templates.
+  # directories shared between tasks, write logs for tasks, downloads artifacts
+  # and renders templates.
   # https://developer.hashicorp.com/nomad/docs/concepts/filesystem
   #
   # For example:
@@ -36,34 +35,26 @@ let
   # Templates are rendered into the task working directory. Drivers without
   # filesystem isolation (such as raw_exec) or drivers that build a chroot in
   # the task working directory (such as exec) can have templates rendered to
-  # arbitrary paths in the task. But task drivers such as docker can only access
-  # templates rendered into the NOMAD_ALLOC_DIR, NOMAD_TASK_DIR, or
-  # NOMAD_SECRETS_DIR. To work around this restriction, you can create a mount
-  # from the template destination to another location in the task.
+  # arbitrary paths in the task.
   ## - https://developer.hashicorp.com/nomad/docs/job-specification/template#template-destinations
   ## - https://developer.hashicorp.com/nomad/docs/runtime/environment#task-directories
   ## - https://developer.hashicorp.com/nomad/docs/concepts/filesystem
 
-  # Task's filesystem / working directory (maybe container or chroot) defaults:
+  # Workbench's ${stateDir} will be created/populated inside NOMAD_TASK_DIR.
   #
-  # When using the isolated fork task driver ("exec")
-  ## Default values below are stored in the job's "meta" stanza to be able to
-  ## overrided them with 'jq' from a workbench shell. These values in "meta"
-  ## are used to programatically create a "template" with "env = true;" so they
-  ## are automagically reachable as envars inside the Task's entrypoint and
-  ## 'supervisord' programs.
+  ## Some workbench default values are stored in the job's "meta" stanza to be
+  ## able to override them with 'jq' from a workbench shell. These values in
+  ## "meta" are used to programmatically create a "template" with "env = true;"
+  ## so they are automagically reachable as envars inside the Task's entrypoint
+  ## and/or 'supervisord' programs.
   ## Values go: Nix (defaults) -> meta -> template -> envars
-  #
-  ## See ./oci-images.nix for further details if using the `podman` driver.
-  ## For the `exec` driver almost everything is here.
-  #
 
-  # A symlink to the supervisord nix-installed inside the OCI image/chroot.
-  # We need to be able to `nomad exec supervisorctl ...` , for this the path
+  # Symlink to the supervisord that is nix-installed inside the deployed chroot.
+  # We need to be able to do `nomad exec supervisorctl ...` , for this the path
   # of the installed supervisor binaries is needed.
   task_supervisor_nix = "${stateDir}/supervisor/nix-store";
   # Location of the supervisord config file inside the container.
-  # This file can be mounted as a volume or created as a template.
+  # This file is created as a template.
   task_supervisord_conf = "${stateDir}/supervisor/supervisord.conf";
   # The URL to the listening inet or socket of the supervisord server:
   # The problem is that if we use "127.0.0.1:9001" as parameter (without the
@@ -111,11 +102,8 @@ let
         }" \
       > "''${NOMAD_TASK_DIR}"/entrypoint.dirs
 
-      # Only needed for "exec" ?
-      if test "''${TASK_DRIVER}" = "exec"
-      then
-        cd "''${NOMAD_TASK_DIR}"
-      fi
+      # Move to stateDir's parent directory.
+      cd "''${NOMAD_TASK_DIR}"
 
       # Create a symlink to 'supervisor' Nix Store folder so we can call it from
       # 'ssh' or 'nomad exec' without having it in PATH or knowing the currently
@@ -141,7 +129,7 @@ let
 
   # About the JSON Job Specification and my odd assumptions:
   #
-  # TL;DR; We are using what HashiCorp calls an unespecified format but it's the
+  # TL;DR; We are using what HashiCorp calls an unspecified format but it's the
   # same format the SRE team is using.
   #
   # At least in Nomad version v1.4.3, the CLI command to submit new jobs
@@ -250,7 +238,6 @@ let
     meta = {
       # Only top level "KEY=STRING" are allowed, no child objects/attributes!
       WORKBENCH_STATEDIR = stateDir;
-      TASK_DRIVER = if execTaskDriver then "exec" else "podman";
       SUPERVISORD_LOGLEVEL = task_supervisord_loglevel;
       ONE_TRACER_PER_NODE = oneTracerPerNode;
     };
@@ -411,8 +398,8 @@ let
           # Actually using the interface specified on Nomad Client startup that
           # for local runs it's forced to "lo" and whatever is automatically
           # fingerprinted or provided for cloud runs.
-          # TODO: Use "bridge" mode for podman, this will allow to run isolated
-          # local cluster with no addresses or ports clashing.
+          # TODO: Use "bridge" mode for local ?? this will allow to run isolated
+          # local cluster with no addresses or ports clashing ??
           mode = "host";
           # Specifies a TCP/UDP port allocation and can be used to specify both
           # dynamic ports and reserved ports.
@@ -453,18 +440,14 @@ let
                       # jobs like load balancers.
                       static = ''${toString portNum}'';
 
-                      # TODO: When switching the network mode to "bridge" for
-                      # podman use "Mapped Ports" to be able to run isolated
-                      # local cluster with no addresses or ports clashing.
+                      # The "exec" driver does not accept "Mapped Ports".
+                      # to = ''${toString portNum}'';
                       # Applicable when using "bridge" mode to configure port
                       # to map to inside the task's network namespace. Omitting
                       # this field or setting it to -1 sets the mapped port
                       # equal to the dynamic port allocated by the scheduler.
                       # The NOMAD_PORT_<label> environment variable will contain
                       # the to value.
-                      # to = ''${toString portNum}'';
-                      # The "podman" driver accepts "Mapped Ports", but not the
-                      # "exec" driver
                       # https://developer.hashicorp.com/nomad/docs/job-specification/network#mapped-ports
                       # https://developer.hashicorp.com/nomad/docs/job-specification/network#bridge-mode
                     }
@@ -639,7 +622,6 @@ let
           template = [
             # Envars
             {
-              # podman container input environment variables.
               env = true;
               # File name to create inside the allocation directory.
               # Created in NOMAD_DATA_DIR/alloc/ALLOC_ID/TASK_NAME/envars
@@ -647,7 +629,6 @@ let
               # See runtime for available variables:
               # https://developer.hashicorp.com/nomad/docs/runtime/environment
               data = ''
-                TASK_DRIVER="{{ env "NOMAD_META_TASK_DRIVER" }}"
                 WORKBENCH_STATEDIR="{{ env "NOMAD_META_WORKBENCH_STATEDIR" }}"
                 SUPERVISORD_LOGLEVEL="{{ env "NOMAD_META_SUPERVISORD_LOGLEVEL" }}"
               '';
@@ -806,13 +787,11 @@ let
               destination = "local/${stateDir}/${nodeSpec.name}/start.sh";
               data = escapeTemplate (
                 let scriptValue = profileData.node-services."${nodeSpec.name}".start.value;
-                in if execTaskDriver
-                  then (startScriptToGoTemplate
+                in (startScriptToGoTemplate
                     nodeSpec                         # nodeSpec
                     servicePortName                  # servicePortName
                     scriptValue                      # startScript
                   )
-                  else scriptValue
               );
               change_mode = "noop";
               error_on_missing_key = true;
@@ -832,36 +811,32 @@ let
               env = false;
               destination = "local/${stateDir}/${nodeSpec.name}/topology.json";
               data = escapeTemplate (
-                if execTaskDriver
-                then
-                  # Recreate the "topology.json" with IPs and ports that are
-                  # nomad template variables.
-                  (topologyToGoTemplate
-                    # Fetch this node's entry in the profile's "topology.json".
-                    (if nodeSpec.name != "explorer"
-                     then
-                      # Non explorer nodes are under "coreNodes"
-                      builtins.head # Must exist!
-                        (builtins.filter
-                          (coreNode: coreNode.name == nodeSpec.name)
-                          profileData.topology.value.coreNodes
-                        )
-                     else
-                      # The explorer node is under "relayNodes"
-                      builtins.head # Must exist!
-                        (builtins.filter
-                          (coreNode: coreNode.name == nodeSpec.name)
-                          profileData.topology.value.relayNodes
-                        )
-                    )
-                    # The P2P flag.
-                    (if profileData.value.node ? verbatim && profileData.value.node.verbatim ? EnableP2P
-                     then profileData.value.node.verbatim.EnableP2P
-                     else false
-                    )
+                # Recreate the "topology.json" with IPs and ports that are
+                # nomad template variables.
+                (topologyToGoTemplate
+                  # Fetch this node's entry in the profile's "topology.json".
+                  (if nodeSpec.name != "explorer"
+                   then
+                    # Non explorer nodes are under "coreNodes"
+                    builtins.head # Must exist!
+                      (builtins.filter
+                        (coreNode: coreNode.name == nodeSpec.name)
+                        profileData.topology.value.coreNodes
+                      )
+                   else
+                    # The explorer node is under "relayNodes"
+                    builtins.head # Must exist!
+                      (builtins.filter
+                        (coreNode: coreNode.name == nodeSpec.name)
+                        profileData.topology.value.relayNodes
+                      )
                   )
-                # Do nothing with the topology files.
-                else (__readFile profileData.node-services."${nodeSpec.name}".topology.JSON )
+                  # The P2P flag.
+                  (if profileData.value.node ? verbatim && profileData.value.node.verbatim ? EnableP2P
+                   then profileData.value.node.verbatim.EnableP2P
+                   else false
+                  )
+                )
               );
               change_mode = "noop";
               error_on_missing_key = true;
@@ -886,23 +861,21 @@ let
               destination = "local/${stateDir}/generator/run-script.json";
               data = escapeTemplate (
                 let runScript = profileData.generator-service.config;
-                in if execTaskDriver
-                  # Recreate the "run-script.json" with IPs and ports that are
-                  # nomad template variables.
-                  then (runScriptToGoTemplate
-                    runScript.value
-                    # Just the node names.
-                    (lib.attrsets.mapAttrsToList
-                      (nodeSpecNodeName: nodeSpecNode: nodeSpecNodeName)
-                      # All the producer nodes. How the workbench creates it.
-                      (lib.attrsets.filterAttrs
-                        (nodeSpecNodeName: nodeSpecNode: nodeSpecNode.isProducer)
-                        profileData.node-specs.value
-                      )
-                    )
-                  )
-                  # Do nothing with the topology files.
-                  else (__readFile runScript.JSON)
+                in
+                   # Recreate the "run-script.json" with IPs and ports that are
+                   # nomad template variables.
+                   (runScriptToGoTemplate
+                     runScript.value
+                     # Just the node names.
+                     (lib.attrsets.mapAttrsToList
+                       (nodeSpecNodeName: nodeSpecNode: nodeSpecNodeName)
+                       # All the producer nodes. How the workbench creates it.
+                       (lib.attrsets.filterAttrs
+                         (nodeSpecNodeName: nodeSpecNode: nodeSpecNode.isProducer)
+                         profileData.node-specs.value
+                       )
+                     )
+                   )
               );
               change_mode = "noop";
               error_on_missing_key = true;
@@ -1017,86 +990,19 @@ let
 
           }
           //
-          (if execTaskDriver
-            then {
-              driver = "exec";
-
-              config = {
-
-                command = "${containerSpecs.containerPkgs.bashInteractive.nix-store-path}/bin/bash";
-
-                args = ["local/entrypoint.sh"];
-
-                nix_installables =
-                  (lib.attrsets.mapAttrsToList
-                    (name: attr: attr.nix-store-path)
-                    containerSpecs.containerPkgs
-                  )
-                ;
-
-              };
-            } else {
-              driver = "podman";
-
-              # Specifies the driver configuration, which is passed directly to the
-              # driver to start the task. The details of configurations are specific
-              # to each driver, so please see specific driver documentation for more
-              # information.
-              # https://github.com/hashicorp/nomad-driver-podman#task-configuration
-              config = {
-
-                command = "${containerSpecs.containerPkgs.bashInteractive.nix-store-path}/bin/bash";
-
-                args = ["local/entrypoint.sh"];
-
-                # The image to run. Accepted transports are docker (default if
-                # missing), oci-archive and docker-archive. Images reference as
-                # short-names will be treated according to user-configured
-                # preferences.
-                image = "${containerSpecs.ociImage.imageName}:${containerSpecs.ociImage.imageTag}";
-
-                # Always pull the latest image on container start.
-                force_pull = false;
-
-                # Podman redirects its combined stdout/stderr logstream directly
-                # to a Nomad fifo. Benefits of this mode are: zero overhead,
-                # don't have to worry about log rotation at system or Podman
-                # level. Downside: you cannot easily ship the logstream to a log
-                # aggregator plus stdout/stderr is multiplexed into a single
-                # stream.
-                logging = {
-                  # The other option is: "journald"
-                  driver = "nomad";
-                };
-
-                # The hostname to assign to the container. When launching more
-                # than one of a task (using count) with this option set, every
-                # container the task starts will have the same hostname.
-                hostname = taskName;
-
-                network_mode = "host";
-
-                # This can be used here but not with "exec"!
-                # These name cannot be used for envars, "-" replaced with "_".
-                ports = [ servicePortName ];
-
-                # A list of /container_path strings for tmpfs mount points. See
-                # podman run --tmpfs options for details.
-                tmpfs = [
-                  "/tmp"
-                ];
-
-                # A list of host_path:container_path:options strings to bind
-                # host paths to container paths. Named volumes are not supported.
-                volumes = [];
-
-                # The working directory for the container. Defaults to the
-                # default set in the image.
-                working_dir = "local/";
-
-              };
-            }
-          );
+          {
+            driver = "exec";
+            config = {
+              command = "${containerSpecs.containerPkgs.bashInteractive.nix-store-path}/bin/bash";
+              args = ["local/entrypoint.sh"];
+              nix_installables =
+                (lib.attrsets.mapAttrsToList
+                  (name: attr: attr.nix-store-path)
+                  containerSpecs.containerPkgs
+                )
+              ;
+            };
+          };
       }));
       in lib.listToAttrs (
         # If not oneTracerPerNode, an individual tracer task is needed (instead
diff --git a/nix/workbench/backend/nomad.nix b/nix/workbench/backend/nomad.nix
index 9e234e94e1f..17f21552fbb 100644
--- a/nix/workbench/backend/nomad.nix
+++ b/nix/workbench/backend/nomad.nix
@@ -160,40 +160,9 @@ let
               [ diretories.run "supervisor" "supervisord.conf"]
             ;
           };
-          ociImage = import ./oci-images.nix
-            { inherit pkgs lib;
-              inherit containerPkgs;
-            }
-          ;
           nomadJob =
             { inherit generatorTaskName; }
             //
-            lib.attrsets.optionalAttrs (subBackendName == "podman") {
-              podman = {
-                # TODO: oneTracerPerGroup
-                oneTracerPerCluster = import ./nomad-job.nix
-                  { inherit pkgs lib stateDir;
-                    inherit profileData;
-                    inherit containerSpecs;
-                    # May evolve to a "cloud" flag!
-                    execTaskDriver = false;
-                    inherit generatorTaskName;
-                    oneTracerPerNode = false;
-                    withSsh = false;
-                  };
-                oneTracerPerNode = import ./nomad-job.nix
-                  { inherit pkgs lib stateDir;
-                    inherit profileData;
-                    inherit containerSpecs;
-                    # May evolve to a "cloud" flag!
-                    execTaskDriver = false;
-                    inherit generatorTaskName;
-                    oneTracerPerNode = true;
-                    withSsh = false;
-                  };
-              };
-            }
-            //
             lib.attrsets.optionalAttrs (subBackendName == "exec") {
               exec = {
                 # TODO: oneTracerPerGroup
@@ -201,8 +170,6 @@ let
                   { inherit pkgs lib stateDir;
                     inherit profileData;
                     inherit containerSpecs;
-                    # May evolve to a "cloud" flag!
-                    execTaskDriver = true;
                     inherit generatorTaskName;
                     oneTracerPerNode = false;
                     withSsh = false;
@@ -211,8 +178,6 @@ let
                   { inherit pkgs lib stateDir;
                     inherit profileData;
                     inherit containerSpecs;
-                    # May evolve to a "cloud" flag!
-                    execTaskDriver = true;
                     inherit generatorTaskName;
                     oneTracerPerNode = true;
                     withSsh = false;
@@ -228,8 +193,6 @@ let
                   { inherit pkgs lib stateDir;
                     inherit profileData;
                     inherit containerSpecs;
-                    # May evolve to a "cloud" flag!
-                    execTaskDriver = true;
                     inherit generatorTaskName;
                     oneTracerPerNode = true;
                     withSsh = false;
@@ -238,8 +201,6 @@ let
                   { inherit pkgs lib stateDir;
                     inherit profileData;
                     inherit containerSpecs;
-                    # May evolve to a "cloud" flag!
-                    execTaskDriver = true;
                     inherit generatorTaskName;
                     oneTracerPerNode = true;
                     withSsh = true;
diff --git a/nix/workbench/backend/nomad.sh b/nix/workbench/backend/nomad.sh
index 0e9a3d2e6f0..1d9554ee620 100644
--- a/nix/workbench/backend/nomad.sh
+++ b/nix/workbench/backend/nomad.sh
@@ -43,7 +43,7 @@ backend_nomad() {
     ############################################################################
     # * Functions in the backend "interface" must use `fatal` when errors!
 
-    # Completely overrided by each sub-backend (exec.sh, podman.sh and cloud.sh)
+    # Completely overrided by each sub-backend (exec.sh and cloud.sh)
     setenv-defaults )
       # The impossible just happened?
       fatal "Function \"setenv-defaults\" is Nomad backend specific"
@@ -56,7 +56,7 @@ backend_nomad() {
     #  (start.sh, config files, etc) are included in the Nomad Job spec file as
     # "template" stanzas and are materialized inside the container when the job
     # is started. This is how it works for every environment combination
-    # (podman/exec-(local/cloud)).
+    # (exec-(local/cloud)).
     #
     # But "genesis" and "CARDANO_MAINNET_MIRROR" are the deployment exceptions:
     # - "CARDANO_MAINNET_MIRROR": is added as a Nix dependency using the
@@ -86,23 +86,21 @@ backend_nomad() {
       backend_nomad allocate-run-directory-healthchecks "${dir}"
       backend_nomad allocate-run-directory-latencies    "${dir}"
 
-      # These ones are decided at "setenv-defaults" of each sub-backend.
+      # This one is decided at "setenv-defaults" of each sub-backend.
       local nomad_environment=$(envjqr 'nomad_environment')
-      local nomad_task_driver=$(envjqr 'nomad_task_driver')
-      # TODO: Store them on disk for later subcommands run from a different shell.
+      # TODO: Store it on disk so commands can run from a different shell.
       # echo "{\"nomad_environment\": $nomad_environment, }" > "$dir"/env.json
     ;;
 
     allocate-run-directory-nomad )
       local usage="USAGE: wb backend $op RUN-DIR"
       local dir=${1:?$usage}; shift
-      local nomad_task_driver=$(envjqr   'nomad_task_driver')
       local one_tracer_per_node=$(envjqr 'one_tracer_per_node')
       # Creates Nomad specific folders to download the entrypoints scripts and
       # its logs for every Nomad Task.
       # The top level "nomad" folder is created at "allocate-run" of each
-      # sub-backend. ("podman", "exec", "cloud") and filled with the Nomad job
-      # spec file to use.
+      # sub-backend. ("exec", "cloud") and filled with the Nomad job spec file
+      # to use.
       local nodes=($(jq_tolist keys "${dir}"/node-specs.json))
       for node in ${nodes[*]}
       do
@@ -145,7 +143,6 @@ backend_nomad() {
     allocate-run-directory-tracers )
       local usage="USAGE: wb backend $op RUN-DIR"
       local dir=${1:?$usage}; shift
-      local nomad_task_driver=$(envjqr   'nomad_task_driver')
       local one_tracer_per_node=$(envjqr 'one_tracer_per_node')
       # A "tracer"(s) is optional.
       if jqtest ".node.tracer" "${dir}"/profile.json
@@ -154,9 +151,7 @@ backend_nomad() {
         mkdir -p "${dir}"/tracer
         # If running "local" without "one_tracer_per_node" this directory will
         # be populated using the workbench!
-        # Right now where are forcing "one_tracer_per_node=true" with "exec"
-        # For podman, than only runs local, this tracer directory is populated
-        # and mounted to every container when only one shared tracer is used.
+        # Right now we are forcing "one_tracer_per_node=true" with "exec"
         if test "${one_tracer_per_node}" = "true"
         then
           local nodes=($(jq_tolist keys "${dir}"/node-specs.json))
@@ -164,10 +159,6 @@ backend_nomad() {
           do
             mkdir -p "${dir}"/tracer/"${node}"
           done
-        elif test "${nomad_task_driver}" = "podman"
-        then
-          cp $(jq '."config"' -r ${dir}/profile/tracer-service.json) "${dir}"/tracer/config.json
-          cp $(jq '."start"'  -r ${dir}/profile/tracer-service.json) "${dir}"/tracer/start.sh
         fi
       fi
     ;;
@@ -495,7 +486,7 @@ backend_nomad() {
       fi
     ;;
 
-    # Completely overrided by each sub-backend (exec.sh, podman.sh and cloud.sh)
+    # Completely overrided by each sub-backend (exec.sh and cloud.sh)
     deploy-genesis )
       # The impossible just happened?
       fatal "Function \"deploy-genesis\" is Nomad backend specific"
@@ -586,7 +577,6 @@ backend_nomad() {
       backend_nomad nomad-agents-topology "${dir}"
 
       local nomad_environment=$(envjqr 'nomad_environment')
-      local nomad_task_driver=$(envjqr 'nomad_task_driver')
       local one_tracer_per_node=$(envjqr 'one_tracer_per_node')
       local server_name=$(envjqr 'nomad_server_name')
       local client_name=$(envjqr 'nomad_client_name')
@@ -610,8 +600,7 @@ backend_nomad() {
           nomad_agents_were_already_running="false"
           setenvjqstr 'nomad_agents_were_already_running' "false"
           # Start server, client and plugins.
-          wb_nomad agents start \
-            "${server_name}" "${client_name}" "${nomad_task_driver}"
+          wb_nomad agents start "${server_name}" "${client_name}"
         fi
       fi
 
@@ -633,8 +622,7 @@ backend_nomad() {
         if test "${nomad_agents_were_already_running}" = "false"
         then
           # No checks, `agents stop` checks for errors and uses msg to show them!
-          wb_nomad agents stop \
-            "${server_name}" "${client_name}" "${nomad_task_driver}"
+          wb_nomad agents stop "${server_name}" "${client_name}"
         fi
         fatal "Failed to start Nomad job"
       else
@@ -678,7 +666,6 @@ backend_nomad() {
 
     stop-nomad-job )
       local dir=${1:?$usage}; shift
-      local nomad_task_driver=$(envjqr                 'nomad_task_driver')
       local server_name=$(envjqr                       'nomad_server_name')
       local client_name=$(envjqr                       'nomad_client_name')
       local nomad_agents_were_already_running=$(envjqr 'nomad_agents_were_already_running')
@@ -689,15 +676,13 @@ backend_nomad() {
       if test "${nomad_agents_were_already_running}" = "false"
       then
         # No checks, `agents stop` checks for errors and uses msg to show them!
-        wb_nomad agents stop \
-          "${server_name}" "${client_name}" "${nomad_task_driver}"
+        wb_nomad agents stop "${server_name}" "${client_name}"
       fi
     ;;
 
     nomad-agents-topology )
       local usage="USAGE: wb backend $op RUN-DIR"
       local dir=${1:?$usage}; shift
-      local nomad_task_driver=$(envjqr 'nomad_task_driver')
       local nomad_job_name=$(jq -r ". [\"job\"] | keys[0]" "${dir}"/nomad/nomad-job.json)
       # The server
       echo "{                                ">> "${dir}"/nomad/agents-topolgy.json
@@ -713,19 +698,11 @@ backend_nomad() {
       echo "      }                          ">> "${dir}"/nomad/agents-topolgy.json
       echo "    }                            ">> "${dir}"/nomad/agents-topolgy.json
       echo "  , \"clients\": {               ">> "${dir}"/nomad/agents-topolgy.json
-      # Task driver based suffix for clients (exec clients run with `sudo`)
-      local suffix
-      if test "${nomad_task_driver}" = "podman"
-      then
-        suffix="pod"
-      else
-        suffix="exe"
-      fi
       local i=0 c=""
       local groups_array=$(jq -r ".[\"job\"][\"${nomad_job_name}\"][\"group\"] | keys | join (\" \")" "${dir}"/nomad/nomad-job.json)
       for group_name in ${groups_array[*]}
       do
-        local client_name=("cli${group_name}-${suffix}")
+        local client_name=("cli${group_name}")
       echo " ${c} \"${client_name}\": {      ">> "${dir}"/nomad/agents-topolgy.json
       echo "          \"region\": \"r1\"     ">> "${dir}"/nomad/agents-topolgy.json
       echo "        , \"datacenter\": \"d1\" ">> "${dir}"/nomad/agents-topolgy.json
@@ -1036,7 +1013,7 @@ backend_nomad() {
       fi
     ;;
 
-    # Completely overrided by each sub-backend (exec.sh, podman.sh and cloud.sh)
+    # Completely overrided by each sub-backend (exec.sh and cloud.sh)
     stop-cluster )
       # The impossible just happened?
       fatal "Function \"stop-cluster\" is Nomad backend specific"
@@ -1075,17 +1052,12 @@ backend_nomad() {
       then
         local nomad_server_name=$(envjqr 'nomad_server_name')
         local nomad_client_name=$(envjqr 'nomad_client_name')
-        local nomad_task_driver=$(envjqr 'nomad_task_driver')
         # No checks, `agents stop` checks for errors and uses msg to show them!
-        wb_nomad agents stop \
-          "${nomad_server_name}" "${nomad_client_name}" "${nomad_task_driver}"
+        wb_nomad agents stop "${nomad_server_name}" "${nomad_client_name}"
       fi
 
       # TODO: Always stop it? It's not always started!
       #wb_nomad webfs stop || true
-
-      local oci_image_was_already_available=$(envjqr 'oci_image_was_already_available')
-      #TODO: Remove it?
     ;;
 
     # Called by `scenario.sh` without exit trap (`scenario_setup_exit_trap`)!
@@ -1093,7 +1065,6 @@ backend_nomad() {
       local usage="USAGE: wb backend $op RUN-DIR"
       local dir=${1:?$usage}; shift
       local nomad_environment=$(envjqr 'nomad_environment')
-      local nomad_task_driver=$(envjqr   'nomad_task_driver')
       local one_tracer_per_node=$(envjqr 'one_tracer_per_node')
       local generator_task=$(envjqr 'generator_task_name')
 
@@ -1807,9 +1778,9 @@ backend_nomad() {
       else
         # Link to "live" logs only available when running local.
         local nomad_environment=$(envjqr 'nomad_environment')
-        local nomad_task_driver=$(envjqr 'nomad_task_driver')
         if test "${nomad_environment}" != "cloud"
         then
+          # Right now we are forcing "one_tracer_per_node=true" with "exec"
           if test "${one_tracer_per_node}" = "true" || test "${task}" != "tracer"
           then
             ln -s                                                            \
@@ -1821,20 +1792,6 @@ backend_nomad() {
             ln -s                                                            \
               ../../nomad/alloc/"${task}"/local/run/current/tracer/exit_code \
               "${dir}"/tracer/"${task}"/exit_code
-          else
-            # When "local" and "podman" "tracer" folder is mounted
-            if ! test "${nomad_task_driver}" = "podman"
-            then
-              ln -s                                                            \
-                ../nomad/alloc/tracer/local/run/current/tracer/stdout          \
-                "${dir}"/tracer/stdout
-              ln -s                                                            \
-                ../nomad/alloc/tracer/local/run/current/tracer/stderr          \
-                "${dir}"/tracer/stderr
-              ln -s                                                            \
-                ../../nomad/alloc/"${task}"/local/run/current/tracer/exit_code \
-                "${dir}"/tracer/"${task}"/exit_code
-            fi
           fi
         fi
         # Always wait for the tracer to be ready.
@@ -2126,66 +2083,38 @@ backend_nomad() {
       local task=${1:?$usage}; shift
 
       local nomad_environment=$(envjqr   'nomad_environment')
-      local nomad_task_driver=$(envjqr   'nomad_task_driver')
       local one_tracer_per_node=$(envjqr 'one_tracer_per_node')
-
       local patience=$(jq '.analysis.cluster_startup_overhead_s | ceil' "${dir}/profile.json")
-      # When "local" and "podman" "tracer" folder is mounted
-      if test "${nomad_task_driver}" = "podman" && test "${nomad_environment}" = "local"
+      local socket_name
+      if test "${one_tracer_per_node}" = "true" || test "${task}" != "tracer"
       then
-        # If the folder is mounted, its contents are not available on the
-        # Nomad Client allocation folder
-        local socket_path_relative=$(jq -r '.network.contents' "${dir}/tracer/config.json")
-        local socket_path_absolute="${dir}"/tracer/"${socket_path_relative}"
-        # Wait for tracer socket
-        #local socket_path_absolute="$dir/tracer/$node/$socket_path_relative"
-        msg "$(blue Waiting) ${patience}s for socket of supervisord $(yellow "program \"tracer\"") inside Nomad $(yellow "Task \"${task}\"") ..."
-        local i=0
-        while ! test -S "${socket_path_absolute}"
-        # TODO: Add the "timer" `printf "%3d" $i;` but for concurrent processes!
-        do
-          sleep 1
-          i=$((i+1))
-          if test "${i}" -ge "${patience}"
-          then
-            msg "$(red "Patience ran out for Task \"${task}\"'s tracer after ${patience}s")"
-            msg "$(yellow "check logs in ${dir}/tracer/[stdout & stderr]")"
-            # Don't use fatal here, let `start-tracer` decide!
-            return 1
-          fi
-        done
+        socket_name=$(jq -r '.network.contents' "${dir}/tracer/${task}/config.json")
       else
-        local socket_name
-        if test "${one_tracer_per_node}" = "true" || test "${task}" != "tracer"
+        socket_name=$(jq -r '.network.contents' "${dir}/tracer/config.json")
+      fi
+      # Wait for tracer socket
+      msg "$(blue Waiting) ${patience}s for socket of supervisord $(yellow "program \"tracer\"") inside Nomad $(yellow "Task \"${task}\"") ..."
+      local i=0
+      # Always keep checking that the supervisord program is still running!
+      while \
+            backend_nomad is-task-program-running "${dir}" "${task}" tracer  \
+        &&                                                                   \
+          ! backend_nomad task-file-stat "${dir}" "${task}" run/current/tracer/"${socket_name}" | grep --quiet "application/octet-stream"
+      do printf "%3d" $i; sleep 1
+        i=$((i+1))
+        if test "${i}" -ge "${patience}"
         then
-          socket_name=$(jq -r '.network.contents' "${dir}/tracer/${task}/config.json")
-        else
-          socket_name=$(jq -r '.network.contents' "${dir}/tracer/config.json")
-        fi
-        # Wait for tracer socket
-        msg "$(blue Waiting) ${patience}s for socket of supervisord $(yellow "program \"tracer\"") inside Nomad $(yellow "Task \"${task}\"") ..."
-        local i=0
-        # Always keep checking that the supervisord program is still running!
-        while \
-              backend_nomad is-task-program-running "${dir}" "${task}" tracer  \
-          &&                                                                   \
-            ! backend_nomad task-file-stat "${dir}" "${task}" run/current/tracer/"${socket_name}" | grep --quiet "application/octet-stream"
-        do printf "%3d" $i; sleep 1
-          i=$((i+1))
-          if test "${i}" -ge "${patience}"
+          msg "$(red "Patience ran out for Task \"${task}\"'s tracer after ${patience}s")"
+          if test "${one_tracer_per_node}" = "true" || test "${task}" != "tracer"
           then
-            msg "$(red "Patience ran out for Task \"${task}\"'s tracer after ${patience}s")"
-            if test "${one_tracer_per_node}" = "true" || test "${task}" != "tracer"
-            then
-              msg "$(yellow "Check logs in ${dir}/tracer/${task}/[stdout & stderr]")"
-            else
-              msg "$(yellow "Check logs in ${dir}/tracer/[stdout & stderr]")"
-            fi
-            # Don't use fatal here, let `start-node` or `start-nodes` decide!
-            return 1
+            msg "$(yellow "Check logs in ${dir}/tracer/${task}/[stdout & stderr]")"
+          else
+            msg "$(yellow "Check logs in ${dir}/tracer/[stdout & stderr]")"
           fi
-        done
-      fi
+          # Don't use fatal here, let `start-node` or `start-nodes` decide!
+          return 1
+        fi
+      done
       msg "$(green "supervisord program \"tracer\" inside Nomad Task \"${task}\" up (${i}s)!")"
       return 0
     ;;
@@ -2636,11 +2565,7 @@ backend_nomad() {
             rm -f "${dir}"/tracer/"${node}"/{stdout,stderr,exit_code}
           done
         else
-          # When "local" and "podman" "tracer" folder is mounted
-          if ! test "${nomad_task_driver}" = "podman"
-          then
-            rm -f "${dir}"/tracer/{stdout,stderr,exit_code}
-          fi
+          rm -f "${dir}"/tracer/{stdout,stderr,exit_code}
         fi
       fi
       if test "${one_tracer_per_node}" = "true" || test "${task}" != "tracer"
@@ -2666,15 +2591,10 @@ backend_nomad() {
         fi
       else
         local download_ok="true"
-        # When "local" and "podman" "tracer" folder is mounted
-        local nomad_task_driver=$(envjqr 'nomad_task_driver')
-        if ! test "${nomad_task_driver}" = "podman"
-        then
-          # Downloads "exit_code", "stdout", "stderr" and GHC files.
-          # Depending on when the start command failed, logs may not be available!
-          backend_nomad download-zstd-tracer "${dir}" "tracer" \
-          || download_ok="false"
-        fi
+        # Downloads "exit_code", "stdout", "stderr" and GHC files.
+        # Depending on when the start command failed, logs may not be available!
+        backend_nomad download-zstd-tracer "${dir}" "tracer" \
+        || download_ok="false"
         if test "${download_ok}" = "false"
         then
           msg "$(red "Failed to download \"tracer\" run files from \"tracer\"")"
@@ -2814,18 +2734,13 @@ backend_nomad() {
             --directory="${dir}"/tracer/"${task}" --file=-      \
             --no-same-owner --no-same-permissions
       else
-        # When "local" and "podman" "tracer" folder is mounted
-        local nomad_task_driver=$(envjqr 'nomad_task_driver')
-        if ! test "${nomad_task_driver}" = "podman"
-        then
-          msg "$(blue Fetching) $(yellow "\"tracer\"") run files from Nomad $(yellow "Task \"tracer\"") ..."
-          # TODO: Add compression, either "--zstd" or "--xz"
-            backend_nomad task-exec-program-run-files-tar-zstd  \
-              "${dir}" "tracer" "tracer"                        \
-          | tar --extract                                       \
-              --directory="${dir}"/tracer/ --file=-             \
-              --no-same-owner --no-same-permissions
-        fi
+        msg "$(blue Fetching) $(yellow "\"tracer\"") run files from Nomad $(yellow "Task \"tracer\"") ..."
+        # TODO: Add compression, either "--zstd" or "--xz"
+          backend_nomad task-exec-program-run-files-tar-zstd  \
+            "${dir}" "tracer" "tracer"                        \
+        | tar --extract                                       \
+            --directory="${dir}"/tracer/ --file=-             \
+            --no-same-owner --no-same-permissions
       fi
     ;;
 
@@ -2850,21 +2765,16 @@ backend_nomad() {
                 --no-same-owner --no-same-permissions
           fi
         else
-          # When "local" and "podman" "tracer" folder is mounted
-          local nomad_task_driver=$(envjqr 'nomad_task_driver')
-          if ! test "${nomad_task_driver}" = "podman"
+          # Logs will only be available if the tracer was started at least once!
+          if test -f "${dir}"/tracer/started
           then
-            # Logs will only be available if the tracer was started at least once!
-            if test -f "${dir}"/tracer/started
-            then
-              msg "$(blue Fetching) $(yellow "\"tracer\"") logRoot from Nomad $(yellow "Task \"tracer\"") ..."
-              # TODO: Add compression, either "--zstd" or "--xz"
-                backend_nomad task-exec-tracer-folders-tar-zstd         \
-                "${dir}" "${node}"                                      \
-              | tar --extract                                           \
-                  --directory="${dir}"/tracer/ --file=-                 \
-                  --no-same-owner --no-same-permissions
-            fi
+            msg "$(blue Fetching) $(yellow "\"tracer\"") logRoot from Nomad $(yellow "Task \"tracer\"") ..."
+            # TODO: Add compression, either "--zstd" or "--xz"
+              backend_nomad task-exec-tracer-folders-tar-zstd         \
+              "${dir}" "tracer"                                       \
+            | tar --extract                                           \
+                --directory="${dir}"/tracer/ --file=-                 \
+                --no-same-owner --no-same-permissions
           fi
         fi
       fi
@@ -2951,19 +2861,13 @@ backend_nomad() {
             run/current/tracer/config.json                    \
           > "${dir}"/tracer/"${task}"/config.json
         else
-          # When "local" and "podman" "tracer" folder is mounted and contents
-          # created locally by the workbench (obtained from the profile services).
-          local nomad_task_driver=$(envjqr 'nomad_task_driver')
-          if ! test "${nomad_task_driver}" = "podman"
-          then
-            # Node files that may suffer interpolation/sed replace.
-            backend_nomad task-file-contents "${dir}" "tracer" \
-              run/current/tracer/start.sh                      \
-            > "${dir}"/tracer/start.sh
-            backend_nomad task-file-contents "${dir}" "tracer" \
-              run/current/tracer/config.json                   \
-            > "${dir}"/tracer/config.json
-          fi
+          # Node files that may suffer interpolation/sed replace.
+          backend_nomad task-file-contents "${dir}" "tracer" \
+            run/current/tracer/start.sh                      \
+          > "${dir}"/tracer/start.sh
+          backend_nomad task-file-contents "${dir}" "tracer" \
+            run/current/tracer/config.json                   \
+          > "${dir}"/tracer/config.json
         fi
       fi
     ;;
@@ -3403,27 +3307,8 @@ backend_nomad() {
 # Deployment topology across multiple regions
 # https://developer.hashicorp.com/nomad/tutorials/enterprise/production-reference-architecture-vm-with-consul#multi-region
 
-# Configure `nomad` and its `podman` plugin / task driver
+# Configure `nomad` and the exec plugin / task driver
 # (Task Drivers are also called plugins because they are pluggable).
-#
-# WARNING: `podman`/`skopeo` are run using default parameters. Every workbench
-# user is responsible for its local/global configurations.
-# TODO: Unless this breaks reproducibility and with every call config files
-# and parameters need to be overriden.
-# For example:
-# Local version of /etc/containers/containers.conf
-#     mkdir -p $HOME/.config/containers/
-#     touch $HOME/.config/containers/containers.conf
-#     CONTAINERS_CONF=$HOME/.config/containers/containers.conf
-# Local version of /etc/containers/storage.conf
-# https://www.mankier.com/5/containers-storage.conf
-#     mkdir -p $HOME/.local/share/containers/storage/volumes
-#     touch $HOME/.config/containers/storage.conf
-#     CONTAINERS_STORAGE_CONF=$HOME/.config/containers/storage.conf
-# Local version of /etc/containers/policy.json
-# https://www.mankier.com/5/containers-policy.json
-#     mkdir -p $HOME/.config/containers/
-#     touch $HOME/.config/containers/policy.json
 nomad_create_server_config() {
   local name=$1
   local http_port=$2 rpc_port=$3 serv_port=$4
@@ -3661,13 +3546,6 @@ EOF
 nomad_create_client_config() {
   local name=$1
   local http_port=$2 rpc_port=$3 serv_port=$4
-  local nomad_task_driver=$5
-  if test "${nomad_task_driver}" = "podman"
-  then
-    local podman_socket_path=$6
-  else
-    local podman_socket_path=""
-  fi
   local cni_plugins_path=$(dirname $(which bridge))
   local state_dir=$(wb_nomad client state-dir-path "${name}")
   local config_file=$(wb_nomad client config-file-path "${name}")
@@ -3694,9 +3572,6 @@ nomad_create_client_config() {
   # - Generic `nomad` plugins / task drivers configuration docs:
   # - - https://www.nomadproject.io/plugins/drivers
   # - - https://www.nomadproject.io/docs/configuration/plugin
-  # - Specific `nomad` `podman` plugin / task driver configuration docs:
-  # - - https://www.nomadproject.io/plugins/drivers/podman#plugin-options
-  # - - https://github.com/hashicorp/nomad-driver-podman#driver-configuration
   cat > "${config_file}" <<- EOF
 # Names:
 ########
@@ -3967,51 +3842,8 @@ client {
 
 }
 
-EOF
-
-# TODO: Host Network
-# https://developer.hashicorp.com/nomad/docs/configuration/client#host_network-block
-
-if test -n "${podman_socket_path}"
-then
-  cat >> "${config_file}" <<- EOF
 # Plugins:
 ##########
-# https://developer.hashicorp.com/nomad/plugins/drivers/podman#plugin-options
-plugin "nomad-driver-podman" {
-  args = []
-  # https://github.com/hashicorp/nomad-driver-podman#driver-configuration
-  config {
-    # Defaults to "unix:///run/podman/podman.sock" when running as root or a
-    # cgroup V1 system, and "unix:///run/user/<USER_ID>/podman/podman.sock" for
-    # rootless cgroup V2 systems.
-    socket_path = "unix://$podman_socket_path"
-    # Allows tasks to bind host paths (volumes) inside their container.
-    volumes {
-      enabled = true
-    }
-    # This option can be used to disable Nomad from removing a container when
-    # the task exits.
-    gc {
-      container = true
-    }
-    # Allows the driver to start and reuse a previously stopped container after
-    # a Nomad client restart. Consider a simple single node system and a
-    # complete reboot. All previously managed containers will be reused instead
-    # of disposed and recreated.
-    recover_stopped = false
-    # Setting this to true will disable Nomad logs collection of Podman tasks.
-    # If you don't rely on nomad log capabilities and exclusively use host based
-    # log aggregation, you may consider this option to disable nomad log
-    # collection overhead. Beware to you also loose automatic log rotation.
-    disable_log_collection = false
-  }
-}
-
-EOF
-else
-  cat >> "${config_file}" <<- EOF
-# TODO: Make the exec plugin config optional ???
 plugin "exec" {
   # Defaults to "private". Set to "private" to enable PID namespace isolation
   # for tasks by default, or "host" to disable isolation.
@@ -4051,10 +3883,6 @@ plugin "raw_exec" {
   }
 }
 
-EOF
-fi
-
-cat >> "${config_file}" <<- EOF
 # Misc:
 #######
 # The vault stanza configures Nomad's integration with HashiCorp's Vault. When
diff --git a/nix/workbench/backend/nomad/cloud.nix b/nix/workbench/backend/nomad/cloud.nix
index 1e62512f207..a35cb57599c 100644
--- a/nix/workbench/backend/nomad/cloud.nix
+++ b/nix/workbench/backend/nomad/cloud.nix
@@ -7,9 +7,9 @@
 }:
 let
 
-  # The exec (non podman) task driver can run in a cloud environment using SRE's
-  # Nomad servers with the "nix_installable" patch and Amazon S3 to distribute
-  # ther genesis files. All credentials are obtained using Vault.
+  # The exec task driver can run in a cloud environment using SRE's Nomad
+  # servers with the "nix_installable" patch and Amazon S3 to distribute the
+  # genesis files. All credentials are obtained using Vault.
   name = "nomadcloud";
 
   # Unlike the supervisor backend `useCabalRun` is always false here.
diff --git a/nix/workbench/backend/nomad/cloud.sh b/nix/workbench/backend/nomad/cloud.sh
index 091e9b07044..20863635f2b 100644
--- a/nix/workbench/backend/nomad/cloud.sh
+++ b/nix/workbench/backend/nomad/cloud.sh
@@ -11,7 +11,6 @@ backend_nomadcloud() {
 
     name )
       # Can be:
-      # nomadpodman       (Using podman Task Driver in the cloud is not planned)
       # nomadexec  (Starts Nomad Agents supporting the "nix_installable" stanza)
       # nomadcloud    (SRE managed Nomad Agents on Amazon S3 (dedicated or not))
       echo 'nomadcloud'
@@ -24,7 +23,6 @@ backend_nomadcloud() {
       local usage="USAGE: wb backend $op BACKEND-DIR"
       local backend_dir=${1:?$usage}; shift
       # Repeated code / envars set by all sub-backends
-      setenvjqstr 'nomad_task_driver'    "exec"
       setenvjqstr 'nomad_environment'    "cloud"
       setenvjqstr 'one_tracer_per_node'  "true"
       # Cloud runs always run the generator inside Nomad Task "explorer"
diff --git a/nix/workbench/backend/nomad/exec.nix b/nix/workbench/backend/nomad/exec.nix
index 656fba9370c..5bcdea1026b 100644
--- a/nix/workbench/backend/nomad/exec.nix
+++ b/nix/workbench/backend/nomad/exec.nix
@@ -7,9 +7,9 @@
 }:
 let
 
-  # The exec (non podman) task driver can run in a local environment were it
-  # starts a Nomad Server and Nomad Agents instances and used webfs to
-  # distribute the genesis folder.
+  # The exec task driver can run in a local environment were it starts a Nomad
+  # Server and Nomad Agents instances and used webfs to distribute the genesis
+  # folder.
   name = "nomadexec";
 
   # Unlike the supervisor backend `useCabalRun` is always false here.
diff --git a/nix/workbench/backend/nomad/exec.sh b/nix/workbench/backend/nomad/exec.sh
index a761fd2784d..b9139da67ab 100644
--- a/nix/workbench/backend/nomad/exec.sh
+++ b/nix/workbench/backend/nomad/exec.sh
@@ -11,7 +11,6 @@ backend_nomadexec() {
 
     name )
       # Can be:
-      # nomadpodman       (Using podman Task Driver in the cloud is not planned)
       # nomadexec  (Starts Nomad Agents supporting the "nix_installable" stanza)
       # nomadcloud    (SRE managed Nomad Agents on Amazon S3 (dedicated or not))
       echo 'nomadexec'
@@ -24,7 +23,6 @@ backend_nomadexec() {
       local usage="USAGE: wb backend $op BACKEND-DIR"
       local backend_dir=${1:?$usage}; shift
       # Repeated code / envars set by all sub-backends
-      setenvjqstr 'nomad_task_driver'    "exec"
       setenvjqstr 'nomad_environment'    "local"
       setenvjqstr 'one_tracer_per_node'  "true"
       # Local runs always run the generator inside Nomad Task "node-0"
@@ -162,8 +160,8 @@ setenv-defaults-nomadexec() {
   local backend_dir="${1}"
 
   setenvjqstr 'nomad_server_name'   "srv1"
-  # As one task driver runs as a normal user and the other as a root, use
-  # different names to allow restarting/reusing without cleaup, this way
+  # As other task driver can run as a normal user and exec needs root, use
+  # different names to allow restarting/reusing without cleanup, this way
   # data folders already there can be accessed without "permission denied"
   # errors.
   setenvjqstr 'nomad_client_name'   "cli1-exe"
diff --git a/nix/workbench/backend/nomad/podman.nix b/nix/workbench/backend/nomad/podman.nix
deleted file mode 100644
index 7883694b8ab..00000000000
--- a/nix/workbench/backend/nomad/podman.nix
+++ /dev/null
@@ -1,101 +0,0 @@
-{ pkgs
-, lib
-, stateDir
-, basePort # Ignored here and just returned to be used by `runner.nix`!
-## `useCabalRun` not used here unlike `supervisor.nix`.
-, ...
-}:
-let
-
-  # The podman (non exec) task driver can only run in a local environment
-  # because we mount the genesis and mainnet mirror local folders, it's the
-  # simplest Nomad testing environment and doesn't need root privileges or
-  # Amazon S3 to run!
-  name = "nomadpodman";
-
-  # Unlike the supervisor backend `useCabalRun` is always false here.
-  useCabalRun = false;
-
-  # The versions of Nomad and the Nomad plugins needed are defined here instead
-  # of inside the flake!
-  extraShellPkgs = let
-    nomad = (pkgs.buildGo119Module rec {
-      pname = "nomad";
-      version = "1.4.3"; # Both Nomad versions are using 1.4.3
-      subPackages = [ "." ];
-      doCheck = true;
-      src = pkgs.fetchFromGitHub {
-        owner = "hashicorp";
-        repo = pname;
-        rev = "v${version}";
-        # nix-prefetch-url --unpack https://github.com/hashicorp/nomad/archive/v1.4.3.tar.gz
-        sha256 = "0j2ik501sg6diyabwwfrqnz1wxx485w5pxry4bfkg5smgyp5y18r";
-      };
-      # error: either `vendorHash` or `vendorSha256` is required
-      # https://discourse.nixos.org/t/buildgomodule-how-to-get-vendorsha256/9317
-      vendorSha256 = "sha256-JQRpsQhq5r/QcgFwtnptmvnjBEhdCFrXFrTKkJioL3A=";
-    });
-    # This plugin is defined but only used if `execTaskDriver` is false.
-    nomad-driver-podman = (pkgs.buildGo119Module rec {
-      pname = "nomad-driver-podman";
-      version = "0.4.2";
-      subPackages = [ "." ];
-      doCheck = false; # some tests require a running podman service to pass
-      src = pkgs.fetchFromGitHub {
-        owner = "hashicorp";
-        repo = pname;
-        rev = "v${version}";
-        # nix-prefetch-url --unpack https://github.com/hashicorp/nomad-driver-podman/archive/v0.4.2.tar.gz
-        # sha256 = "03856ws02xkqg5374x35zzz5900456rvpsridsjgwvvyqnysn9ls";
-        sha256 = "1wz3pqv0ib52g378sgq3ahhp3p4fm6xm41c53lliavrxcy6msy58";
-      };
-      # error: either `vendorHash` or `vendorSha256` is required
-      # https://discourse.nixos.org/t/buildgomodule-how-to-get-vendorsha256/9317
-      vendorSha256 = "sha256-UIUavFdBuSiaUsNaibPjRMURMLLK5UjNHVoyNSIRNQ4=";
-    });
-  in
-    [
-      nomad pkgs.podman nomad-driver-podman
-      # Network tools to be able to use bridge networking and the HTTP server
-      # to upload/download the genesis tar file.
-      pkgs.cni-plugins
-    ]
-  ;
-
-  # Nomad-generic "container-specs.json"
-  # Build a Nomad Job specification for this Nomad "sub-backend".
-  materialise-profile =
-    let params = {
-      inherit pkgs lib stateDir;
-      subBackendName = "podman";
-    };
-    in (import ../nomad.nix params).materialise-profile
-  ;
-
-  overlay =
-    proTopo: self: super:
-    {
-    }
-  ;
-
-  service-modules = {
-    node = { config, ... }:
-      let selfCfg = config.services.cardano-node;
-          i       = toString selfCfg.nodeId;
-      in {
-          services.cardano-node.stateDir = stateDir + "/node-${i}";
-        }
-    ;
-  };
-
-in
-{
-  inherit name;
-
-  inherit extraShellPkgs;
-  inherit materialise-profile;
-  inherit overlay service-modules;
-  inherit stateDir basePort;
-
-  inherit useCabalRun;
-}
diff --git a/nix/workbench/backend/nomad/podman.sh b/nix/workbench/backend/nomad/podman.sh
deleted file mode 100644
index 296dfec7a77..00000000000
--- a/nix/workbench/backend/nomad/podman.sh
+++ /dev/null
@@ -1,340 +0,0 @@
-usage_nomadpodman() {
-  # Using a unique help message for all Nomad "sub-backends"
-  usage_nomad
-}
-
-backend_nomadpodman() {
-
-  op=${1:?$(usage_nomadpodman)}; shift
-
-  case "${op}" in
-
-    name )
-      # Can be:
-      # nomadpodman       (Using podman Task Driver in the cloud is not planned)
-      # nomadexec  (Starts Nomad Agents supporting the "nix_installable" stanza)
-      # nomadcloud    (SRE managed Nomad Agents on Amazon S3 (dedicated or not))
-      echo 'nomadpodman'
-    ;;
-
-    # Overrided backend "methods"
-
-    # All sub-backends set these same jq envars.
-    setenv-defaults )
-      local usage="USAGE: wb backend $op BACKEND-DIR"
-      local backend_dir=${1:?$usage}; shift
-      # Repeated code / envars set by all sub-backends
-      setenvjqstr 'nomad_task_driver'    "podman"
-      setenvjqstr 'nomad_environment'    "local"
-      setenvjqstr 'one_tracer_per_node'  "false"
-      # Local runs always run the generator inside Nomad Task "node-0"
-      setenvjqstr 'generator_task_name'           \
-        "$(                                       \
-          jq -r                                   \
-            .nomadJob.generatorTaskName           \
-            "${backend_dir}"/container-specs.json \
-        )"
-      # Store the location of the Nix-built "container-specs" file.
-      # TODO/FIXME: This is the only way to be able to later copy it to "$dir" ?
-      setenvjqstr 'profile_container_specs_file' \
-        "${backend_dir}"/container-specs.json
-      # It "overrides" completely `backend_nomad`'s `setenv-defaults`.
-      setenv-defaults-nomadpodman        "${backend_dir}"
-    ;;
-
-    allocate-run )
-      # Default allocation before calling the backend specific allocation.
-      backend_nomad allocate-run            "$@"
-      allocate-run-nomadpodman              "$@"
-    ;;
-
-    # Called by `run.sh` without exit trap (unlike `scenario_setup_exit_trap`)!
-    deploy-genesis )
-      # It "overrides" completely `backend_nomad`'s `deploy-genesis`.
-      deploy-genesis-nomadpodman            "$@"
-    ;;
-
-    wait-pools-stopped )
-      # It passes the sleep time (in seconds) required argument.
-      # This time is different between local and cloud backends to avoid
-      # unnecesary Nomad specific traffic (~99% happens waiting for node-0, the
-      # first one it waits to stop inside a loop) and at the same time be less
-      # sensitive to network failures.
-      backend_nomad wait-pools-stopped      1 "$@"
-    ;;
-
-    wait-latencies-stopped )
-      # It passes the sleep time (in seconds) required argument.
-      # This time is different between local and cloud backends to avoid
-      # unnecesary Nomad specific traffic (~99% happens waiting for node-0, the
-      # first one it waits to stop inside a loop) and at the same time be less
-      # sensitive to network failures.
-      backend_nomad wait-latencies-stopped  1 "$@"
-    ;;
-
-    # All or clean up everything!
-    # Called after `scenario.sh` without an exit trap!
-    stop-cluster )
-      # Shared code between Nomad sub-backends that internally only takes care
-      # of the Nomad job.
-      backend_nomad stop-cluster-internal   "$@"
-      # Takes care of any Nomad agents (server and client(s)) that were setup
-      # locally for only this run.
-      backend_nomad stop-cluster-local      "$@"
-    ;;
-
-    # Generic backend sub-commands, shared code between Nomad sub-backends.
-
-    describe-run )
-      backend_nomad describe-run            "$@"
-    ;;
-
-    is-running )
-      backend_nomad is-running              "$@"
-    ;;
-
-    start-cluster )
-      backend_nomad start-cluster           "$@"
-    ;;
-
-    start-tracers )
-      backend_nomad start-tracers           "$@"
-    ;;
-
-    start-nodes )
-      backend_nomad start-nodes             "$@"
-    ;;
-
-    start-generator )
-      backend_nomad start-generator         "$@"
-    ;;
-
-    start-healthchecks )
-      backend_nomad start-healthchecks      "$@"
-    ;;
-
-    start-latencies )
-      backend_nomad start-latencies         "$@"
-    ;;
-
-    start-node )
-      backend_nomad start-node              "$@"
-    ;;
-
-    stop-node )
-      backend_nomad stop-node               "$@"
-    ;;
-
-    get-node-socket-path )
-      backend_nomad get-node-socket-path    "$@"
-    ;;
-
-    wait-node )
-      backend_nomad wait-node               "$@"
-    ;;
-
-    wait-node-stopped )
-      backend_nomad wait-node-stopped       "$@"
-    ;;
-
-    stop-all )
-      backend_nomad stop-all                "$@"
-    ;;
-
-    fetch-logs )
-      backend_nomad fetch-logs              "$@"
-    ;;
-
-    cleanup-cluster )
-      backend_nomad cleanup-cluster         "$@"
-    ;;
-
-    * )
-      backend_nomad "${op}" "$@"
-    ;;
-
-  esac
-
-}
-
-# Sets the envars not shared by all the other sub-backends.
-setenv-defaults-nomadpodman() {
-  local backend_dir="${1}"
-
-  setenvjqstr 'nomad_server_name'   "srv1"
-  # As one task driver runs as a normal user and the other as a root, use
-  # different names to allow restarting/reusing without cleaup, this way
-  # data folders already there can be accessed without "permission denied"
-  # errors.
-  setenvjqstr 'nomad_client_name'   "cli1-pod"
-}
-
-# Sub-backend specific allocs and calls `backend_nomad`'s `allocate-run`.
-allocate-run-nomadpodman() {
-  local usage="USAGE: wb backend $op RUN-DIR"
-  local dir=${1:?$usage}; shift
-
-  # Copy the container specs file (container-specs.json)
-  # This is the output file of the Nix derivation
-  local profile_container_specs_file=$(envjqr 'profile_container_specs_file')
-  # Create a nicely sorted and indented copy
-  jq . "${profile_container_specs_file}" > "${dir}"/container-specs.json
-
-  # Select which version of the Nomad job spec file we are running and
-  # create a nicely sorted and indented copy it "nomad/nomad-job.json".
-  jq -r ".nomadJob.podman.oneTracerPerCluster" \
-    "${dir}"/container-specs.json              \
-  > "${dir}"/nomad/nomad-job.json
-  # Update the Nomad Job specs file accordingly
-  ## - Job Name
-  ### Must match `^[a-zA-Z0-9-]{1,128}$)` or it won't be possible to use it
-  ### as namespace.: "invalid name "2023-02-10-06.34.f178b.ci-test-bage.nom"".
-  local nomad_job_name=$(basename "${dir}")
-  backend_nomad allocate-run-nomad-job-patch-name "${dir}" "${nomad_job_name}"
-  # The job file is "slightly" modified (jq) to suit the running environment.
-  ## Empty the global namespace. Local runs ignore "${NOMAD_NAMESPACE:-}"
-  backend_nomad allocate-run-nomad-job-patch-namespace "${dir}"
-  podman_create_image "${dir}"
-  # Make sure the "genesis-volume" dir is present when the Nomad job is
-  # started because with the podman task driver (always local, not used for
-  # cloud) these directories are going to be mounted by adding them to the
-  # "volume" stanza.
-  mkdir "${dir}"/genesis-volume
-  # It needs to mount the tracer directory if "one_tracer_per_node" is
-  # false and mount the genesis and CARDANO_MAINNET_MIRROR (if needed).
-  nomad_job_file_create_mounts "${dir}"
-}
-
-# Called by `run.sh` without exit trap (unlike `scenario_setup_exit_trap`)!
-deploy-genesis-nomadpodman() {
-  local usage="USAGE: wb backend $op RUN-DIR"
-  local dir=${1:?$usage}; shift
-  # The "$dir/genesis-volume" folder is mounted to "run/local/genesis"
-  # inside the container that for security reasons does not follow symlinks,
-  # so everything is copied from the generated/patched genesis in
-  # "$dir/genesis" to "$dir/genesis-volume"
-  mkdir "${dir}"/genesis-volume/byron
-  mkdir "${dir}"/genesis-volume/utxo-keys
-  mkdir "${dir}"/genesis-volume/node-keys
-  cp -a "${dir}"/genesis/genesis.alonzo.json        \
-        "${dir}"/genesis-volume/genesis.alonzo.json
-  cp -a "${dir}"/genesis/genesis.conway.json        \
-        "${dir}"/genesis-volume/genesis.conway.json
-  cp -a "${dir}"/genesis/genesis-shelley.json       \
-        "${dir}"/genesis-volume/genesis-shelley.json
-  cp -a "${dir}"/genesis/byron/genesis.json         \
-        "${dir}"/genesis-volume/byron/genesis.json
-  cp -a                                             \
-        "${dir}"/genesis/utxo-keys/*.skey           \
-        "${dir}"/genesis-volume/utxo-keys/
-  cp -a                                             \
-        "${dir}"/genesis/utxo-keys/*.vkey           \
-        "${dir}"/genesis-volume/utxo-keys/
-  cp -a                                             \
-        "${dir}"/genesis/node-keys/*.skey           \
-        "${dir}"/genesis-volume/node-keys/
-  cp -a                                             \
-        "${dir}"/genesis/node-keys/*.vkey           \
-        "${dir}"/genesis-volume/node-keys/
-  cp -a                                             \
-        "${dir}"/genesis/node-keys/*.opcert         \
-        "${dir}"/genesis-volume/node-keys/
-}
-
-podman_create_image() {
-  local dir=${1:?$usage}; shift
-  # Look up the OCI image's name and tag (Nix profile).
-  local oci_image_name=$(jq -r .ociImage.imageName "${dir}"/container-specs.json)
-  local oci_image_tag=$( jq -r .ociImage.imageTag  "${dir}"/container-specs.json)
-  if podman image exists "${oci_image_name}:${oci_image_tag}"
-  then
-    setenvjqstr 'oci_image_was_already_available' "true"
-    msg "OCI image ${oci_image_name}:${oci_image_tag} is already available"
-  else
-    setenvjqstr 'oci_image_was_already_available' "false"
-    msg "Creating OCI image ..."
-    # Script that creates the OCI image from nix2container layered output.
-    local oci_image_skopeo_script=$(jq -r .ociImage.copyToPodman "${dir}"/container-specs.json)
-    # TODO: for further research.
-    # STORAGE_DRIVER=overlay "$oci_image_skopeo_script"
-    # If podman 4.2.1 and nomad v1.3.5 this fix is not needed anymore
-    # Forced the `overlay` storage driver or podman won't see the image.
-    # https://docs.podman.io/en/latest/markdown/podman.1.html#note-unsupported-file-  systems-in-rootless-mode
-    # Error was: workbench:  FATAL: OCI image registry.workbench.iog.io/  cluster:2l7wi7sh1zyp2mnl24m13ibnh2wsjvwg cannot be found by podman
-    if ! "${oci_image_skopeo_script}"
-    then
-      fatal "Creation of OCI image ${oci_image_name}:${oci_image_tag} failed"
-    else
-      # Now check that `podman` can see the "cluster" OCI image.
-      if ! podman image exists "${oci_image_name}:${oci_image_tag}"
-      then
-        fatal "OCI image ${oci_image_name}:${oci_image_tag} was created but cannot be found by podman"
-      else
-        msg "OCI image named \"${oci_image_name}:${oci_image_tag}\" created"
-      fi
-    fi
-  fi
-}
-
-nomad_job_file_create_mounts() {
-  local dir=$1
-  local absolute_dir=$(realpath "${dir}")
-  local nomad_job_name=$(jq -r ". [\"job\"] | keys[0]" "${dir}"/nomad/nomad-job.json)
-  local one_tracer_per_node=$(envjqr 'one_tracer_per_node')
-  # If CARDANO_MAINNET_MIRROR is present generate a list of needed volumes.
-  if test -n "${CARDANO_MAINNET_MIRROR}"
-  then
-    # The nix-store path contains 3 levels of symlinks. This is a hack to
-    # avoid creating a container image with all these files.
-    local immutable_store=$(readlink -f "${CARDANO_MAINNET_MIRROR}"/immutable)
-    local mainnet_mirror_volumes="[
-        \"${CARDANO_MAINNET_MIRROR}:${CARDANO_MAINNET_MIRROR}:ro\"
-      , \"${immutable_store}:${immutable_store}:ro\"
-      $(find -L "${immutable_store}" -type f -exec realpath {} \; | xargs dirname | sort | uniq | xargs -I "{}" echo ", \"{}:{}:ro\"")
-    ]"
-  else
-    local mainnet_mirror_volumes="[]"
-  fi
-  # Hint:
-  # - Working dir is: /tmp/cluster/
-  # - Mount point is: /tmp/cluster/run/current
-  ## The workbench is expecting an specific hierarchy of folders and files.
-  local container_mountpoint=$(jq -r ". [\"job\"][\"${nomad_job_name}\"][\"meta\"][\"TASK_STATEDIR\"]" "${dir}"/nomad/nomad-job.json)
-  # Nodes
-  for node in $(jq_tolist 'keys' "${dir}"/node-specs.json)
-  do
-    local task_stanza_name="${node}"
-    # Every node needs access to "./genesis/" and tracer when only 1 is used.
-    local jq_filter="
-      [
-        \"${absolute_dir}/genesis-volume:${container_mountpoint}/genesis:ro\"
-      ]
-      +
-      (
-        if \$one_tracer_per_node == true
-        then
-          [ ]
-        else
-          [ \"${absolute_dir}/tracer:${container_mountpoint}/tracer:rw\" ]
-        end
-      )
-      +
-      \$mainnet_mirror_volumes
-    "
-    local podman_volumes=$(jq "${jq_filter}" --argjson one_tracer_per_node "${one_tracer_per_node}" --argjson mainnet_mirror_volumes "${mainnet_mirror_volumes}" "${dir}"/profile/node-specs.json)
-    jq ".job[\"${nomad_job_name}\"][\"group\"][\"${node}\"][\"task\"][\"${node}\"][\"config\"][\"volumes\"] = \$podman_volumes" --argjson podman_volumes "${podman_volumes}" "${dir}"/nomad/nomad-job.json | sponge "${dir}"/nomad/nomad-job.json
-  done
-  # Tracer
-  if jqtest ".node.tracer" "${dir}"/profile.json && ! test "${one_tracer_per_node}" = "true"
-  then
-    local task_stanza_name_t="tracer"
-    # Tracer only needs access to itself (its shared folder).
-    local jq_filter_t="
-      [
-        \"${absolute_dir}/tracer:${container_mountpoint}/tracer:rw\"
-      ]
-    "
-    local podman_volumes_t=$(jq "${jq_filter_t}" "${dir}"/profile/node-specs.json)
-    jq ".job[\"${nomad_job_name}\"][\"group\"][\"tracer\"][\"task\"][\"tracer\"][\"config\"][\"volumes\"] = \$podman_volumes_t" --argjson podman_volumes_t "${podman_volumes_t}" "${dir}"/nomad/nomad-job.json | sponge "${dir}"/nomad/nomad-job.json
-  fi
-}
diff --git a/nix/workbench/backend/oci-images.nix b/nix/workbench/backend/oci-images.nix
deleted file mode 100644
index 0cffe4ae654..00000000000
--- a/nix/workbench/backend/oci-images.nix
+++ /dev/null
@@ -1,49 +0,0 @@
-{ pkgs
-, lib
-, containerPkgs
-}:
-
-let
-
-  # Why `nix2container` instead of the built-in `dockerTools` ?:
-  # - https://lewo.abesis.fr/posts/nix-build-container-image/
-  # - https://discourse.nixos.org/t/nix2container-another-dockertools-buildimage-implementation-based-on-skopeo/21688
-  n2c = pkgs.nix2container.outputs.packages.x86_64-linux.nix2container;
-
-  clusterNode = n2c.buildImage {
-    name = "registry.ci.iog.io/workbench-cluster-node";
-    # Adds `/etc/protocols` and ``/etc/services` to the root directory.
-    # FIXME: Inside the container still can't resolve `localhost` but can
-    # resolve WAN domains using public DNS servers.
-    # Running `bash-5.1# /nix/store/*-glibc-#-bin/bin/getent hosts localhost`
-    # inside the container returns nothing and python stuff like `supervisord`
-    # breaks: "error: <class 'socket.gaierror'>, [Errno -2] Name or service not known: file: /nix/store/hb1lzaisgx2m9n29hqhh6yp6hasplq1v-python3-3.9.10/lib/python3.9/socket.py line: 954"
-    # Further reading for hints:
-    # https://stackoverflow.com/questions/39965432/docker-container-unable-to-resolve-localhost
-    copyToRoot = with pkgs; [ iana-etc ];
-    maxLayers = 25;
-    # Transform the input binaries sections into layers.
-    # These layers are added to the container's /nix/store, nothing in `$PATH`.
-    layers = (lib.attrsets.mapAttrsToList
-      (name: attr: n2c.buildLayer {deps = [ attr.nix-store-path ];})
-      containerPkgs
-    );
-    # OCI container specification:
-    # https://github.com/opencontainers/image-spec/blob/3a7f492d3f1bcada656a7d8c08f3f9bbd05e7406/specs-go/v1/config.go#L24
-    config = {
-      # Volumes are mounted as user `0:0`, I have no choice here.
-      User = "0:0";
-      # The stanza `WorkingDir` is not used because the config file of
-      # `supervisord` depends on the working directory.
-      Entrypoint = []; # `nix2container` includes these derivations inside the image
-    };
-  };
-
-in
-
-  {
-    imageName = clusterNode.imageName;
-    imageTag = clusterNode.imageTag;
-    # https://github.com/containers/skopeo/blob/main/docs/skopeo-copy.1.md
-    copyToPodman = "${clusterNode.copyToPodman}/bin/copy-to-podman";
-  }
diff --git a/nix/workbench/nomad.sh b/nix/workbench/nomad.sh
index 960ab5f0c96..2734cf9e405 100644
--- a/nix/workbench/nomad.sh
+++ b/nix/workbench/nomad.sh
@@ -1,6 +1,6 @@
 usage_nomad() {
      usage "nomad" "nomad helpers" <<EOF
-    $(helpcmd dir-path \(vault\|client\|server\|plugin\|webfs\))
+    $(helpcmd dir-path \(vault\|client\|server\|webfs\))
                      Gets the corresponding cache directory file path.
 
     $(helpcmd vault \(ci\|world\) login)
@@ -40,9 +40,9 @@ usage_nomad() {
                      Needed envars (NOMAD_TOKEN, NOMAD_ADDR or NOMAD_NAMESPACE)
                      must be provided by the user.
 
-    $(helpcmd agents start SERVER-NAME CLIENT-NAME TASK-DRIVER-NAME)
+    $(helpcmd agents start SERVER-NAME CLIENT-NAME)
                      Start a default 1 server 1 client Nomad cluster.
-    $(helpcmd agents stop  SERVER-NAME CLIENT-NAME TASK-DRIVER-NAME)
+    $(helpcmd agents stop  SERVER-NAME CLIENT-NAME)
                      Stop the default 1 server 1 client Nomad cluster.
 
     $(helpcmd \(server/client\) state-dir-path           NAME)
@@ -65,13 +65,6 @@ usage_nomad() {
     $(helpcmd \(server/client\) cleanup                  NAME)
                      Deletes all logs and state files
 
-    $(helpcmd plugin nomad-driver-podman socket-path)
-    $(helpcmd plugin nomad-driver-podman pid-filepath)
-    $(helpcmd plugin nomad-driver-podman pid)
-    $(helpcmd plugin nomad-driver-podman is-running)
-    $(helpcmd plugin nomad-driver-podman start)
-    $(helpcmd plugin nomad-driver-podman stop)
-
     $(helpcmd webfs state-dir-path)
     $(helpcmd webfs document-root-path)
     $(helpcmd webfs add-genesis-dir)
@@ -117,7 +110,7 @@ wb_nomad() {
 ### dir-path ) #################################################################
 ################################################################################
     dir-path )
-      local usage="USAGE: wb nomad ${op} vault|server|client|plugin|webfs"
+      local usage="USAGE: wb nomad ${op} vault|server|client|webfs"
       # Calling `wb nomad dir-path  XXX` inside a Nix derivation will fail:
       # "mkdir: cannot create directory '/homeless-shelter': Permission denied"
       local nomad_cache_dir="$(envjqr 'cacheDir')"/nomad
@@ -139,11 +132,6 @@ wb_nomad() {
           mkdir -p "${nomad_clients_dir}"
           echo "${nomad_clients_dir}"
         ;;
-        plugin )
-          local plugin_dir="${nomad_cache_dir}"/plugin
-          mkdir -p "${plugin_dir}"
-          echo "${plugin_dir}"
-        ;;
         webfs )
           local webfs_dir="${nomad_cache_dir}"/webfs
           mkdir -p "${webfs_dir}"
@@ -633,10 +621,9 @@ EOL
       case "${subop}" in
 ####### agents -> start )#######################################################
         start )
-          local usage="USAGE:wb nomad ${op} ${subop} SERVER-NAME CLIENT-NAME DRIVER-NAME"
+          local usage="USAGE:wb nomad ${op} ${subop} SERVER-NAME CLIENT-NAME"
           local server_name=${1:?$usage}; shift
           local client_name=${1:?$usage}; shift
-          local task_driver=${1:?$usage}; shift
           # Create config files for the server and start it.
           if ! wb_nomad server configure "${server_name}" 4646 4647 4648
           then
@@ -646,45 +633,28 @@ EOL
           then
             fatal "Failed to start Nomad server \"${server_name}\""
           fi
-          # Set up the podman driver and start it if it's needed.
-          if test "${task_driver}" = "podman"
-          then
-            # Create config files for the client and the Podman plugin/task driver.
-            wb_nomad plugin nomad-driver-podman start
-          fi
           # Create config files for the client and start it.
           # WARNING: Actually the client is configured to connect to all the
           # running servers, so if there are no servers ready the Nomad
           # cluster state is uknown (at least to me with the actual config).
-          if ! wb_nomad client configure "${client_name}" 14646 14647 14648 "${task_driver}"
+          if ! wb_nomad client configure "${client_name}" 14646 14647 14648
           then
             wb_nomad server stop "${server_name}" || true
             fatal "Failed to configure Nomad client \"${client_name}\""
           fi
           # Only the exec driver must be run as root.
-          if test "${task_driver}" = "exec"
+          # Pass the "root prefix" (command prefix)
+          if ! wb_nomad client start "${client_name}" "sudo "
           then
-            # Pass the "root prefix" (command prefix)
-            if ! wb_nomad client start "${client_name}" "sudo "
-            then
-              wb_nomad server stop "${server_name}" || true
-              fatal "Failed to start Nomad agents"
-            fi
-          else
-            if ! wb_nomad client start "${client_name}"
-            then
-              wb_nomad plugin nomad-driver-podman stop || true
-              wb_nomad server stop "${server_name}" || true
-              fatal "Failed to start Nomad agents"
-            fi
+            wb_nomad server stop "${server_name}" || true
+            fatal "Failed to start Nomad agents"
           fi
         ;;
 ####### agents -> stop )########################################################
         stop )
-          local usage="USAGE:wb nomad ${op} ${subop} SERVER-NAME CLIENT-NAME DRIVER-NAME"
+          local usage="USAGE:wb nomad ${op} ${subop} SERVER-NAME CLIENT-NAME"
           local server_name=${1:?$usage}; shift
           local client_name=${1:?$usage}; shift
-          local task_driver=${1:?$usage}; shift
           # Collect garbage to avoid orphaned mounts
           # https://support.hashicorp.com/hc/en-us/articles/360000654467-Removing-Orphaned-Mounts-from-Nomad-Allocation-Directory
           nomad system gc 2>&1 >/dev/null || true
@@ -692,13 +662,6 @@ EOL
             wb_nomad client stop "${client_name}" \
           ||                                      \
             msg "$(red "Failed to stop Nomad client \"${client_name}\"")"
-          # Stop driver(s)
-          if test "${task_driver}" = "podman"
-          then
-              wb_nomad plugin nomad-driver-podman stop \
-            ||                                         \
-              msg "$(red "Failed to stop nomad-driver-podman")"
-          fi
           # Stop server
             wb_nomad server stop "${server_name}" \
           ||                                      \
@@ -939,21 +902,18 @@ EOL
         ;;
 ####### client -> configure )###################################################
         configure )
-          local usage="USAGE: wb nomad ${op} ${subop} CLIENT-NAME HTTP-PORT RPC-PORT SERV-PORT DRIVER-NAME [GENESIS-DIR]"
+          local usage="USAGE: wb nomad ${op} ${subop} CLIENT-NAME HTTP-PORT RPC-PORT SERV-PORT [GENESIS-DIR]"
           local name=${1:?$usage}; shift
           # Ports
           local http_port=${1:?$usage}; shift
           local rpc_port=${1:?$usage}; shift
           local serv_port=${1:?$usage}; shift
-          # Unlike the server, the client can have different task drivers!
-          local task_driver=${1:?$usage}; shift
           # Checks
           # Assume the presence of the PID file means "running" because it
           # can represent an abnormal exit / uknown state!
           if wb_nomad client is-running "${name}"
           then
-            # When reusing, remember to check that client is running with
-            # the needed task driver!
+            # When reusing, remember to check that client is running!
             msg "$(red "FATAL: Nomad client \"${name}\" is already running or in an uknown state, call 'wb nomad client stop ${name}' or 'wb nomad nuke' first")"
             return 1
           else
@@ -965,30 +925,9 @@ EOL
             mkdir -p "${state_dir}"/data/{client,plugins,alloc}
             # Store the ports config
             echo "{\"http\": ${http_port}, \"rpc\": ${rpc_port}, \"serv\": ${serv_port}}" > "${state_dir}"/ports.json
-            # Store tast driver parameter
-            echo "${task_driver}" > "${state_dir}"/task_driver
-            # Task driver specific client configuration
-            if test "${task_driver}" = "podman"
-            then
-              local podman_socket_path=$(wb_nomad plugin nomad-driver-podman socket-path)
-              # Podman Task Driver - Client Requirements:
-              ## "Ensure that Nomad can find the plugin, refer to `plugin_dir`."
-              ### https://www.nomadproject.io/plugins/drivers/podman#client-  requirements
-              ## On every call to `wb nomad client configure` the
-              ## available `nomad-driver-podman` is replaced.
-              # TODO: Somehow move this logic to `wb nomad plugin`
-              rm -f "${state_dir}"/data/plugins/nomad-driver-podman
-              ln -s -f "$(which nomad-driver-podman)" "${state_dir}"/data/plugins/nomad-driver-podman
-              # Create configuration file
-              nomad_create_client_config "${name}" \
-                "${http_port}" "${rpc_port}" "${serv_port}" \
-                "${task_driver}" "${podman_socket_path}"
-            else
-              # Create configuration file
-              nomad_create_client_config "${name}" \
-                "${http_port}" "${rpc_port}" "${serv_port}" \
-                "${task_driver}"
-            fi
+            # Create configuration file
+            nomad_create_client_config "${name}" \
+              "${http_port}" "${rpc_port}" "${serv_port}"
           fi
         ;;
 ####### client -> port )########################################################
@@ -1144,36 +1083,18 @@ EOL
           # TODO: Configure the node?
           # nomad node eligibility -enable "${client_id}"
           # nomad node drain -disable "${client_id}"
-          local task_driver=$(cat "${state_dir}"/task_driver)
-          if test "${task_driver}" == "exec"
+          # Look for "Drivers":{"exec":  {"Detected":true,"Healthy":true}}
+          if ! test $(nomad node status -filter "\"workbench-nomad-client-${name}\" in Name" -json | jq '.[0].Drivers.exec.Detected') = "true"
           then
-            # Look for "Drivers":{"exec":  {"Detected":true,"Healthy":true}}
-            if ! test $(nomad node status -filter "\"workbench-nomad-client-${name}\" in Name" -json | jq '.[0].Drivers.exec.Detected') = "true"
-            then
-              # Not using `fatal` here, let the caller decide!
-              msg "$(red "FATAL: Task driver \"exec\" was not detected")"
-              return 1
-            fi
-            if ! test $(nomad node status -filter "\"workbench-nomad-client-${name}\" in Name" -json | jq '.[0].Drivers.exec.Healthy') = "true"
-            then
-              # Not using `fatal` here, let the caller decide!
-              msg "$(red "FATAL: Task driver \"exec\" is not healthy")"
-              return 1
-            fi
-          else
-            # Look for "Drivers":{"podman":{"Detected":true,"Healthy":true}}
-            if ! test $(nomad node status -filter "\"workbench-nomad-client-${name}\" in Name" -json | jq '.[0].Drivers.podman.Detected') = "true"
-            then
-              # Not using `fatal` here, let the caller decide!
-              msg "$(red "FATAL: Task driver \"podman\" was not detected")"
-              return 1
-            fi
-            if ! test $(nomad node status -filter "\"workbench-nomad-client-${name}\" in Name" -json | jq '.[0].Drivers.podman.Healthy') = "true"
-            then
-              # Not using `fatal` here, let the caller decide!
-              msg "$(red "FATAL: Task driver \"podman\" is not healthy")"
-              return 1
-            fi
+            # Not using `fatal` here, let the caller decide!
+            msg "$(red "FATAL: Task driver \"exec\" was not detected")"
+            return 1
+          fi
+          if ! test $(nomad node status -filter "\"workbench-nomad-client-${name}\" in Name" -json | jq '.[0].Drivers.exec.Healthy') = "true"
+          then
+            # Not using `fatal` here, let the caller decide!
+            msg "$(red "FATAL: Task driver \"exec\" is not healthy")"
+            return 1
           fi
           true
           # TODO: Check all the clients connected to the server!
@@ -1266,125 +1187,6 @@ EOL
       esac # client
     ;;
 ################################################################################
-### plugin ) ###################################################################
-################################################################################
-    plugin )
-      local usage="USAGE: wb nomad ${op} nomad-driver-podman"
-      local plugin=${1:?$usage}; shift
-      case "${plugin}" in
-####### plugin -> nomad-driver-podman )#########################################
-        nomad-driver-podman )
-          local usage="USAGE: wb nomad ${op} ${plugin}"
-          local subop=${1:?$usage}; shift
-          case "$subop" in
-########### plugin -> nomad-driver-podman -> socket-path )######################
-            socket-path )
-              # Socket of the process that connects nomad-driver-podman with podman.
-              # Can't reside inside "$dir", can't use a path longer than 108 characters!
-              # See: https://man7.org/linux/man-pages/man7/unix.7.html
-              # char        sun_path[108];            /* Pathname */
-              echo "${XDG_RUNTIME_DIR:-/run/user/$UID}/workbench-podman.sock"
-            ;;
-########### plugin -> nomad-driver-podman -> pid-filepath )#####################
-            pid-filepath )
-              local plugin_dir="$(wb_nomad dir-path plugin)"
-              echo "${plugin_dir}"/nomad-driver-podman.pid
-            ;;
-########### plugin -> nomad-driver-podman -> pid )##############################
-            pid )
-              local pid_file=$(wb_nomad plugin nomad-driver-podman pid-filepath)
-              if test -f $pid_file
-              then
-                local pid_number=$(cat "${pid_file}")
-                # Check if the process is running
-                if kill -0 "${pid_number}" >/dev/null 2>&1
-                then
-                  echo "${pid_number}"
-                else
-                  rm "${pid_file}"
-                  false
-                fi
-              else
-                false
-              fi
-            ;;
-########### plugin -> nomad-driver-podman -> is-running )#######################
-            is-running )
-              wb_nomad plugin nomad-driver-podman pid >/dev/null
-            ;;
-########### plugin -> nomad-driver-podman -> start )############################
-            # Start the `podman` API service needed by `nomad`.
-            start ) # TODO: Check that it's not already running!
-              msg "Preparing podman API service for nomad driver \`nomad-driver-podman\` ..."
-              local podman_socket_path=$(wb_nomad plugin nomad-driver-podman socket-path)
-        #      if test -S "$socket"
-        #      then
-        #          msg "Podman API service was already running"
-        #      else
-                # The session is kept open waiting for a new connection for 60 seconds.
-                # https://discuss.hashicorp.com/t/nomad-podman-rhel8-driver-difficulties/21877/4
-                # `--time`: Time until the service session expires in seconds. Use 0
-                # to disable the timeout (default 5).
-                local pid_file=$(wb_nomad plugin nomad-driver-podman pid-filepath)
-                podman system service --time 60 "unix://$podman_socket_path" &
-                local pid_number="$!"
-                echo "${pid_number}" > "${pid_file}"
-                local i=0 patience=5
-                while test ! -S "$podman_socket_path"
-                do printf "%3d" $i; sleep 1
-                  i=$((i+1))
-                  if test $i -ge $patience
-                  then echo
-                      progress "nomad-driver-podman" "$(red FATAL):  workbench:  nomad-driver-podman:  patience ran out after ${patience}s, socket $podman_socket_path"
-                      fatal "nomad-driver-podman startup did not succeed:  check logs"
-                      rm "${pid_file}"
-                  fi
-                  echo -ne "\b\b\b"
-                done >&2
-        #      fi
-              msg "Podman API service started"
-            ;;
-########### plugin -> nomad-driver-podman -> stop )#############################
-            stop )
-              local pid_number
-              local pid_file=$(wb_nomad plugin nomad-driver-podman pid-filepath "${name}")
-              # Call without `local` to obtain the subcommand's return code.
-              if pid_number=$(wb_nomad plugin nomad-driver-podman pid)
-              then
-                msg "Killing nomad-driver-podman (PID ${pid_number}) ..."
-                if ! kill -SIGINT "${pid_number}"
-                then
-                  fatal \
-                    "Killing nomad-driver-podman failed, \
-                    is PID \"${pid_number}\" (${pid_file}) running?"
-                else
-                  # Wait 15 seconds for the process to fully exit or kill it.
-                  if ! timeout 15 tail --pid="${pid_number}" -f /dev/null
-                  then
-                    kill -SIGKILL "${pid_number}" || true
-                  fi
-                fi
-                # Remove PID file
-                rm "${pid_file}"
-              else
-                msg "nomad-driver-podman API service is not running"
-                # If a PID file was already there it's not removed!
-                false
-              fi
-            ;;
-########### plugin -> nomad-driver-podman -> * )################################
-            * )
-              usage_nomad
-            ;;
-          esac  # plugin -> nomad-driver-podman
-        ;;
-####### plugin -> * )###########################################################
-        * )
-          usage_nomad
-        ;;
-      esac # plugin
-    ;;
-################################################################################
 ### rsync ) ####################################################################
 ################################################################################
     rsync )
@@ -1554,16 +1356,6 @@ EOF
           msg "Failed to remove config folder of Nomad client \"${client_name}\", now in unknown state, manual cleanup needed"
         fi
       done
-      # Nuke the nomad-driver-podman plugin
-      if wb_nomad plugin nomad-driver-podman is-running
-      then
-        wb_nomad plugin nomad-driver-podman stop
-      fi
-      local podman_socket_path=$(wb_nomad plugin nomad-driver-podman socket-path)
-      if test -S "${podman_socket_path}"
-      then
-        rm "${podman_socket_path}"
-      fi
       # Nuke all Nomad servers
       for server_name in $(ls "${nomad_servers_dir}"); do
         msg "Config folder of Nomad server \"${server_name}\" found"
@@ -1598,10 +1390,6 @@ EOF
         wb_nomad webfs stop
       fi
       rm -rf "$(wb_nomad webfs state-dir-path)"
-      # TODO: podman ?
-      # rm -rf ~/.local/share/containers/cache/
-      # rm -rf ~/.local/share/containers/storage/
-      # rm -rf ~/.config/containers/podman/
     ;;
 ################################################################################
 ### job ) ######################################################################
@@ -1676,12 +1464,12 @@ EOF
           # Some docs on the scheduling flow:
           # https://developer.hashicorp.com/nomad/docs/concepts/scheduling/scheduling
           # Notes:
-          # 1) In "misterious" cases a new evaluation ID is given later
+          # 1) In "mysterious" cases a new evaluation ID is given later
           #    and the initial one is forgotten!
           #    For example if there are placement errors, like when the
           #    requested task driver is not available, the initial
           #    deployment stays "running" and new evaluation IDs can be
-          #    found whith the following message:
+          #    found with the following message:
           #    "StatusDescription": "created to place remaining allocations"
           #    This is weird/unintuitive to me!
           # 2) An evaluation can be marked as "complete" in the `-json`
diff --git a/nix/workbench/wb b/nix/workbench/wb
index 2bea2c04900..1aa94ac0290 100755
--- a/nix/workbench/wb
+++ b/nix/workbench/wb
@@ -33,7 +33,6 @@ global_basedir=${global_basedir:-$(realpath "$(dirname "$0")")}
 . "$global_basedir"/backend/nomad.sh
 . "$global_basedir"/backend/nomad/cloud.sh
 . "$global_basedir"/backend/nomad/exec.sh
-. "$global_basedir"/backend/nomad/podman.sh
 . "$global_basedir"/backend/supervisor.sh
 
 usage_main() {

From 2ff7643b6866b8898da676ed71422a143a22c177 Mon Sep 17 00:00:00 2001
From: Federico Mastellone <federico.pascual-mastellone@iohk.io>
Date: Fri, 6 Sep 2024 15:36:47 +0000
Subject: [PATCH 2/7] Remove nix2container from flake

---
 flake.lock | 72 ++----------------------------------------------------
 flake.nix  |  5 +---
 2 files changed, 3 insertions(+), 74 deletions(-)

diff --git a/flake.lock b/flake.lock
index 9768a46c506..7fc9f8d9016 100644
--- a/flake.lock
+++ b/flake.lock
@@ -450,24 +450,6 @@
         "type": "github"
       }
     },
-    "flake-utils_5": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
     "ghc-8.6.5-iohk": {
       "flake": false,
       "locked": {
@@ -1050,25 +1032,6 @@
         "type": "github"
       }
     },
-    "nix2container_2": {
-      "inputs": {
-        "flake-utils": "flake-utils_5",
-        "nixpkgs": "nixpkgs_6"
-      },
-      "locked": {
-        "lastModified": 1712990762,
-        "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=",
-        "owner": "nlewo",
-        "repo": "nix2container",
-        "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nlewo",
-        "repo": "nix2container",
-        "type": "github"
-      }
-    },
     "nixago": {
       "inputs": {
         "flake-utils": [
@@ -1326,21 +1289,6 @@
       }
     },
     "nixpkgs_6": {
-      "locked": {
-        "lastModified": 1712920918,
-        "narHash": "sha256-1yxFvUcJfUphK9V91KufIQom7gCsztza0H4Rz2VCWUU=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "92323443a56f4e9fc4e4b712e3119f66d0969297",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_7": {
       "locked": {
         "lastModified": 1708343346,
         "narHash": "sha256-qlzHvterVRzS8fS0ophQpkh0rqw0abijHEOAKm0HmV0=",
@@ -1464,7 +1412,6 @@
           "nixpkgs"
         ],
         "iohkNix": "iohkNix",
-        "nix2container": "nix2container_2",
         "nixpkgs": [
           "haskellNix",
           "nixpkgs-unstable"
@@ -1593,7 +1540,7 @@
           "std",
           "blank"
         ],
-        "nixpkgs": "nixpkgs_7",
+        "nixpkgs": "nixpkgs_6",
         "paisano": "paisano",
         "paisano-tui": "paisano-tui",
         "terranix": [
@@ -1631,21 +1578,6 @@
         "type": "github"
       }
     },
-    "systems_2": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
     "tullia": {
       "inputs": {
         "nix-nomad": "nix-nomad",
@@ -1687,7 +1619,7 @@
     },
     "utils_2": {
       "inputs": {
-        "systems": "systems_2"
+        "systems": "systems"
       },
       "locked": {
         "lastModified": 1710146030,
diff --git a/flake.nix b/flake.nix
index 5ba7786b3a9..4052b906485 100644
--- a/flake.nix
+++ b/flake.nix
@@ -56,8 +56,6 @@
 
     std.url = "github:divnix/std";
 
-    nix2container.url = "github:nlewo/nix2container";
-
     cardano-automation = {
       url = "github:input-output-hk/cardano-automation";
       inputs = {
@@ -78,7 +76,6 @@
     , ops-lib
     , cardano-mainnet-mirror
     , std
-    , nix2container
     , cardano-automation
     , em
     , ...
@@ -106,7 +103,7 @@
         iohkNix.overlays.cardano-lib
         iohkNix.overlays.utils
         (final: prev: {
-          inherit customConfig nix2container;
+          inherit customConfig;
           bench-data-publish = cardano-automation.outputs.packages.${final.system}."bench-data-publish:exe:bench-data-publish";
           em = import em { inherit (final) system;
                            nixpkgsSrcs = nixpkgs.outPath;

From 745f5ecec526d95dfc04ca10c5e67cf775e5fa90 Mon Sep 17 00:00:00 2001
From: Federico Mastellone <federico.pascual-mastellone@iohk.io>
Date: Mon, 9 Sep 2024 16:19:18 +0000
Subject: [PATCH 3/7] wb | update openssh version

---
 nix/workbench/backend/nomad.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix/workbench/backend/nomad.nix b/nix/workbench/backend/nomad.nix
index 17f21552fbb..744dd9a58a7 100644
--- a/nix/workbench/backend/nomad.nix
+++ b/nix/workbench/backend/nomad.nix
@@ -140,7 +140,7 @@ let
             //
             lib.attrsets.optionalAttrs (subBackendName == "cloud") {
               openssh_hacks = rec {
-                commit = "01076c7118939c90cb5c9d6320e9813740ec3534"; # Branch "9.3p1";
+                commit = "5d8c5913c70723318acf47496e2abf7d2c99384f"; # OpenSSH version 9.8 (Branch "9.8");
                 nix-store-path  = (__getFlake "github:fmaste/openssh-portable-hacks/${commit}").packages.x86_64-linux.openssh_hacks;
                 flake-reference = "github:fmaste/openssh-portable-hacks";
                 flake-output = "packages.x86_64-linux.openssh_hacks";

From 0515ca56f9e6eb2d02f1716ef3ee4fb302df1f36 Mon Sep 17 00:00:00 2001
From: Federico Mastellone <federico.pascual-mastellone@iohk.io>
Date: Mon, 2 Sep 2024 19:58:37 +0000
Subject: [PATCH 4/7] wb | don't use nomad version from nixpkgs

---
 nix/workbench/backend/nomad/cloud.nix |  8 +++++++-
 nix/workbench/backend/nomad/exec.nix  | 18 +++++++++---------
 nix/workbench/backend/nomad/patch.nix |  9 +++++++++
 nix/workbench/shell.nix               |  2 +-
 4 files changed, 26 insertions(+), 11 deletions(-)
 create mode 100644 nix/workbench/backend/nomad/patch.nix

diff --git a/nix/workbench/backend/nomad/cloud.nix b/nix/workbench/backend/nomad/cloud.nix
index a35cb57599c..38ce4ab2f65 100644
--- a/nix/workbench/backend/nomad/cloud.nix
+++ b/nix/workbench/backend/nomad/cloud.nix
@@ -15,9 +15,15 @@ let
   # Unlike the supervisor backend `useCabalRun` is always false here.
   useCabalRun = false;
 
-  # No override of the Nomad and Vault binaries in "workbench/shell.nix"
   extraShellPkgs =
     [
+      # SRE's patched Nomad 1.6.3 that enables `nix_installables` as artifacts.
+      (
+        # Only x86_64-linux is supported.
+        if builtins.currentSystem != "x86_64-linux"
+        then builtins.abort "Nomad backends only available for x86_64-linux"
+        else (import ./patch.nix {})
+      )
       # Amazon S3 HTTP to upload/download the genesis tar file.
       pkgs.awscli
     ]
diff --git a/nix/workbench/backend/nomad/exec.nix b/nix/workbench/backend/nomad/exec.nix
index 5bcdea1026b..275c3af289d 100644
--- a/nix/workbench/backend/nomad/exec.nix
+++ b/nix/workbench/backend/nomad/exec.nix
@@ -15,15 +15,15 @@ let
   # Unlike the supervisor backend `useCabalRun` is always false here.
   useCabalRun = false;
 
-  # The versions of Nomad and the Nomad plugins needed are defined here instead
-  # of inside the flake!
-  extraShellPkgs = let
-    # If we are going to use the `exec` driver we use the SRE patched version of
-    # Nomad that allows to use `nix_installables` as artifacts.
-    commit = "8f3b74796a8f56f38a812813c64dba995956a66e"; # Patched 1.6.3
-    nomad-sre = (__getFlake "github:input-output-hk/cardano-perf/${commit}").packages.x86_64-linux.nomad;
-  in
-    [ nomad-sre
+  extraShellPkgs =
+    [
+      # SRE's patched Nomad 1.6.3 that enables `nix_installables` as artifacts.
+      (
+        # Only x86_64-linux is supported.
+        if builtins.currentSystem != "x86_64-linux"
+        then builtins.abort "Nomad backends only available for x86_64-linux"
+        else (import ./patch.nix {})
+      )
       # The HTTP server to upload/download the genesis tar file in a local env.
       pkgs.webfs
     ]
diff --git a/nix/workbench/backend/nomad/patch.nix b/nix/workbench/backend/nomad/patch.nix
new file mode 100644
index 00000000000..f959ff6301f
--- /dev/null
+++ b/nix/workbench/backend/nomad/patch.nix
@@ -0,0 +1,9 @@
+# SRE patched version of Nomad allowing `nix_installables` as artifacts.
+{...}@args:
+let
+  # Patched 1.6.3.
+  commit = "8f3b74796a8f56f38a812813c64dba995956a66e";
+  flake = (__getFlake "github:input-output-hk/cardano-perf/${commit}");
+  nomad-sre = flake.packages.${builtins.currentSystem}.nomad;
+in
+  nomad-sre
diff --git a/nix/workbench/shell.nix b/nix/workbench/shell.nix
index 4c38d514c14..1e6b43310b8 100644
--- a/nix/workbench/shell.nix
+++ b/nix/workbench/shell.nix
@@ -140,7 +140,7 @@ in project.shellFor {
       # Publish
       bench-data-publish
       # Publish tunnel
-      yq nomad vault-bin norouter socat
+      yq vault-bin norouter socat
       # Debugging
       postgresql
       # Performance report generation

From d4f1d7cfdf7285314dc02170485db5b64bacbcca Mon Sep 17 00:00:00 2001
From: Federico Mastellone <federico.pascual-mastellone@iohk.io>
Date: Tue, 3 Sep 2024 19:06:03 +0000
Subject: [PATCH 5/7] wb | Remove publish functionality dependencies while the
 new infrastructure is decided

---
 nix/workbench/backend/nomad/cloud.nix | 3 +++
 nix/workbench/publish.sh              | 6 ++++++
 nix/workbench/shell.nix               | 2 --
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/nix/workbench/backend/nomad/cloud.nix b/nix/workbench/backend/nomad/cloud.nix
index 38ce4ab2f65..b0301bb3924 100644
--- a/nix/workbench/backend/nomad/cloud.nix
+++ b/nix/workbench/backend/nomad/cloud.nix
@@ -24,8 +24,11 @@ let
         then builtins.abort "Nomad backends only available for x86_64-linux"
         else (import ./patch.nix {})
       )
+
       # Amazon S3 HTTP to upload/download the genesis tar file.
       pkgs.awscli
+      # Use to obtain the AWS credentials.
+      pkgs.vault-bin
     ]
   ;
 
diff --git a/nix/workbench/publish.sh b/nix/workbench/publish.sh
index 6ae82d139c7..94c3e0cd54b 100644
--- a/nix/workbench/publish.sh
+++ b/nix/workbench/publish.sh
@@ -135,6 +135,12 @@ publish() {
       bench-data-publish $@
     ;;
 
+    * )
+      # Needed dependencies in `nix/workbench/shell.nix`:
+      #   vault-bin yq norouter socat
+      fatal "Sorry, non-local functionality is currently not available!"
+    ;;
+
     cloud )
       local usage="USAGE: wb publish $op"
       # Publish script vars.
diff --git a/nix/workbench/shell.nix b/nix/workbench/shell.nix
index 1e6b43310b8..763342c2ae3 100644
--- a/nix/workbench/shell.nix
+++ b/nix/workbench/shell.nix
@@ -139,8 +139,6 @@ in project.shellFor {
   ++ [
       # Publish
       bench-data-publish
-      # Publish tunnel
-      yq vault-bin norouter socat
       # Debugging
       postgresql
       # Performance report generation

From 17977d349365ecff37cc5637bd6ca706718459cb Mon Sep 17 00:00:00 2001
From: Federico Mastellone <federico.pascual-mastellone@iohk.io>
Date: Mon, 9 Sep 2024 16:55:34 +0000
Subject: [PATCH 6/7] wb | remove vault

---
 nix/workbench/backend/nomad.nix       |   7 +
 nix/workbench/backend/nomad/cloud.nix |   5 +-
 nix/workbench/backend/nomad/cloud.sh  |  25 +---
 nix/workbench/nomad.sh                | 207 +-------------------------
 4 files changed, 17 insertions(+), 227 deletions(-)

diff --git a/nix/workbench/backend/nomad.nix b/nix/workbench/backend/nomad.nix
index 744dd9a58a7..a0cfcaea040 100644
--- a/nix/workbench/backend/nomad.nix
+++ b/nix/workbench/backend/nomad.nix
@@ -205,6 +205,13 @@ let
                     oneTracerPerNode = true;
                     withSsh = true;
                   };
+                # AWS S3 bucket that will be used to deploy the genesis files.
+                s3 = rec {
+                  bucket = "cardano-perf-deploy";
+                  region = "eu-central-1";
+                  host   = "s3.${region}.amazonaws.com";
+                  uri    = "https://${bucket}.${host}";
+                };
               };
             }
           ;
diff --git a/nix/workbench/backend/nomad/cloud.nix b/nix/workbench/backend/nomad/cloud.nix
index b0301bb3924..7b949dea1c8 100644
--- a/nix/workbench/backend/nomad/cloud.nix
+++ b/nix/workbench/backend/nomad/cloud.nix
@@ -9,7 +9,7 @@ let
 
   # The exec task driver can run in a cloud environment using SRE's Nomad
   # servers with the "nix_installable" patch and Amazon S3 to distribute the
-  # genesis files. All credentials are obtained using Vault.
+  # genesis files (Buckets needs write permissions for the deployer machine).
   name = "nomadcloud";
 
   # Unlike the supervisor backend `useCabalRun` is always false here.
@@ -24,11 +24,8 @@ let
         then builtins.abort "Nomad backends only available for x86_64-linux"
         else (import ./patch.nix {})
       )
-
       # Amazon S3 HTTP to upload/download the genesis tar file.
       pkgs.awscli
-      # Use to obtain the AWS credentials.
-      pkgs.vault-bin
     ]
   ;
 
diff --git a/nix/workbench/backend/nomad/cloud.sh b/nix/workbench/backend/nomad/cloud.sh
index 20863635f2b..35a98bc1899 100644
--- a/nix/workbench/backend/nomad/cloud.sh
+++ b/nix/workbench/backend/nomad/cloud.sh
@@ -252,18 +252,8 @@ setenv-defaults-nomadcloud() {
   #########
   # AWS_* #
   #########
-  # Check all the AWS S3 envars needed for the HTTP PUT request
-  # Using same names as the AWS CLI
-  # https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
-  if test -z "${AWS_ACCESS_KEY_ID:-}" || test -z "${AWS_SECRET_ACCESS_KEY:-}"
-  then
-    msg $(blue "INFO: Amazon S3 \"AWS_ACCESS_KEY_ID\" or \"AWS_SECRET_ACCESS_KEY\" envar is not set")
-    msg $(yellow "WARNING: Fetching \"AWS_ACCESS_KEY_ID\" and \"AWS_SECRET_ACCESS_KEY\" from SRE provided Vault for \"Performance and Tracing\"")
-    local aws_credentials
-    aws_credentials="$(wb_nomad vault world aws-s3-credentials)"
-    export AWS_ACCESS_KEY_ID=$(echo "${aws_credentials}" | jq -r .data.access_key)
-    export AWS_SECRET_ACCESS_KEY=$(echo "${aws_credentials}" | jq -r .data.secret_key)
-  fi
+  local s3_bucket_name="$(jq -r .nomadJob.cloud.s3.bucket "${backend_dir}"/container-specs.json)"
+  msg $(blue "INFO: Using Amazon S3 \"${s3_bucket_name}\" as bucket")
 }
 
 # Sub-backend specific allocs and calls `backend_nomad`'s `allocate-run`.
@@ -858,11 +848,9 @@ deploy-genesis-nomadcloud() {
       --directory="${dir}"/genesis --files-from=-
 
   # Upload genesis tar file
-  local s3_region="eu-central-1"
-  local s3_host="s3.${s3_region}.amazonaws.com";
-  local s3_bucket_name="iog-cardano-perf";
-  local s3_access_key="${AWS_ACCESS_KEY_ID}";
-  local s3_access_key_secret="${AWS_SECRET_ACCESS_KEY}"
+  local s3_bucket_name="$(jq -r .nomadJob.cloud.s3.bucket "${dir}"/container-specs.json)"
+  local s3_region="$(jq -r .nomadJob.cloud.s3.region "${dir}"/container-specs.json)"
+  local s3_uri="$(jq -r .nomadJob.cloud.s3.uri "${dir}"/container-specs.json)"
   local s3_storage_class="STANDARD"
   local return_code=0
   msg "$(blue Uploading) $(yellow "\"${genesis_file_name}\"") to $(yellow "\"s3://${s3_bucket_name}/\"") ..."
@@ -887,8 +875,7 @@ deploy-genesis-nomadcloud() {
   fi
 
   # Generic download from every node.
-  local uri="https://${s3_bucket_name}.${s3_host}/${genesis_file_name}"
-  if ! backend_nomad deploy-genesis-wget "${dir}" "${uri}"
+  if ! backend_nomad deploy-genesis-wget "${dir}" "${s3_uri}"/"${genesis_file_name}"
   then
     # File kept for debugging!
     msg "$(red "FATAL: deploy-genesis-wget \"${dir}\" \"${uri}\"")"
diff --git a/nix/workbench/nomad.sh b/nix/workbench/nomad.sh
index 2734cf9e405..3aadb68af9a 100644
--- a/nix/workbench/nomad.sh
+++ b/nix/workbench/nomad.sh
@@ -1,26 +1,8 @@
 usage_nomad() {
      usage "nomad" "nomad helpers" <<EOF
-    $(helpcmd dir-path \(vault\|client\|server\|webfs\))
+    $(helpcmd dir-path \(client\|server\|webfs\))
                      Gets the corresponding cache directory file path.
 
-    $(helpcmd vault \(ci\|world\) login)
-                     Login with your GitHub token. First copy the token by doing:
-                     Your profile -> Settings -> Developer Settings ->
-                     Tokens (Classic) -> Generate New Token (Classic)
-                     and create a new token with only the "read:org" permission.
-    $(helpcmd vault \(ci\|world\) nomad-token)
-                     Gets the corresponding Nomad token from the chosen Vault
-                     (WARNING: shows secrets!!!).
-    $(helpcmd vault ci pg-user)
-                     Gets SRE's Postgres server username from Vault
-                     (WARNING: shows secrets!!!).
-    $(helpcmd vault ci pg-pass)
-                     Gets SRE's Postgres server password from Vault
-                     (WARNING: shows secrets!!!).
-    $(helpcmd vault world aws-s3-credentials)
-                     Gets Cardano World's AWS S3 crdentials from Vault in JSON
-                     (WARNING: shows secrets!!!).
-
     $(helpcmd clients ready)
                      Creates a JSON array with "id", "name", "datacenter" and
                      "ip" of all SRE's Nomad client nodes available
@@ -110,18 +92,13 @@ wb_nomad() {
 ### dir-path ) #################################################################
 ################################################################################
     dir-path )
-      local usage="USAGE: wb nomad ${op} vault|server|client|webfs"
+      local usage="USAGE: wb nomad ${op} server|client|webfs"
       # Calling `wb nomad dir-path  XXX` inside a Nix derivation will fail:
       # "mkdir: cannot create directory '/homeless-shelter': Permission denied"
       local nomad_cache_dir="$(envjqr 'cacheDir')"/nomad
       mkdir -p "${nomad_cache_dir}"
       local subop=${1:?$usage}; shift
       case "${subop}" in
-        vault )
-          local vault_dir="${nomad_cache_dir}"/vault
-          mkdir -p "${vault_dir}"
-          echo "${vault_dir}"
-        ;;
         server )
           local nomad_servers_dir="${nomad_cache_dir}"/server
           mkdir -p "${nomad_servers_dir}"
@@ -148,184 +125,6 @@ wb_nomad() {
       esac
     ;;
 ################################################################################
-### vault ) ####################################################################
-################################################################################
-    vault )
-      local usage="USAGE: wb nomad ${op} world|ci"
-      local vault_dir="$(wb_nomad dir-path vault)"
-      local entity=${1:?$usage}; shift
-      case "${entity}" in
-####### vault -> ci )###########################################################
-        ci )
-          mkdir -p "${vault_dir}"/ci
-          local login_file="${vault_dir}"/ci/login.json
-          local vault_address="https://vault.ci.iog.io"
-          local action=${1:?$usage}; shift
-          case "${action}" in
-            login )
-              msg "First create and copy your GitHub token by doing: "
-              msg "Your profile -> Settings -> Developer Settings -> Tokens (Classic) -> Generate New Token (Classic)"
-              msg "and create a new token with only the \"read:org\" permission."
-              read -p "Hit enter to continue ..."
-              vault login                             \
-                -address="${vault_address}"           \
-                -method=github -path=github-employees \
-                -no-store -format=json                \
-              > "${login_file}"
-            ;;
-            enabled )
-              if test -f "${login_file}"
-              then
-                # Fetch token info from vault
-                local client_token
-                client_token=$(jq -r '.auth.client_token' "${login_file}")
-                local token_lookup_response
-                if token_lookup_response=$(VAULT_TOKEN="${client_token}" vault token lookup -address="${vault_address}" -namespace=perf -format=json)
-                then
-                  local expire_time
-                  expire_time=$(echo "${token_lookup_response}" | jq -r .data.expire_time)
-                  # Compare expire date with the actual date minus one day.
-                  # This avoids a token expiring while a profile is running.
-                  if test "$(date -u -d "${expire_time}" "+%s")" -ge "$(($(date -u "+%s") - 86400))"
-                  then
-                    true
-                  else
-                    rm "${login_file}"
-                    false
-                  fi
-                else
-                  fatal "Are you logged in to Vault? Call 'wb nomad vault ${entity} login' with your IOHK GitHub token (classic)"
-                fi
-              else
-                false
-              fi
-            ;;
-            nomad-token )
-              if ! wb_nomad vault "${entity}" enabled
-              then
-                wb_nomad vault "${entity}" login
-              fi
-              local client_token
-              client_token=$(jq -r '.auth.client_token' "${login_file}")
-              local nomad_token_json
-              if nomad_token_json=$(VAULT_TOKEN="${client_token}" vault read -address="${vault_address}" -non-interactive -format=json nomad/creds/perf)
-              then
-                echo "${nomad_token_json}" | jq -r .data.secret_id
-              else
-                fatal "Unable to fetch Nomad token from Vault"
-              fi
-            ;;
-            pg-user )
-              if ! wb_nomad vault "${entity}" enabled
-              then
-                wb_nomad vault "${entity}" login
-              fi
-              local client_token
-              client_token=$(jq -r '.auth.client_token' "${login_file}")
-              VAULT_TOKEN="${client_token}" vault kv get \
-                --address="${vault_address}"             \
-                -non-interactive                         \
-                -format=json                             \
-                kv/postgrest/perf                        \
-              | jq -r .data.data.postgrestDbUser
-            ;;
-            pg-pass )
-              if ! wb_nomad vault "${entity}" enabled
-              then
-                wb_nomad vault "${entity}" login
-              fi
-              local client_token
-              client_token=$(jq -r '.auth.client_token' "${login_file}")
-              VAULT_TOKEN="${client_token}" vault kv get \
-                --address="${vault_address}"             \
-                -non-interactive                         \
-                -format=json                             \
-                kv/postgrest/perf                        \
-              | jq -r .data.data.postgrestDbPass
-            ;;
-####### vault -> ci -> * )######################################################
-            * )
-              usage_nomad
-            ;;
-          esac
-        ;;
-####### vault -> world )########################################################
-        world )
-          mkdir -p "${vault_dir}"/world
-          local login_file="${vault_dir}"/world/login.json
-          local vault_address="https://vault.world.dev.cardano.org"
-          local action=${1:?$usage}; shift
-          case "${action}" in
-            login )
-              msg "First create and copy your GitHub token by doing: "
-              msg "Your profile -> Settings -> Developer Settings -> Tokens (Classic) -> Generate New Token (Classic)"
-              msg "and create a new token with only the \"read:org\" permission."
-              read -p "Hit enter to continue ..."
-              vault login                             \
-                -address="${vault_address}"           \
-                -method=github -path=github-employees \
-                -no-store -format=json                \
-              > "${login_file}"
-            ;;
-            enabled )
-              if test -f "${login_file}"
-              then
-                local client_token
-                client_token=$(jq -r '.auth.client_token' "${login_file}")
-                local token_lookup_response
-                if token_lookup_response=$(VAULT_TOKEN="${client_token}" vault token lookup -address="${vault_address}" -namespace=perf -format=json)
-                then
-                  # TODO: I need to check the expiration time?
-                  # echo "${token_lookup_response}" | jq -r .data.expire_time
-                  # 2023-02-19T13:07:26.125306646Z
-                  true
-                else
-                  fatal "Are you logged in to Vault? Call 'wb nomad vault ${entity} login' with your IOHK GitHub token (classic)"
-                fi
-              else
-                false
-              fi
-            ;;
-            nomad-token )
-              if ! wb_nomad vault "${entity}" enabled
-              then
-                wb_nomad vault "${entity}" login
-              fi
-              local client_token
-              client_token=$(jq -r '.auth.client_token' "${login_file}")
-              local nomad_token_json
-              if nomad_token_json=$(VAULT_TOKEN="${client_token}" vault read -address="${vault_address}" -non-interactive -format=json nomad/creds/perf)
-              then
-                echo "${nomad_token_json}" | jq -r .data.secret_id
-              else
-                fatal "Unable to fetch Nomad token from Vault"
-              fi
-            ;;
-            aws-s3-credentials )
-              if ! wb_nomad vault "${entity}" enabled
-              then
-                wb_nomad vault "${entity}" login
-              fi
-              local client_token
-              client_token=$(jq -r '.auth.client_token' "${login_file}")
-              VAULT_TOKEN="${client_token}" vault read \
-                --address="${vault_address}"           \
-                -format=json                           \
-                aws/creds/perf
-            ;;
-####### vault -> world -> * )###################################################
-            * )
-              usage_nomad
-            ;;
-          esac
-        ;;
-####### vault -> * )############################################################
-        * )
-          usage_nomad
-        ;;
-      esac
-    ;;
-################################################################################
 ### ssh ) ######################################################################
 ################################################################################
     ssh )
@@ -1381,7 +1180,7 @@ EOF
         fi
       done
       # Nuke the Nomad Agents' .cache dir
-      # Keep top level Nomad cache dir because it includes Vault's dirs.
+      # Keep top level Nomad cache dir because it includes webfs and ssh dirs.
       rm -rf "${nomad_clients_dir}" >/dev/null 2>&1
       rm -rf "${nomad_servers_dir}" >/dev/null 2>&1
       # Bye HTTP server

From da7dee7f9d85dbc6c4c4f58b3f3f658b8827986b Mon Sep 17 00:00:00 2001
From: Federico Mastellone <federico.pascual-mastellone@iohk.io>
Date: Tue, 10 Sep 2024 01:29:55 +0000
Subject: [PATCH 7/7] wb | code cleanups

---
 nix/workbench/backend/nomad-job.nix   |  20 +-
 nix/workbench/backend/nomad.nix       | 476 ++++++++++++++------------
 nix/workbench/backend/nomad/cloud.nix |   2 +
 nix/workbench/backend/nomad/exec.sh   |   2 +-
 nix/workbench/service/ssh.nix         |   4 +-
 5 files changed, 278 insertions(+), 226 deletions(-)

diff --git a/nix/workbench/backend/nomad-job.nix b/nix/workbench/backend/nomad-job.nix
index 7d2a0d57968..57bcdaed097 100644
--- a/nix/workbench/backend/nomad-job.nix
+++ b/nix/workbench/backend/nomad-job.nix
@@ -7,8 +7,8 @@
 , lib
 , stateDir
 , profileData
-, containerSpecs
 , generatorTaskName
+, containerPkgs
 , oneTracerPerNode ? false
 , withSsh ? false
 }:
@@ -73,8 +73,8 @@ let
 
   entrypoint =
     let
-      coreutils  = containerSpecs.containerPkgs.coreutils.nix-store-path;
-      supervisor = containerSpecs.containerPkgs.supervisor.nix-store-path;
+      coreutils  = containerPkgs.coreutils.nix-store-path;
+      supervisor = containerPkgs.supervisor.nix-store-path;
     in escapeTemplate
       ''
       # Store entrypoint's envars and "uname" in a file for debugging purposes.
@@ -508,7 +508,7 @@ let
             # ERROR: cannot verify iog-cardano-perf.s3.eu-central-1.amazonaws.com's certificate, issued by 'CN=Amazon RSA 2048 M01,O=Amazon,C=US':
             # Unable to locally verify the issuer's authority.
             # To connect to iog-cardano-perf.s3.eu-central-1.amazonaws.com insecurely, use `--no-check-certificate'.
-            SSL_CERT_FILE = "${containerSpecs.containerPkgs.cacert.nix-store-path}/etc/ssl/certs/ca-bundle.crt";
+            SSL_CERT_FILE = "${containerPkgs.cacert.nix-store-path}/etc/ssl/certs/ca-bundle.crt";
           };
 
           # Sensible defaults.
@@ -937,9 +937,9 @@ let
                   ../service/ssh.nix
                   {
                     inherit pkgs;
-                    bashInteractive = containerSpecs.containerPkgs.bashInteractive.nix-store-path;
-                    coreutils = containerSpecs.containerPkgs.coreutils.nix-store-path;
-                    openssh_hacks = containerSpecs.containerPkgs.openssh_hacks.nix-store-path;
+                    bashInteractive = containerPkgs.bashInteractive.nix-store-path;
+                    coreutils = containerPkgs.coreutils.nix-store-path;
+                    sshdExecutable = "${containerPkgs.openssh_hacks.nix-store-path}/bin/sshd";
                   }
               ;
             in [
@@ -993,12 +993,12 @@ let
           {
             driver = "exec";
             config = {
-              command = "${containerSpecs.containerPkgs.bashInteractive.nix-store-path}/bin/bash";
+              command = "${containerPkgs.bashInteractive.nix-store-path}/bin/bash";
               args = ["local/entrypoint.sh"];
               nix_installables =
                 (lib.attrsets.mapAttrsToList
-                  (name: attr: attr.nix-store-path)
-                  containerSpecs.containerPkgs
+                  (name: attr: attr.installable)
+                  containerPkgs
                 )
               ;
             };
diff --git a/nix/workbench/backend/nomad.nix b/nix/workbench/backend/nomad.nix
index a0cfcaea040..a63db0ec8e2 100644
--- a/nix/workbench/backend/nomad.nix
+++ b/nix/workbench/backend/nomad.nix
@@ -9,221 +9,271 @@
 let
 
   # Backend-specific Nix bits:
-  materialise-profile =
-    { profileData }:
-      let
-        # TODO: Repeated code, add the generator's node name to profile.json
-        generatorTaskName = if builtins.hasAttr "explorer" profileData.node-specs.value
-          then "explorer"
-          else "node-0"
+  materialise-profile = { profileData }:
+    # Intermediate / workbench-adhoc container specifications
+    let containerSpecs = rec {
+      ##########################################################################
+      # The actual commit. The one used when entering the workbench.
+      gitrev = pkgs.gitrev;
+      # Where to deploy inside the Task, needed to send commands (`nomad exec`).
+      diretories = rec {
+        work = "/local";
+        state = stateDir;
+        run = lib.strings.concatStringsSep "/" [ work state ];
+      };
+      # How to configure supervisor and how to access it from the deployer.
+      supervisord = {
+        url = "unix:///tmp/supervisor.sock";
+        conf = lib.strings.concatStringsSep "/"
+          [ diretories.run "supervisor" "supervisord.conf"]
         ;
-        # Intermediate / workbench-adhoc container specifications
-        containerSpecs = rec {
-          #
-          diretories = rec {
-            work = "/local";
-            state = stateDir;
-            run = lib.strings.concatStringsSep "/" [ work state ];
-          };
-          # The actual commit.
-          gitrev = pkgs.gitrev;
-          # Binaries.
-          containerPkgs =
-            {
-              coreutils = rec {
-                nix-store-path  = pkgs.coreutils;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "legacyPackages.x86_64-linux.coreutils";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              bashInteractive = rec {
-                nix-store-path  = pkgs.bashInteractive;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "legacyPackages.x86_64-linux.bashInteractive";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              findutils = rec {
-                nix-store-path  = pkgs.findutils;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "legacyPackages.x86_64-linux.findutils";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              iputils = rec {
-                nix-store-path  = pkgs.iputils;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "legacyPackages.x86_64-linux.iputils";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              gnutar = rec {
-                nix-store-path  = pkgs.gnutar;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "legacyPackages.x86_64-linux.gnutar";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              zstd = rec {
-                nix-store-path  = pkgs.zstd;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "legacyPackages.x86_64-linux.zstd";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              wget = rec {
-                nix-store-path  = pkgs.wget;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "legacyPackages.x86_64-linux.wget";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              cacert = rec {
-                nix-store-path  = pkgs.cacert;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "legacyPackages.x86_64-linux.cacert";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              supervisor = rec {
-                nix-store-path  = pkgs.python3Packages.supervisor;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "legacyPackages.x86_64-linux.python3Packages.supervisor";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              gnugrep = rec {
-                nix-store-path  = pkgs.gnugrep;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "legacyPackages.x86_64-linux.gnugrep";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              jq = rec {
-                nix-store-path  = pkgs.jq;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "legacyPackages.x86_64-linux.jq";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-            }
-            //
-            # TODO: - cardano-node.passthru.profiled
-            #       - cardano-node.passthru.eventlogged
-            #       - cardano-node.passthru.asserted
-            # profileData.node-services."node-0".serviceConfig.value.eventlog
-            # builtins.trace (builtins.attrNames profileData.node-services."node-0".serviceConfig.value.eventlog) XXXX
-            {
-              cardano-node = rec {
-                nix-store-path  = with pkgs;
-                  if eventlogged
-                    then cardanoNodePackages.cardano-node.passthru.eventlogged
-                    else cardanoNodePackages.cardano-node
-                ;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output =
-                  if eventlogged
-                    then "cardanoNodePackages.cardano-node.passthru.eventlogged"
-                    else "cardanoNodePackages.cardano-node"
-                ;
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              cardano-cli = rec {
-                nix-store-path  = pkgs.cardanoNodePackages.cardano-cli;
-                flake-reference = "github:input-output-hk/cardano-cli";
-                flake-output = "cardanoNodePackages.cardano-cli";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              cardano-tracer = rec {
-                nix-store-path  = pkgs.cardanoNodePackages.cardano-tracer;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "cardanoNodePackages.cardano-tracer";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-              tx-generator = rec {
-                nix-store-path  = pkgs.cardanoNodePackages.tx-generator;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "cardanoNodePackages.tx-generator";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-            }
-            //
-            lib.attrsets.optionalAttrs (subBackendName == "cloud") {
-              openssh_hacks = rec {
-                commit = "5d8c5913c70723318acf47496e2abf7d2c99384f"; # OpenSSH version 9.8 (Branch "9.8");
-                nix-store-path  = (__getFlake "github:fmaste/openssh-portable-hacks/${commit}").packages.x86_64-linux.openssh_hacks;
-                flake-reference = "github:fmaste/openssh-portable-hacks";
-                flake-output = "packages.x86_64-linux.openssh_hacks";
-                installable = "${flake-reference}/${commit}#${flake-output}";
-              };
-              rsync = rec {
-                nix-store-path  = pkgs.rsync;
-                flake-reference = "github:intersectmbo/cardano-node";
-                flake-output = "legacyPackages.x86_64-linux.rsync";
-                installable = "${flake-reference}/${gitrev}#${flake-output}";
-              };
-            }
-          ;
-          supervisord = {
-            url = "unix:///tmp/supervisor.sock";
-            conf = lib.strings.concatStringsSep "/"
-              [ diretories.run "supervisor" "supervisord.conf"]
-            ;
+      };
+      # Binaries. Flake references to the local nix store or remote repos.
+      containerPkgs = installables {inherit gitrev;};
+      # The Nomad Job description for the requested sub-backend.
+      nomadJob = nomad-job {inherit profileData containerPkgs;};
+      ##########################################################################
+    };
+    in pkgs.runCommand "workbench-backend-output-${profileData.profileName}-nomad"
+      ({
+        containerSpecsJSON = pkgs.writeText "workbench-cluster-container-pkgs.json"
+          (lib.generators.toJSON {} containerSpecs);
+      })
+      ''
+      mkdir $out
+      ln -s $containerSpecsJSON      $out/container-specs.json
+      ''
+    ;
+
+  # The installables are all the Nix packages that will be avaiable inside the
+  # Nomad Task. This dependencies are defined inside the Nomad Job as strings
+  # that can be either a path to the local nix store or a flake reference to
+  # fetch from some repo.
+  # In the case of cloud deployment the "installable" must always reference
+  # a commit accesible from every Nomad client machine and for local / "exec"
+  # the "nix-store-path" property is used to allow to run with local changes.
+  # The workbench will by default insert in the Nomad Job description the
+  # "installable" property as defined here while keeping the extra details as a
+  # `jq` friendly reference that are used to change it later.
+  installables = {gitrev}:
+    ############################################################################
+    # The "default" / basic environment where the node will run. ###############
+    # This pkgs rarely change and are almost always cached.      ###############
+    ############################################################################
+    (lib.attrsets.mapAttrs
+      (name: attr:
+        # The installable property is always the same.
+        let flakeReference = attr.flake-reference;
+            flakeOutput = attr.flake-output;
+            # The commit must come from `pkgs` because all script are using
+            # this for the shebang and other basic tools ("coreutils") and are
+            # also the dependencies used inside the workbench, like `jq`.
+            commit = pkgs.gitrev;
+        in attr // {installable="${flakeReference}/${commit}#${flakeOutput}";}
+      )
+      {
+        coreutils = {
+          nix-store-path  = pkgs.coreutils;
+          flake-reference = "github:intersectmbo/cardano-node";
+          flake-output = "legacyPackages.x86_64-linux.coreutils";
+          installable = null;
+        };
+        bashInteractive = {
+          nix-store-path  = pkgs.bashInteractive;
+          flake-reference = "github:intersectmbo/cardano-node";
+          flake-output = "legacyPackages.x86_64-linux.bashInteractive";
+          installable = null;
+        };
+        findutils = {
+          nix-store-path  = pkgs.findutils;
+          flake-reference = "github:intersectmbo/cardano-node";
+          flake-output = "legacyPackages.x86_64-linux.findutils";
+          installable = null;
+        };
+        iputils = {
+          nix-store-path  = pkgs.iputils;
+          flake-reference = "github:intersectmbo/cardano-node";
+          flake-output = "legacyPackages.x86_64-linux.iputils";
+          installable = null;
+        };
+        gnutar = {
+          nix-store-path  = pkgs.gnutar;
+          flake-reference = "github:intersectmbo/cardano-node";
+          flake-output = "legacyPackages.x86_64-linux.gnutar";
+          installable = null;
+        };
+        zstd = {
+          nix-store-path  = pkgs.zstd;
+          flake-reference = "github:intersectmbo/cardano-node";
+          flake-output = "legacyPackages.x86_64-linux.zstd";
+          installable = null;
+        };
+        wget = {
+          nix-store-path  = pkgs.wget;
+          flake-reference = "github:intersectmbo/cardano-node";
+          flake-output = "legacyPackages.x86_64-linux.wget";
+          installable = null;
+        };
+        cacert = {
+          nix-store-path  = pkgs.cacert;
+          flake-reference = "github:intersectmbo/cardano-node";
+          flake-output = "legacyPackages.x86_64-linux.cacert";
+          installable = null;
+        };
+        supervisor = {
+          nix-store-path  = pkgs.python3Packages.supervisor;
+          flake-reference = "github:intersectmbo/cardano-node";
+          flake-output = "legacyPackages.x86_64-linux.python3Packages.supervisor";
+          installable = null;
+        };
+        gnugrep = {
+          nix-store-path  = pkgs.gnugrep;
+          flake-reference = "github:intersectmbo/cardano-node";
+          flake-output = "legacyPackages.x86_64-linux.gnugrep";
+          installable = null;
+        };
+        jq = {
+          nix-store-path  = pkgs.jq;
+          flake-reference = "github:intersectmbo/cardano-node";
+          flake-output = "legacyPackages.x86_64-linux.jq";
+          installable = null;
+        };
+      }
+    )
+    //
+    ############################################################################
+    # Optional container enabled OpenSSH to properly fetch ~1TB cloud logs. ####
+    ############################################################################
+    lib.attrsets.optionalAttrs (subBackendName == "cloud") {
+      openssh_hacks = rec {
+        commit = "5d8c5913c70723318acf47496e2abf7d2c99384f"; # OpenSSH version 9.8 (Branch "9.8");
+        # Not used locally. Needed to create the SSH "start.sh" script.
+        nix-store-path  = (__getFlake "github:fmaste/openssh-portable-hacks/${commit}").packages.x86_64-linux.openssh_hacks;
+        flake-reference = "github:fmaste/openssh-portable-hacks";
+        flake-output = "packages.x86_64-linux.openssh_hacks";
+        installable = "${flake-reference}/${commit}#${flake-output}";
+      };
+      rsync = rec {
+        nix-store-path  = pkgs.rsync; # Not used locally.
+        flake-reference = "github:intersectmbo/cardano-node";
+        flake-output = "legacyPackages.x86_64-linux.rsync";
+        # Same commit as the basic packages.
+        installable = "${flake-reference}/${pkgs.gitrev}#${flake-output}";
+      };
+    }
+    //
+    ############################################################################
+    # Binaries from `cardanoNodePackages` (node / tracer / generator). #########
+    ############################################################################
+    {
+      # Provided that all the "start.sh" scripts are taking it into account,
+      # this packages could be configured to come from a different commit than
+      # the one used to enter the shell ??????????
+      cardano-node = rec {
+        # Local reference only used if not "cloud".
+        nix-store-path = with pkgs;
+          # TODO: - cardano-node.passthru.profiled
+          #       - cardano-node.passthru.eventlogged
+          #       - cardano-node.passthru.asserted
+          # profileData.node-services."node-0".serviceConfig.value.eventlog
+          # builtins.trace (builtins.attrNames profileData.node-services."node-0".serviceConfig.value.eventlog) XXXX
+          if eventlogged
+            then cardanoNodePackages.cardano-node.passthru.eventlogged
+            else cardanoNodePackages.cardano-node
+        ;
+        flake-reference = "github:intersectmbo/cardano-node";
+        flake-output =
+          if eventlogged
+            then "cardanoNodePackages.cardano-node.passthru.eventlogged"
+            else "cardanoNodePackages.cardano-node"
+        ;
+        installable = "${flake-reference}/${gitrev}#${flake-output}";
+      };
+      cardano-cli = rec {
+        # Local reference only used if not "cloud".
+        nix-store-path = pkgs.cardanoNodePackages.cardano-cli;
+        flake-reference = "github:input-output-hk/cardano-cli";
+        flake-output = "cardanoNodePackages.cardano-cli";
+        installable = "${flake-reference}/${gitrev}#${flake-output}";
+      };
+      cardano-tracer = rec {
+        # Local reference only used if not "cloud".
+        nix-store-path = pkgs.cardanoNodePackages.cardano-tracer;
+        flake-reference = "github:intersectmbo/cardano-node";
+        flake-output = "cardanoNodePackages.cardano-tracer";
+        installable = "${flake-reference}/${gitrev}#${flake-output}";
+      };
+      tx-generator = rec {
+        # Local reference only used if not "cloud".
+        nix-store-path = pkgs.cardanoNodePackages.tx-generator;
+        flake-reference = "github:intersectmbo/cardano-node";
+        flake-output = "cardanoNodePackages.tx-generator";
+        installable = "${flake-reference}/${gitrev}#${flake-output}";
+      };
+    }
+  ;
+
+  # The "exec" or "cloud" Nomad Job description.
+  nomad-job = {profileData, containerPkgs}:
+    # TODO: Repeated code, add the generator's node name to profile.json
+    let generatorTaskName = if builtins.hasAttr "explorer" profileData.node-specs.value
+      then "explorer"
+      else "node-0"
+      ;
+    in
+      { inherit generatorTaskName; }
+      //
+      lib.attrsets.optionalAttrs (subBackendName == "exec") {
+        exec = {
+          # TODO: oneTracerPerGroup
+          oneTracerPerCluster = import ./nomad-job.nix
+            { inherit pkgs lib stateDir;
+              inherit profileData;
+              inherit generatorTaskName;
+              inherit containerPkgs;
+              oneTracerPerNode = false;
+              withSsh = false;
+            };
+          oneTracerPerNode = import ./nomad-job.nix
+            { inherit pkgs lib stateDir;
+              inherit profileData;
+              inherit generatorTaskName;
+              inherit containerPkgs;
+              oneTracerPerNode = true;
+              withSsh = false;
+            };
+        };
+      }
+      //
+      lib.attrsets.optionalAttrs (subBackendName == "cloud") {
+        cloud = {
+          # Always "oneTracerPerNode"
+          # TODO: oneTracerPerCluster and oneTracerPerGroup
+          nomadExec = import ./nomad-job.nix
+            { inherit pkgs lib stateDir;
+              inherit profileData;
+              inherit generatorTaskName;
+              inherit containerPkgs;
+              oneTracerPerNode = true;
+              withSsh = false;
+            };
+          ssh = import ./nomad-job.nix
+            { inherit pkgs lib stateDir;
+              inherit profileData;
+              inherit generatorTaskName;
+              inherit containerPkgs;
+              oneTracerPerNode = true;
+              withSsh = true;
+            };
+          # AWS S3 bucket that will be used to deploy the genesis files.
+          s3 = rec {
+            bucket = "cardano-perf-deploy";
+            region = "eu-central-1";
+            host   = "s3.${region}.amazonaws.com";
+            uri    = "https://${bucket}.${host}";
           };
-          nomadJob =
-            { inherit generatorTaskName; }
-            //
-            lib.attrsets.optionalAttrs (subBackendName == "exec") {
-              exec = {
-                # TODO: oneTracerPerGroup
-                oneTracerPerCluster = import ./nomad-job.nix
-                  { inherit pkgs lib stateDir;
-                    inherit profileData;
-                    inherit containerSpecs;
-                    inherit generatorTaskName;
-                    oneTracerPerNode = false;
-                    withSsh = false;
-                  };
-                oneTracerPerNode = import ./nomad-job.nix
-                  { inherit pkgs lib stateDir;
-                    inherit profileData;
-                    inherit containerSpecs;
-                    inherit generatorTaskName;
-                    oneTracerPerNode = true;
-                    withSsh = false;
-                  };
-              };
-            }
-            //
-            lib.attrsets.optionalAttrs (subBackendName == "cloud") {
-              cloud = {
-                # Always "oneTracerPerNode"
-                # TODO: oneTracerPerCluster and oneTracerPerGroup
-                nomadExec = import ./nomad-job.nix
-                  { inherit pkgs lib stateDir;
-                    inherit profileData;
-                    inherit containerSpecs;
-                    inherit generatorTaskName;
-                    oneTracerPerNode = true;
-                    withSsh = false;
-                  };
-                ssh = import ./nomad-job.nix
-                  { inherit pkgs lib stateDir;
-                    inherit profileData;
-                    inherit containerSpecs;
-                    inherit generatorTaskName;
-                    oneTracerPerNode = true;
-                    withSsh = true;
-                  };
-                # AWS S3 bucket that will be used to deploy the genesis files.
-                s3 = rec {
-                  bucket = "cardano-perf-deploy";
-                  region = "eu-central-1";
-                  host   = "s3.${region}.amazonaws.com";
-                  uri    = "https://${bucket}.${host}";
-                };
-              };
-            }
-          ;
         };
-      in pkgs.runCommand "workbench-backend-output-${profileData.profileName}-nomad"
-        ({
-          containerSpecsJSON = pkgs.writeText "workbench-cluster-container-pkgs.json"
-            (lib.generators.toJSON {} containerSpecs);
-        })
-        ''
-        mkdir $out
-        ln -s $containerSpecsJSON      $out/container-specs.json
-        '';
+      }
+    ;
 
 in { inherit materialise-profile; }
diff --git a/nix/workbench/backend/nomad/cloud.nix b/nix/workbench/backend/nomad/cloud.nix
index 7b949dea1c8..041d9fb0600 100644
--- a/nix/workbench/backend/nomad/cloud.nix
+++ b/nix/workbench/backend/nomad/cloud.nix
@@ -26,6 +26,8 @@ let
       )
       # Amazon S3 HTTP to upload/download the genesis tar file.
       pkgs.awscli
+      # Used to download the logs.
+      pkgs.rsync
     ]
   ;
 
diff --git a/nix/workbench/backend/nomad/exec.sh b/nix/workbench/backend/nomad/exec.sh
index b9139da67ab..38a21875f9e 100644
--- a/nix/workbench/backend/nomad/exec.sh
+++ b/nix/workbench/backend/nomad/exec.sh
@@ -193,7 +193,7 @@ allocate-run-nomadexec() {
   ## Empty the global namespace. Local runs ignore "${NOMAD_NAMESPACE:-}"
   backend_nomad allocate-run-nomad-job-patch-namespace "${dir}"
   # Will set the /nix/store paths from ".nix-store-path" in container-specs.json
-# backend_nomad allocate-run-nomad-job-patch-nix "${dir}"
+  backend_nomad allocate-run-nomad-job-patch-nix "${dir}"
 }
 
 # Called by `run.sh` without exit trap (unlike `scenario_setup_exit_trap`)!
diff --git a/nix/workbench/service/ssh.nix b/nix/workbench/service/ssh.nix
index 49f41ff6ea9..798aa8f49a6 100644
--- a/nix/workbench/service/ssh.nix
+++ b/nix/workbench/service/ssh.nix
@@ -1,7 +1,7 @@
 { pkgs
 , bashInteractive
 , coreutils
-, openssh_hacks
+, sshdExecutable
 }:
 
 with pkgs.lib;
@@ -46,7 +46,7 @@ in {
       # Specifies the name  of the configuration file. The default is
       # /etc/ssh/sshd_config. sshd refuses to start if there is no configuration
       # file.
-      ${openssh_hacks}/bin/sshd -D -e -f /local/run/current/ssh/sshd_config
+      ${sshdExecutable} -D -e -f /local/run/current/ssh/sshd_config
     '';
   };