Backport GPU provisioning updates to releases/2.10.0 (red-hat-data-services#1578)

Generalize GPU provisioning script to ease the addition of new providers (red-hat-data-services#1568)

* refactor gpu operators directories

* generalize GPU script away from AWS provider

* minor changes in kustomize yaml

* fix aws gpu overlay

* fix filepath

* add missing end lines

Author: bdattoma

ods_ci/tasks/Resources/Provisioning/GPU/amd_operator/amd_gpu_install.yaml renamed to ods_ci/tasks/Resources/Provisioning/GPU/AMD/amd_gpu_install.yaml

@@ -12,7 +12,7 @@ CSVNAME="$(oc get packagemanifests/gpu-operator-certified -n openshift-marketpla
 sed -i'' -e "0,/v1.11/s//$CHANNEL/g" -e "s/gpu-operator-certified.v1.11.0/$CSVNAME/g"  "$GPU_INSTALL_DIR/gpu_install.yaml"
 
 oc apply -f "$GPU_INSTALL_DIR/gpu_install.yaml"
-oc apply -f "$GPU_INSTALL_DIR/nfd_operator.yaml"
+oc apply -f "$GPU_INSTALL_DIR/../nfd_operator.yaml"
 echo "Wait for Nvidia GPU Operator Subscription, InstallPlan and Deployment to complete"
 
 oc wait --timeout=3m --for jsonpath='{.status.state}'=AtLatestKnown -n openshift-nfd sub nfd
@@ -80,7 +80,7 @@ function rerun_accelerator_migration() {
 }
 
 wait_until_pod_ready_status  "gpu-operator"
-oc apply -f "$GPU_INSTALL_DIR/nfd_deploy.yaml"
+oc apply -f "$GPU_INSTALL_DIR/../nfd_deploy.yaml"
 oc get csv -n nvidia-gpu-operator "$CSVNAME" -o jsonpath='{.metadata.annotations.alm-examples}' | jq .[0] > clusterpolicy.json
 oc apply -f clusterpolicy.json
 wait_until_pod_ready_status "nvidia-device-plugin-daemonset"

ods_ci/tasks/Resources/Provisioning/GPU/amd_operator/amd_operator.sh renamed to ods_ci/tasks/Resources/Provisioning/GPU/AMD/amd_operator.sh

@@ -3,6 +3,9 @@ set -e
 
 # Optional params
 INSTANCE_TYPE=${1:-"g4dn.xlarge"}
+PROVIDER=${2:-"AWS"}
+KUSTOMIZE_PATH="$PWD/tasks/Resources/Provisioning/Hive/GPU"
+MACHINESET_PATH="$KUSTOMIZE_PATH/base/source-machineset.yaml"
 
 # Check if existing machineset GPU already exists
 EXISTING_GPU_MACHINESET="$(oc get machineset -n openshift-machine-api -o jsonpath="{.items[?(@.metadata.annotations['machine\.openshift\.io/GPU']>'0')].metadata.name}")"
@@ -14,28 +17,18 @@ fi
 
 # Select the first machineset as a template for the GPU machineset
 SOURCE_MACHINESET=$(oc get machineset -n openshift-machine-api -o name | head -n1)
+oc get -o yaml -n openshift-machine-api $SOURCE_MACHINESET  > $MACHINESET_PATH
 
-# Reformat with jq, for better diff result.
-oc get -o json -n openshift-machine-api $SOURCE_MACHINESET  | jq -r > /tmp/source-machineset.json
-
-OLD_MACHINESET_NAME=$(jq '.metadata.name' -r /tmp/source-machineset.json )
+# rename machine set in the template file
+OLD_MACHINESET_NAME=$(yq '.metadata.name' $MACHINESET_PATH )
 NEW_MACHINESET_NAME=${OLD_MACHINESET_NAME/worker/gpu}
+sed -i'' -e "s/$OLD_MACHINESET_NAME/$NEW_MACHINESET_NAME/g" $MACHINESET_PATH
 
+# set the desired node flavor in the kustomize overlay
+sed -i'' -e "s/INSTANCE_TYPE/$INSTANCE_TYPE/g" $KUSTOMIZE_PATH/overlays/$PROVIDER/gpu.yaml
 
-# Change instanceType and delete some stuff
-jq -r --arg INSTANCE_TYPE "$INSTANCE_TYPE" '.spec.template.spec.providerSpec.value.instanceType=$INSTANCE_TYPE
-  | del(.metadata.selfLink)
-  | del(.metadata.uid)
-  | del(.metadata.creationTimestamp)
-  | del(.metadata.resourceVersion)
-' /tmp/source-machineset.json > /tmp/gpu-machineset.json
-
-# Change machineset name
-sed -i'' -e "s/$OLD_MACHINESET_NAME/$NEW_MACHINESET_NAME/g" /tmp/gpu-machineset.json
-# Create new machineset
-oc apply -f /tmp/gpu-machineset.json
-rm /tmp/source-machineset.json
-rm /tmp/gpu-machineset.json
+# create the new MachineSet using kustomize
+oc apply --kustomize $KUSTOMIZE_PATH/overlays/$PROVIDER
 
 # Add GPU label to the new machine-set
 oc patch machinesets -n openshift-machine-api "$NEW_MACHINESET_NAME" -p '{"metadata":{"labels":{"gpu-machineset":"true"}}}' --type=merge

ods_ci/tasks/Resources/Provisioning/GPU/amd_operator/blacklist_driver.yaml renamed to ods_ci/tasks/Resources/Provisioning/GPU/AMD/blacklist_driver.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - source-machineset.yaml

ods_ci/tasks/Resources/Provisioning/GPU/amd_operator/kmm_operator_install.yaml renamed to ods_ci/tasks/Resources/Provisioning/GPU/AMD/kmm_operator_install.yaml

@@ -0,0 +1 @@
+# PLACEHOLDER - the content is dynamically generated by ods_ci/tasks/Resources/Provisioning/GPU/provision-gpu.sh

ods_ci/tasks/Resources/Provisioning/GPU/gpu_deploy.sh renamed to ods_ci/tasks/Resources/Provisioning/GPU/NVIDIA/gpu_deploy.sh

@@ -0,0 +1,3 @@
+- op: replace
+  path: /spec/template/spec/providerSpec/value/instanceType
+  value: INSTANCE_TYPE

ods_ci/tasks/Resources/Provisioning/GPU/gpu_install.yaml renamed to ods_ci/tasks/Resources/Provisioning/GPU/NVIDIA/gpu_install.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+metadata:
+  name: add-gpu
+resources:
+  - ../../base/
+patches:
+  - path: gpu.yaml
+    target:
+      kind: MachineSet

ods_ci/tasks/Resources/Provisioning/Hive/AWS/provision-gpu.sh renamed to ods_ci/tasks/Resources/Provisioning/GPU/provision-gpu.sh

@@ -0,0 +1,3 @@
+- op: replace
+  path: /spec/template/spec/providerSpec/value/profile
+  value: INSTANCE_TYPE

ods_ci/tasks/Resources/Provisioning/Hive/GPU/base/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+metadata:
+  name: add-gpu
+resources:
+  - ../../base/
+patches:
+  - path: gpu.yaml
+    target:
+      kind: MachineSet