chore: add overrides integrationtests (#490)

* chore: add hint for pod template

* recapture pod_template to immutable

* add overrides integrationtest

* fix overrides integrationtest

* bump stackable-operator version

* add todo key

* remove recapturing for now

* remove recapturing for now

* test env overrides for celery executors

* reorganize steps

* test env overrides for k8s executors

* fix yaml lint

---------

Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>

Cargo.toml

@@ -22,7 +22,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.8"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.73.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.74.0" }
 strum = { version = "0.26", features = ["derive"] }
 tokio = { version = "1.39", features = ["full"] }
 tracing = "0.1"

tests/templates/kuttl/overrides/00-assert.yaml.j2

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: airflow-postgresql
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: airflow-redis-master
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: airflow-redis-replicas
+status:
+  availableReplicas: 1

tests/templates/kuttl/overrides/00-install-dbs.yaml.j2

@@ -0,0 +1,21 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: >-
+      helm install airflow-postgresql
+      --namespace $NAMESPACE
+      --version 12.1.5
+      --set auth.username=airflow
+      --set auth.password=airflow
+      --set auth.database=airflow
+      --repo https://charts.bitnami.com/bitnami postgresql
+    timeout: 240
+  - script: >-
+      helm install airflow-redis
+      --namespace $NAMESPACE
+      --version 17.3.7
+      --set auth.password=redis
+      --set replica.replicaCount=1
+      --repo https://charts.bitnami.com/bitnami redis
+    timeout: 240

tests/templates/kuttl/overrides/10-assert.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: test-available-condition
+timeout: 600
+commands:
+  - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow-celery --timeout 301s

tests/templates/kuttl/overrides/10-install-airflow.yaml.j2

@@ -0,0 +1,99 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: airflow-credentials
+type: Opaque
+stringData:
+  adminUser.username: airflow
+  adminUser.firstname: Airflow
+  adminUser.lastname: Admin
+  adminUser.email: airflow@airflow.com
+  adminUser.password: airflow
+  connections.secretKey: thisISaSECRET_1234
+  connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow
+  connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow
+  connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: test-override
+type: Opaque
+stringData:
+  adminUser.username: airflow
+  adminUser.firstname: Airflow
+  adminUser.lastname: Admin
+  adminUser.email: airflow@airflow.com
+  adminUser.password: airflow
+  connections.secretKey: thisISaSECRET_1234
+  connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow
+  connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow
+  connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0
+---
+apiVersion: airflow.stackable.tech/v1alpha1
+kind: AirflowCluster
+metadata:
+  name: airflow-celery
+spec:
+  image:
+    productVersion: "{{ test_scenario['values']['airflow-latest'] }}"
+  clusterConfig:
+    loadExamples: true
+    exposeConfig: false
+    listenerClass: external-unstable
+    credentialsSecret: airflow-credentials
+  webservers:
+    envOverrides:
+      COMMON_VAR: role-value # overridden by role group below
+      ROLE_VAR: role-value   # only defined here at role level
+      credentialsSecret: test-override
+    roleGroups:
+      default:
+        replicas: 1
+        envOverrides:
+          COMMON_VAR: group-value # overrides role value
+          GROUP_VAR: group-value # only defined here at group level
+  celeryExecutors:
+    envOverrides:
+      COMMON_VAR: role-value # overridden by role group below
+      ROLE_VAR: role-value   # only defined here at role level
+    roleGroups:
+      default:
+        replicas: 2
+        config:
+          resources:
+            cpu:
+              min: 410m
+        podOverrides:
+          spec:
+            containers:
+            - name: airflow
+              resources:
+                limits:
+                  cpu: 810m
+        envOverrides:
+          COMMON_VAR: group-value # overrides role value
+          GROUP_VAR: group-value # only defined here at group level
+    config:
+      resources:
+        cpu:
+          min: 400m
+          max: 800m
+        memory:
+          limit: 2Gi
+    podOverrides:
+      spec:
+        containers:
+        - name: airflow
+          resources:
+            limits:
+              cpu: 750m
+            requests:
+              cpu: 350m
+  schedulers:
+    roleGroups:
+      default:
+        replicas: 1
+        envOverrides:
+          credentialsSecret: test-override

tests/templates/kuttl/overrides/11-assert.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+commands:
+  #
+  # Test envOverrides
+  #
+  - script: |
+      kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")'
+      kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "credentialsSecret" and .value == "test-override")'
+
+
+  - script: |
+      kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")'

tests/templates/kuttl/overrides/20-assert.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: test-available-condition
+timeout: 600
+commands:
+  - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow-kubernetes --timeout 301s

tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2

@@ -0,0 +1,66 @@
+---
+apiVersion: airflow.stackable.tech/v1alpha1
+kind: AirflowCluster
+metadata:
+  name: airflow-kubernetes
+spec:
+  image:
+    productVersion: "{{ test_scenario['values']['airflow-latest'] }}"
+  clusterConfig:
+    loadExamples: true
+    exposeConfig: false
+    listenerClass: external-unstable
+    credentialsSecret: airflow-credentials
+  webservers:
+    roleGroups:
+      default:
+        replicas: 1
+        config:
+          resources:
+            cpu:
+              min: 410m
+        podOverrides:
+          spec:
+            containers:
+            - name: airflow
+              resources:
+                limits:
+                  cpu: 810m
+    config:
+      resources:
+        cpu:
+          min: 400m
+          max: 800m
+        memory:
+          limit: 2Gi
+    podOverrides:
+      spec:
+        containers:
+        - name: airflow
+          resources:
+            limits:
+              cpu: 750m
+            requests:
+              cpu: 350m
+  kubernetesExecutors:
+    config:
+      resources:
+        cpu:
+          min: 400m
+          max: 800m
+        memory:
+          limit: 2Gi
+    podOverrides:
+      spec:
+        containers:
+        - name: base
+          resources:
+            limits:
+              cpu: 750m
+    envOverrides:
+      ROLE_VAR: role-value   # there are no role groups for kubernetes executors
+      AIRFLOW__METRICS__STATSD_ON: "False" # also set by the operator
+  schedulers:
+    roleGroups:
+      default:
+        replicas: 1

tests/templates/kuttl/overrides/21-assert.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+commands:
+- script: |
+    kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | yq -e '.spec.containers.[0].resources.limits | select (.cpu == "750m")'
+    kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | yq -e '.spec.containers[] | select (.name == "base") | .env[] | select (.name == "AIRFLOW__METRICS__STATSD_ON" and .value == "False")'

tests/test-definition.yaml

@@ -69,6 +69,10 @@ tests:
     dimensions:
       - airflow-latest
       - openshift
+  - name: overrides
+    dimensions:
+      - airflow-latest
+      - openshift
 suites:
   - name: nightly
     # Run nightly with the latest airflow