New airflow version 2.6.3, 2.7.2 (#334)

* adapt supported versions * bump docs and examples to 2.7.1 * fix python imports to be compatible with 2.7.1 * adapted changelog * use 2.7.2 instead of 2.7.1 * bump operator-rs to 0.55.0 * mention vector and operator-rs bumps * increase memory of scheudler * increase celery worker memory for testing * remove extra resources and use 2 celery workers * use 1 worker and increase memory * fix tests * remove pullpolicy * remove pullpolicy * improve comment in test
stackabletech · Oct 20, 2023 · 4a2a5b2 · 4a2a5b2
1 parent 41d4e5c
commit 4a2a5b2
Show file tree

Hide file tree

Showing 21 changed files with 126 additions and 62 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,25 +7,31 @@
 - [BREAKING] Implement KubernetesExecutor ([#311]).
 - Default stackableVersion to operator version ([#312]).
 - Support PodDisruptionBudgets ([#330]).
+- Added support for versions 2.6.3, 2.7.2 ([#334]).
 
 ### Changed
 
 - [BREAKING] Consolidated `spec.clusterConfig.authenticationConfig` to `spec.clusterConfig.authentication` which now takes a vector of AuthenticationClass references  ([#303]).
-- `vector` `0.26.0` -> `0.31.0` ([#308]).
-- `operator-rs` `0.44.0` -> `0.52.1` ([#308], [#330]).
+- `vector` `0.26.0` -> `0.33.0` ([#308], [#334]).
+- `operator-rs` `0.44.0` -> `0.55.0` ([#308], [#330], [#334]).
 - [BREAKING] Removed AirflowDB object, since it created some problems when reinstalling or upgrading an Airflow cluster. Instead, the initialization of the database was moved to the startup phase of each scheduler pod. To make sure the initialization does not run in parallel, the `PodManagementPolicy` of the scheduler StatefulSet was set to `OrderedReady`. The `.spec.clusterConfig.databaseInitialization` option was removed from the CRD, since it was just there to enable logging for the database initialization Job, which doesn't exist anymore ([#322]).
 
 ### Fixed
 
 - BREAKING: Rename Service port name from `airflow` to `http` for consistency reasons. This change should normally not be breaking, as we only change the name, not the port. However, there might be some e.g. Ingresses that rely on the port name and need to be updated ([#316]).
 - Fix user-supplied gitsync git-config settings ([#335]).
 
+### Removed
+
+- Removed support for 2.2.3, 2.2.4, 2.2.5, 2.4.1 ([#334]).
+
 [#303]: https://github.com/stackabletech/airflow-operator/pull/303
 [#308]: https://github.com/stackabletech/airflow-operator/pull/308
 [#311]: https://github.com/stackabletech/airflow-operator/pull/311
 [#312]: https://github.com/stackabletech/airflow-operator/pull/312
 [#316]: https://github.com/stackabletech/airflow-operator/pull/316
 [#330]: https://github.com/stackabletech/airflow-operator/pull/330
+[#334]: https://github.com/stackabletech/airflow-operator/pull/334
 [#335]: https://github.com/stackabletech/airflow-operator/pull/335
 
 ## [23.7.0] - 2023-07-14

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -22,7 +22,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.7"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.52.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.55.0" }
 strum = { version = "0.25", features = ["derive"] }
 tokio = { version = "1.29", features = ["full"] }
 tracing = "0.1"

diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml
diff --git a/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml b/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml
@@ -5,7 +5,7 @@ metadata:
   name: airflow
 spec:
   image:
-    productVersion: 2.6.1
+    productVersion: 2.7.2
   clusterConfig:
     loadExamples: false
     exposeConfig: false

diff --git a/docs/modules/airflow/examples/example-airflow-gitsync.yaml b/docs/modules/airflow/examples/example-airflow-gitsync.yaml
@@ -5,7 +5,7 @@ metadata:
   name: airflow
 spec:
   image:
-    productVersion: "2.6.1"
+    productVersion: "2.7.2"
   clusterConfig:
     loadExamples: false
     exposeConfig: false

diff --git a/docs/modules/airflow/examples/example-airflow-incluster.yaml b/docs/modules/airflow/examples/example-airflow-incluster.yaml
@@ -5,7 +5,7 @@ metadata:
   name: airflow
 spec:
   image:
-    productVersion: 2.6.1
+    productVersion: 2.7.2
   clusterConfig:
     loadExamples: false
     exposeConfig: false

diff --git a/docs/modules/airflow/examples/getting_started/code/airflow.yaml b/docs/modules/airflow/examples/getting_started/code/airflow.yaml
@@ -5,7 +5,7 @@ metadata:
   name: airflow
 spec:
   image:
-    productVersion: 2.6.1
+    productVersion: 2.7.2
   clusterConfig:
     loadExamples: true
     exposeConfig: false

diff --git a/docs/modules/airflow/pages/usage-guide/security.adoc b/docs/modules/airflow/pages/usage-guide/security.adoc
@@ -23,7 +23,7 @@ metadata:
   name: airflow-with-ldap
 spec:
   image:
-    productVersion: 2.6.1
+    productVersion: 2.7.2
   clusterConfig:
     authentication:
       - authenticationClass: ldap    # <1>

diff --git a/docs/modules/airflow/partials/supported-versions.adoc b/docs/modules/airflow/partials/supported-versions.adoc
@@ -2,8 +2,6 @@
 // This is a separate file, since it is used by both the direct Airflow-Operator documentation, and the overarching
 // Stackable Platform documentation.
 
-- 2.2.3
-- 2.2.4
-- 2.2.5
-- 2.4.1
-- 2.6.1
+- 2.7.2
+- 2.6.3
+- 2.6.1 (deprecated)
diff --git a/examples/simple-airflow-cluster-dags-cmap.yaml b/examples/simple-airflow-cluster-dags-cmap.yaml
@@ -81,7 +81,7 @@ metadata:
   name: airflow-dags-cmap
 spec:
   image:
-    productVersion: 2.6.1
+    productVersion: 2.7.2
     stackableVersion: 0.0.0-dev
   clusterConfig:
     loadExamples: false

diff --git a/examples/simple-airflow-cluster-ldap-insecure-tls.yaml b/examples/simple-airflow-cluster-ldap-insecure-tls.yaml
@@ -147,7 +147,7 @@ metadata:
   name: airflow-insecure-tls
 spec:
   image:
-    productVersion: 2.6.1
+    productVersion: 2.7.2
     stackableVersion: 0.0.0-dev
   clusterConfig:
     loadExamples: true

diff --git a/examples/simple-airflow-cluster-ldap.yaml b/examples/simple-airflow-cluster-ldap.yaml
@@ -145,7 +145,7 @@ metadata:
   name: airflow-with-ldap-server-veri-tls
 spec:
   image:
-    productVersion: 2.6.1
+    productVersion: 2.7.2
     stackableVersion: 0.0.0-dev
   clusterConfig:
     loadExamples: true

diff --git a/examples/simple-airflow-cluster.yaml b/examples/simple-airflow-cluster.yaml
@@ -21,7 +21,7 @@ metadata:
   name: airflow
 spec:
   image:
-    productVersion: 2.6.1
+    productVersion: 2.7.2
     stackableVersion: 0.0.0-dev
   clusterConfig:
     loadExamples: true

diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs
@@ -62,7 +62,7 @@ mod tests {
           name: airflow
         spec:
           image:
-            productVersion: 2.6.1
+            productVersion: 2.7.2
           loadExamples: true
           exposeConfig: false
           credentialsSecret: simple-airflow-credentials
@@ -159,7 +159,7 @@ mod tests {
           name: airflow
         spec:
           image:
-            productVersion: 2.6.1
+            productVersion: 2.7.2
           loadExamples: true
           exposeConfig: false
           credentialsSecret: simple-airflow-credentials

diff --git a/rust/crd/src/git_sync.rs b/rust/crd/src/git_sync.rs
@@ -70,7 +70,7 @@ mod tests {
           name: airflow
         spec:
           image:
-            productVersion: 2.6.1
+            productVersion: 2.7.2
           clusterConfig:
             loadExamples: false
             exposeConfig: false
@@ -116,7 +116,7 @@ mod tests {
           name: airflow
         spec:
           image:
-            productVersion: 2.6.1
+            productVersion: 2.7.2
           clusterConfig:
             loadExamples: false
             exposeConfig: false
@@ -189,7 +189,7 @@ mod tests {
           name: airflow
         spec:
           image:
-            productVersion: 2.6.1
+            productVersion: 2.7.2
           clusterConfig:
             loadExamples: false
             exposeConfig: false

diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
@@ -713,7 +713,7 @@ mod tests {
           name: airflow
         spec:
           image:
-            productVersion: 2.6.1
+            productVersion: 2.7.2
           clusterConfig:
             loadExamples: true
             exposeConfig: true
@@ -737,7 +737,7 @@ mod tests {
         let resolved_airflow_image: ResolvedProductImage =
             cluster.spec.image.resolve("airflow", "0.0.0-dev");
 
-        assert_eq!("2.6.1", &resolved_airflow_image.product_version);
+        assert_eq!("2.7.2", &resolved_airflow_image.product_version);
 
         assert_eq!("KubernetesExecutor", cluster.spec.executor.to_string());
         assert!(cluster.spec.cluster_config.load_examples.unwrap_or(false));

diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs
@@ -41,7 +41,6 @@ use stackable_operator::{
         product_image_selection::ResolvedProductImage,
         rbac::build_rbac_resources,
     },
-    duration::Duration,
     k8s_openapi::{
         api::{
             apps::v1::{StatefulSet, StatefulSetSpec},
@@ -69,6 +68,7 @@ use stackable_operator::{
         compute_conditions, operations::ClusterOperationsConditionBuilder,
         statefulset::StatefulSetConditionBuilder,
     },
+    time::Duration,
 };
 use std::{
     collections::{BTreeMap, HashMap},

diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs
@@ -9,7 +9,7 @@ use std::collections::BTreeMap;
 
 pub const PYTHON_IMPORTS: &[&str] = &[
     "import os",
-    "from airflow.www.fab_security.manager import (AUTH_DB, AUTH_LDAP, AUTH_OAUTH, AUTH_OID, AUTH_REMOTE_USER)",
+    "from flask_appbuilder.const import (AUTH_DB, AUTH_LDAP, AUTH_OAUTH, AUTH_OID, AUTH_REMOTE_USER)",
     "basedir = os.path.abspath(os.path.dirname(__file__))",
     "WTF_CSRF_ENABLED = True",
 ];

diff --git a/tests/templates/kuttl/ldap/06-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/ldap/06-install-airflow-cluster.yaml.j2
@@ -69,6 +69,11 @@ commands:
           config:
             logging:
               enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+            # There are some OOM kill events when using celery executor with the default (currently 1.75Gi)
+            # resources and version 2.7.2, so we provide more memory here.
+            resources:
+              memory:
+                limit: '2.5Gi'
           roleGroups:
             default:
               replicas: 1

diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml
@@ -7,13 +7,12 @@
 dimensions:
   - name: airflow
     values:
-      - 2.2.4
-      - 2.2.5
-      - 2.4.1
       - 2.6.1
+      - 2.6.3
+      - 2.7.2
   - name: airflow-latest
     values:
-      - 2.6.1
+      - 2.7.2
   - name: ldap-authentication
     values:
       - no-tls