feat: Update demos for Release 24.11

.yamllint.yaml

@@ -7,7 +7,4 @@ rules:
     check-keys: false
   comments:
     min-spaces-from-content: 1  # Needed due to https://github.com/adrienverge/yamllint/issues/443
-  braces:
-    ignore: |
-      stacks/signal-processing/*
-      stacks/_templates/*
+  braces: disable # because the yaml files are templates which can have {{ ... }}

demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml

@@ -43,13 +43,13 @@ data:
         stackable.tech/vendor: Stackable
     spec:
       sparkImage:
-        productVersion: 3.5.1
+        productVersion: 3.5.2
       mode: cluster
       mainApplicationFile: local:///stackable/spark/jobs/spark-ingest-into-lakehouse.py
       deps:
         packages:
-          - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.5.0
-          - org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.1
+          - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1
+          - org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.2
       s3connection:
         reference: minio
       sparkConf:

demos/end-to-end-security/create-spark-report.yaml

@@ -55,7 +55,7 @@ data:
       name: spark-report
     spec:
       sparkImage:
-        productVersion: 3.5.1
+        productVersion: 3.5.2
       mode: cluster
       mainApplicationFile: local:///stackable/spark/jobs/spark-report.py
       deps:

demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml

@@ -11,14 +11,15 @@ spec:
           # We use 24.3.0 here which contains the distcp MapReduce components
           # This is not included in the 24.7 images and will fail.
           # See: https://github.com/stackabletech/docker-images/issues/793
-          image: docker.stackable.tech/stackable/hadoop:3.3.4-stackable24.3.0
+          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev
           env:
             - name: HADOOP_USER_NAME
               value: stackable
             - name: HADOOP_CONF_DIR
               value: "/stackable/conf/hdfs"
             - name: HADOOP_CLASSPATH
               value: "/stackable/hadoop/share/hadoop/tools/lib/*.jar"
+          # yamllint disable-line rule:line-length
           command: ["bash", "-c", "bin/hdfs dfs -mkdir -p /data/raw && bin/hadoop distcp -D fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider s3a://public-backup-nyc-tlc/cycling-tripdata/demo-cycling-tripdata.csv.gz hdfs://hdfs/data/raw"]
           volumeMounts:
             - name: config-volume-hdfs

demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml

@@ -8,13 +8,15 @@ spec:
     spec:
       containers:
         - name: load-ny-taxi-data
-          image: docker.stackable.tech/stackable/hadoop:3.3.4-stackable24.7.0
+          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev
+          # yamllint disable rule:line-length
           command: ["bash", "-c", "/stackable/hadoop/bin/hdfs dfs -mkdir -p /ny-taxi-data/raw \
           && cd /tmp \
           && for month in 2020-09; do \
             curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet \
             && /stackable/hadoop/bin/hdfs dfs -put fhvhv_tripdata_$month.parquet /ny-taxi-data/raw/; \
           done"]
+          # yamllint enable rule:line-length
           volumeMounts:
             - name: hdfs-discovery-configmap
               mountPath: /hdfs

demos/signal-processing/DownloadAndWriteToDB.xml

@@ -61,7 +61,7 @@
             <bundle>
                 <artifact>nifi-dbcp-service-nar</artifact>
                 <group>org.apache.nifi</group>
-                <version>1.21.0</version>
+                <version>1.27.0</version>
             </bundle>
             <comments></comments>
             <descriptors>
@@ -258,7 +258,7 @@
             <bundle>
                 <artifact>nifi-record-serialization-services-nar</artifact>
                 <group>org.apache.nifi</group>
-                <version>1.21.0</version>
+                <version>1.27.0</version>
             </bundle>
             <comments></comments>
             <descriptors>
@@ -561,7 +561,7 @@
             </position>
             <height>88.0</height>
             <label>This flow downloads a dataset, writing it to a temporary table in TimescaleDB.
-This data is then written to the target table with the time offsets preserved, 
+This data is then written to the target table with the time offsets preserved,
 but re-based to the current time. This means that the data can be displayed
 in Grafana as if it were being streamed, whereas in fact the dashboard moves
 through "future" data that has already been persisted.</label>
@@ -584,7 +584,7 @@ through "future" data that has already been persisted.</label>
             <bundle>
                 <artifact>nifi-standard-nar</artifact>
                 <group>org.apache.nifi</group>
-                <version>1.21.0</version>
+                <version>1.27.0</version>
             </bundle>
             <config>
                 <backoffMechanism>PENALIZE_FLOWFILE</backoffMechanism>
@@ -1069,7 +1069,7 @@ through "future" data that has already been persisted.</label>
             <bundle>
                 <artifact>nifi-standard-nar</artifact>
                 <group>org.apache.nifi</group>
-                <version>1.21.0</version>
+                <version>1.27.0</version>
             </bundle>
             <config>
                 <backoffMechanism>PENALIZE_FLOWFILE</backoffMechanism>
@@ -1223,7 +1223,7 @@ from conditions_temp;</value>
             <bundle>
                 <artifact>nifi-standard-nar</artifact>
                 <group>org.apache.nifi</group>
-                <version>1.21.0</version>
+                <version>1.27.0</version>
             </bundle>
             <config>
                 <backoffMechanism>PENALIZE_FLOWFILE</backoffMechanism>

demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml

@@ -37,7 +37,7 @@ data:
       name: spark-ad
     spec:
       sparkImage:
-        productVersion: 3.5.1
+        productVersion: 3.5.2
       mode: cluster
       mainApplicationFile: local:///spark-scripts/spark-ad.py
       deps:

demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml

@@ -9,6 +9,7 @@ spec:
       containers:
         - name: load-ny-taxi-data
           image: "bitnami/minio:2022-debian-10"
+          # yamllint disable-line rule:line-length
           command: ["bash", "-c", "cd /tmp && for month in 2020-09 2020-10 2020-11 2020-12; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet && mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && mc cp fhvhv_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/ && mc mb --ignore-existing minio/prediction; done"]
           volumeMounts:
             - name: minio-s3-credentials

demos/trino-taxi-data/load-test-data.yaml

@@ -9,6 +9,7 @@ spec:
       containers:
         - name: load-ny-taxi-data
           image: "bitnami/minio:2024-debian-12"
+          # yamllint disable-line rule:line-length
           command: ["bash", "-c", "cd /tmp && for month in 2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/yellow_tripdata_$month.parquet && mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && mc cp yellow_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/; done"]
           volumeMounts:
             - name: minio-s3-credentials

docs/modules/demos/pages/trino-iceberg.adoc

@@ -78,7 +78,7 @@ As an alternative, you can use https://trino.io/download.html[trino-cli] by runn
 
 [source,console]
 ----
-$ java -jar ~/Downloads/trino-cli-451-executable.jar --user admin --insecure --password --server https://172.18.0.2:30856
+$ java -jar ~/Downloads/trino-cli-455-executable.jar --user admin --insecure --password --server https://172.18.0.2:30856
 ----
 
 Make sure to replace the server endpoint with the endpoint listed in the `stackablectl stacklet list` output.

stacks/_templates/vector-aggregator.yaml

@@ -4,11 +4,11 @@ name: vector
 repo:
   name: vector
   url: https://helm.vector.dev
-version: 0.37.0
+version: 0.36.1 # app version 0.41.1
 options:
   commonLabels:
     stackable.tech/vendor: Stackable
-  podLabels: # Doesn't seem to work?
+  podLabels:
     stackable.tech/vendor: Stackable
   role: Aggregator
   customConfig:
@@ -26,7 +26,7 @@ options:
           - https://opensearch-cluster-master.default.svc.cluster.local:9200
         mode: bulk
         # The auto-detection of the API version does not work in Vector
-        # 0.39.0 for OpenSearch, so the version must be set explicitly
+        # 0.41.1 for OpenSearch, so the version must be set explicitly
         # (see https://github.com/vectordotdev/vector/issues/17690).
         api_version: v8
         tls:

stacks/airflow/airflow.yaml

@@ -1,12 +1,12 @@
-{% raw %}
 ---
+# {% raw %}
 apiVersion: airflow.stackable.tech/v1alpha1
 kind: AirflowCluster
 metadata:
   name: airflow
 spec:
   image:
-    productVersion: 2.9.2
+    productVersion: 2.9.3
   clusterConfig:
     listenerClass: external-unstable
     loadExamples: false
@@ -272,7 +272,7 @@ data:
     spec:
       version: "1.0"
       sparkImage:
-        productVersion: 3.5.1
+        productVersion: 3.5.2
       mode: cluster
       mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py
       job:
@@ -285,22 +285,22 @@ data:
               limit: 512Mi
       driver:
         config:
-          resources: 
+          resources:
             cpu:
               min: 1000m
               max: 1200m
             memory:
               limit: 1024Mi
       executor:
         config:
-          resources: 
+          resources:
             cpu:
               min: 500m
               max: 1000m
             memory:
               limit: 1024Mi
         replicas: 3
-{% endraw %}
+# {% endraw %}
 ---
 apiVersion: v1
 kind: Secret

stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml

@@ -5,7 +5,7 @@ metadata:
   name: nifi
 spec:
   image:
-    productVersion: 1.25.0
+    productVersion: 1.27.0
   clusterConfig:
     authentication:
       - authenticationClass: nifi-admin-credentials

stacks/data-lakehouse-iceberg-trino-spark/trino.yaml

@@ -5,7 +5,7 @@ metadata:
   name: trino
 spec:
   image:
-    productVersion: "451"
+    productVersion: "455"
   clusterConfig:
     listenerClass: external-unstable
     catalogLabelSelector:
@@ -114,7 +114,7 @@ metadata:
   name: opa
 spec:
   image:
-    productVersion: 0.66.0
+    productVersion: 0.67.1
   servers:
     roleGroups:
       default: {}

stacks/dual-hive-hdfs-s3/hdfs.yaml

@@ -25,7 +25,7 @@ metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.3.4
+    productVersion: 3.4.0
   clusterConfig:
     listenerClass: external-unstable
     dfsReplication: 1

stacks/dual-hive-hdfs-s3/trino.yaml

@@ -61,7 +61,7 @@ metadata:
   name: trino
 spec:
   image:
-    productVersion: "451"
+    productVersion: "455"
   clusterConfig:
     authorization:
       opa:

stacks/end-to-end-security/hdfs.yaml

@@ -1,10 +1,11 @@
+---
 apiVersion: hdfs.stackable.tech/v1alpha1
 kind: HdfsCluster
 metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.3.4
+    productVersion: 3.4.0
   clusterConfig:
     zookeeperConfigMapName: hdfs-znode
     authentication:
@@ -29,8 +30,9 @@ spec:
                 level: DEBUG
     configOverrides: &configOverrides
       core-site.xml:
-        # The idea is that the user "hive" can't do anything in hdfs, *but* it can impersonate other users
-        # (such as trino), that have the needed permissions
+        # The idea is that the user "hive" can't do anything in hdfs,
+        # *but* it can impersonate other users (such as trino),
+        # that have the needed permissions
         hadoop.proxyuser.hive.users: "*"
         hadoop.proxyuser.hive.hosts: "*"
     roleGroups:

stacks/end-to-end-security/opa.yaml

@@ -5,7 +5,7 @@ metadata:
   name: opa
 spec:
   image:
-    productVersion: 0.66.0
+    productVersion: 0.67.1
   clusterConfig:
     userInfo:
       backend:

stacks/end-to-end-security/trino-regorules.yaml

@@ -1,5 +1,5 @@
-{% raw %}
 ---
+# {% raw %}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -1460,7 +1460,7 @@ data:
     #         "groups": ["group1", ...],
     #         "user": "username",
     #       },
-    #       "softwareStack": {"trinoVersion": "442"},
+    #       "softwareStack": {"trinoVersion": "455"},
     #     }
     #   }
     #
@@ -1610,7 +1610,7 @@ data:
     #         "groups": ["group1", ...],
     #         "user": "username",
     #       },
-    #       "softwareStack": {"trinoVersion": "442"},
+    #       "softwareStack": {"trinoVersion": "455"},
     #     }
     #   }
     #
@@ -1671,7 +1671,7 @@ data:
     #         "groups": ["group1", ...],
     #         "user": "username",
     #       },
-    #       "softwareStack": {"trinoVersion": "442"},
+    #       "softwareStack": {"trinoVersion": "455"},
     #     }
     #   }
     #
@@ -1736,7 +1736,7 @@ data:
     #         "groups": ["group1", ...],
     #         "user": "username",
     #       },
-    #       "softwareStack": {"trinoVersion": "442"},
+    #       "softwareStack": {"trinoVersion": "455"},
     #     }
     #   }
     #
@@ -1797,4 +1797,4 @@ data:
 
       regex.match(pattern_with_anchors, value)
     }
-{% endraw %}
+# {% endraw %}

stacks/end-to-end-security/trino.yaml

@@ -5,7 +5,7 @@ metadata:
   name: trino
 spec:
   image:
-    productVersion: "451"
+    productVersion: "455"
   clusterConfig:
     listenerClass: external-unstable
     tls:

stacks/hdfs-hbase/hdfs.yaml

@@ -1,10 +1,11 @@
+---
 apiVersion: hdfs.stackable.tech/v1alpha1
 kind: HdfsCluster
 metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.3.4
+    productVersion: 3.4.0
   clusterConfig:
     dfsReplication: 1
     zookeeperConfigMapName: hdfs-znode

stacks/jupyterhub-pyspark-hdfs/hdfs.yaml

@@ -13,7 +13,7 @@ metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.3.4
+    productVersion: 3.4.0
   clusterConfig:
     dfsReplication: 1
     zookeeperConfigMapName: hdfs-znode