Added helm charts (#85)

Helm Charts for easy Kubernetes deployments.

With the Charts you can:

deploys KSML as StatefulSet
configures KSML Runner settings, like application server, prometheus export
define KSML Definition files
define Schema Files
define namespaces and corresponding KSML Definition file
logging configurations
Collect metrics with servicemonitor
Define an ingress for the application server
Allows setting of Kubernetes options for the deployment

Configure the VolumeClaimTemplate is made and configurable for state store
SecurityContext
ServiceAccount
Resources
Affinity
NodeSelector
topologySpreadConstraints
tolerations
The repo configured to build and release to multiple registries
Images to

Docker Hub docker.io/axual/ksml
GitHub KSML registry ghcr.io/axual/ksml
Axual OpenSource registry.axual.io/opensource/images/axual/ksml
Charts to
Axual OpenSource registry.axual.io/opensource/charts/ksml
KSML Charts are configured to pull from Axual OpenSource, since that is where the user pulled the charts from as well

.github/workflows/build-push-docker.yml

@@ -13,26 +13,33 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: set up Docker buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: login to GHCR
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: login to Docker hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: login to Axual Open Source Registry
+        uses: docker/login-action@v3
+        with:
+          registry: registry.axual.io
+          username: ${{ secrets.AXUAL_REGISTRY_USER }}
+          password: ${{ secrets.AXUAL_REGISTRY_TOKEN }}
+
       - name: build and push KSML Runner
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           file: Dockerfile
           target: ksml
@@ -41,3 +48,12 @@ jobs:
           tags: |
             ghcr.io/axual/ksml:snapshot
             axual/ksml:snapshot
+            registry.axual.io/opensource/images/axual/ksml:snapshot
+
+  helm-chart-release:
+    needs: build-push-docker
+    uses: './.github/workflows/package-push-helm.yml'
+    secrets: inherit
+    with:
+      app-version: snapshot
+      version: 0.0.0-snapshot

.github/workflows/package-push-helm.yml

@@ -1,12 +1,22 @@
-name: build and push Helm Charts
+name: Package and Push Helm Chart
 on:
   workflow_dispatch:
+    inputs:
+      app-version:
+        required: true
+        type: string
+        description: "The app version that should be set in the chart"
+      version:
+        required: false
+        default: 0.0.0-snapshot
+        type: string
+        description: "The version of the chart to be released"
   workflow_call:
     inputs:
-      tag-name:
+      app-version:
         required: true
         type: string
-      target-version:
+      version:
         required: false
         default: 0.0.0-snapshot
         type: string
@@ -21,23 +31,27 @@ jobs:
         run: |
           curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
 
+      - name: Checkout
+        uses: actions/checkout@v4
+
       - name: Package Helm
         shell: bash
         run: |
-          helm package --version '${{ inputs.target-version }}' --app-version '${{ inputs.tag-name }}' packaging/helm-charts/ksml
+          helm package --version '${{ inputs.version }}' --app-version '${{ inputs.app-version }}' ./packaging/helm-charts/ksml && ls ksml*.tgz
 
       - name: Login to registry
         shell: bash
         run: |
-          helm registry login '${{ env.AXUAL_REGISTRY }}' -u '${{ secrets.AXUAL_REGISTRY_USER }}' -p '${{ secrets.AXUAL_REGISTRY_TOKEN }}'
+          helm registry login registry.axual.io/opensource/charts -u '${{ secrets.AXUAL_REGISTRY_USER }}' -p '${{ secrets.AXUAL_REGISTRY_TOKEN }}'
 
       - name: Push to registry
         shell: bash
         run: |
-          helm push 'ksml-${{ inputs.target-version }}.tgz'
+          helm push 'ksml-${{ inputs.version }}.tgz' oci://registry.axual.io/opensource/charts
 
       - name: Log out of registry
         shell: bash
         if: ${{ always() }}
+
         run: |
-          helm registry logout '${{ env.AXUAL_REGISTRY }}' &
+          helm registry logout registry.axual.io/opensource/charts &

.github/workflows/release-push-docker.yml

@@ -1,10 +1,16 @@
 name: build and push release Docker images
 on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        required: true
+        type: string
+        description: "The image tag to create"
   release:
     types: [published]
 
 env:
-  RELEASE_TAG: ${{ github.event.release.tag_name }}
+  RELEASE_TAG: ${{ inputs.tag || github.event.release.tag_name }}
 
 jobs:
   release-push-docker:
@@ -33,6 +39,13 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: login to Axual Open Source Registry
+        uses: docker/login-action@v2
+        with:
+          registry: registry.axual.io
+          username: ${{ secrets.AXUAL_REGISTRY_USER }}
+          password: ${{ secrets.AXUAL_REGISTRY_TOKEN }}
+
       - name: build and push KSML Runner
         uses: docker/build-push-action@v4
         with:
@@ -42,5 +55,16 @@ jobs:
           push: true
           tags: |
             axual/ksml:${{ github.event.release.tag_name }}
+            axual/ksml:latest
             ghcr.io/axual/ksml:${{ github.event.release.tag_name }}
             ghcr.io/axual/ksml:latest
+            registry.axual.io/opensource/images/axual/ksml:${{ github.event.release.tag_name }}
+            registry.axual.io/opensource/images/axual/ksml:latest
+
+  helm-chart-release:
+    needs: release-push-docker
+    uses: './.github/workflows/package-push-helm.yml'
+    secrets: inherit
+    with:
+      app-version: ${{ inputs.tag || github.event.release.tag_name }}
+      version: ${{ inputs.tag || github.event.release.tag_name }}

.gitignore

@@ -13,3 +13,4 @@ examples/io.axual*
 examples/io.ksml*
 
 workspace/*
+/packaging/helm-charts/**/*.tgz

Dockerfile

@@ -11,7 +11,7 @@
 
 
 # Step 1: Create the common base image with the ksml user and group and the required packages
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9-1137 AS base
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.4-1134 AS base
 ENV LANG=en_US.UTF-8
 
 # Environment variable for Connect Build and Runtime
@@ -20,7 +20,8 @@ ENV PATH=/opt/graal/bin:$PATH \
     	KSML_INSTALL="/opt/ksml"
 
 RUN set -eux \
-        && microdnf install -y procps curl tar unzip gzip zlib openssl-devel gcc gcc-c++ make patch glibc-langpack-en libxcrypt shadow-utils \
+    	&& microdnf upgrade -y --nodocs \
+        && microdnf install -y --nodocs procps tar unzip gzip zlib openssl-devel gcc gcc-c++ make patch glibc-langpack-en libxcrypt shadow-utils \
 	&& groupadd --gid 1024 ksml \
         && useradd -g ksml -u 1024 -d "$KSML_HOME" -ms /bin/sh -f -1 ksml \
         && chown -R ksml:0 "$KSML_HOME" /opt                               \
@@ -50,7 +51,6 @@ RUN set -eux \
     && mkdir -p /opt/maven \
     && MVN_PKG=https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz \
     && curl --fail --silent --location --retry 3 ${MVN_PKG} | gunzip | tar x -C /opt/maven --strip-components=1 \
-#   && GRAALVM_PKG=https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-21.0.1/graalvm-community-jdk-21.0.1_linux-x64_bin.tar.gz
     && GRAALVM_PKG=https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-${GRAALVM_JDK_VERSION}/graalvm-community-jdk-${GRAALVM_JDK_VERSION}_linux-${JAVA_ARCH}_bin.tar.gz \
     && mkdir -p /opt/graal \
     && curl --fail --silent --location --retry 3 ${GRAALVM_PKG} | gunzip | tar x -C /opt/graal --strip-components=1
@@ -73,18 +73,17 @@ ENV JAVA_HOME=/opt/graalvm
 COPY --chown=ksml:0 --from=graal-builder /opt/graal/ /opt/graal/
 
 WORKDIR /home/ksml
-USER ksml
+USER 1024
 #There is no more GraalPy command here and no venv
 #RUN graalpy -m venv graalenv && \
 #    echo "source $HOME/graalenv/bin/activate" >> ~/.bashrc
 
 
 # Step 5: Create the KSML Runner image
 FROM ksml-graal AS ksml
-COPY --chown=ksml:0 --from=builder /project_dir/ksml-runner/NOTICE.txt /licences/THIRD-PARTY-LICENSES.txt
-COPY --chown=ksml:0 --from=builder /project_dir/LICENSE.txt /licences/LICENSE.txt
+COPY --chown=ksml:0 --from=builder /project_dir/ksml-runner/NOTICE.txt /licenses/THIRD-PARTY-LICENSES.txt
+COPY --chown=ksml:0 --from=builder /project_dir/LICENSE.txt /licenses/LICENSE.txt
 COPY --chown=ksml:0 --from=builder /project_dir/ksml-runner/target/libs/ /opt/ksml/libs/
-COPY --chown=ksml:0 --from=builder /project_dir/graalpy-module-collection/target/modules/ /opt/ksml/modules/
 COPY --chown=ksml:0 --from=builder /project_dir/ksml-runner/target/ksml-runner*.jar /opt/ksml/ksml.jar
 
-ENTRYPOINT ["java", "--upgrade-module-path", "/opt/ksml/modules", "-jar", "/opt/ksml/ksml.jar"]
+ENTRYPOINT ["java", "-jar", "/opt/ksml/ksml.jar"]

README.md

@@ -27,7 +27,6 @@ The submodules are as follows:
 
 | Module                                                    | Description                                                                                                        |
 |-----------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------|
-| [`graalpy-module-collection`](graalpy-module-collection/) | contains dependencies to all GraalVM runtime modules for Python.                                                   |
 | [`ksml-data`](ksml-data/)                                 | contains core data type logic, including BINARY and JSON types, used in Kafka Streams topologies and by Producers. |
 | [`ksml-data-avro`](ksml-data-avro/)                       | extension to the data library for AVRO support.                                                                    |
 | [`ksml-data-csv`](ksml-data-csv/)                         | extension to the data library for CSV support.                                                                     |

ksml-runner/src/main/resources/prometheus/default_config.yaml

@@ -2,7 +2,7 @@ lowercaseOutputName: true
 lowercaseOutputLabelNames: true
 rules:
   - pattern: 'ksml<type=app-info, app-id=(.+), app-name=(.+), app-version=(.+), build-time=(.+)>Value:'
-    name: ksml_app_info
+    name: ksml_app
     labels:
       app_id: "$1"
       name: "$2"
@@ -31,4 +31,3 @@ rules:
     name: os_$1
     type: GAUGE
     attrNameSnakeCase: true
-  - pattern: .*