From dce4d1a3d7074c757ba9262e488d71a0b6381bbc Mon Sep 17 00:00:00 2001 From: Abhinav Sonkar Date: Fri, 28 Jun 2024 12:44:02 +0200 Subject: [PATCH] Added helm charts (#85) Helm Charts for easy Kubernetes deployments. With the Charts you can: deploys KSML as StatefulSet configures KSML Runner settings, like application server, prometheus export define KSML Definition files define Schema Files define namespaces and corresponding KSML Definition file logging configurations Collect metrics with servicemonitor Define an ingress for the application server Allows setting of Kubernetes options for the deployment Configure the VolumeClaimTemplate is made and configurable for state store SecurityContext ServiceAccount Resources Affinity NodeSelector topologySpreadConstraints tolerations The repo configured to build and release to multiple registries Images to Docker Hub docker.io/axual/ksml GitHub KSML registry ghcr.io/axual/ksml Axual OpenSource registry.axual.io/opensource/images/axual/ksml Charts to Axual OpenSource registry.axual.io/opensource/charts/ksml KSML Charts are configured to pull from Axual OpenSource, since that is where the user pulled the charts from as well --- .github/workflows/build-push-docker.yml | 26 +- .github/workflows/package-push-helm.yml | 28 +- .github/workflows/release-push-docker.yml | 26 +- .gitignore | 1 + Dockerfile | 15 +- README.md | 1 - graalpy-module-collection/NOTICE.txt | 23 -- graalpy-module-collection/pom.xml | 102 ----- .../src/main/java/module-info.java | 23 -- .../resources/prometheus/default_config.yaml | 3 +- packaging/helm-charts/ksml/.helmignore | 27 ++ packaging/helm-charts/ksml/Chart.yaml | 22 ++ packaging/helm-charts/ksml/LICENSE.txt | 201 ++++++++++ packaging/helm-charts/ksml/README.md | 370 ++++++++++++++++++ .../helm-charts/ksml/templates/NOTES.txt | 1 + .../helm-charts/ksml/templates/_helpers.tpl | 62 +++ .../ksml/templates/configmap-logging.yaml | 70 ++++ .../ksml/templates/configmap-runner.yaml | 60 +++ .../ksml/templates/headless-service.yaml | 16 + .../helm-charts/ksml/templates/ingress.yaml | 61 +++ .../helm-charts/ksml/templates/service.yaml | 19 + .../ksml/templates/serviceaccount.yaml | 13 + .../ksml/templates/servicemonitor.yaml | 20 + .../ksml/templates/statefulset.yaml | 104 +++++ packaging/helm-charts/ksml/values.schema.json | 330 ++++++++++++++++ packaging/helm-charts/ksml/values.yaml | 270 +++++++++++++ pom.xml | 1 - 27 files changed, 1722 insertions(+), 173 deletions(-) delete mode 100644 graalpy-module-collection/NOTICE.txt delete mode 100644 graalpy-module-collection/pom.xml delete mode 100644 graalpy-module-collection/src/main/java/module-info.java create mode 100644 packaging/helm-charts/ksml/.helmignore create mode 100644 packaging/helm-charts/ksml/Chart.yaml create mode 100644 packaging/helm-charts/ksml/LICENSE.txt create mode 100644 packaging/helm-charts/ksml/README.md create mode 100644 packaging/helm-charts/ksml/templates/NOTES.txt create mode 100644 packaging/helm-charts/ksml/templates/_helpers.tpl create mode 100644 packaging/helm-charts/ksml/templates/configmap-logging.yaml create mode 100644 packaging/helm-charts/ksml/templates/configmap-runner.yaml create mode 100644 packaging/helm-charts/ksml/templates/headless-service.yaml create mode 100644 packaging/helm-charts/ksml/templates/ingress.yaml create mode 100644 packaging/helm-charts/ksml/templates/service.yaml create mode 100644 packaging/helm-charts/ksml/templates/serviceaccount.yaml create mode 100644 packaging/helm-charts/ksml/templates/servicemonitor.yaml create mode 100644 packaging/helm-charts/ksml/templates/statefulset.yaml create mode 100644 packaging/helm-charts/ksml/values.schema.json create mode 100644 packaging/helm-charts/ksml/values.yaml diff --git a/.github/workflows/build-push-docker.yml b/.github/workflows/build-push-docker.yml index 11c89a8f..3b518041 100644 --- a/.github/workflows/build-push-docker.yml +++ b/.github/workflows/build-push-docker.yml @@ -13,26 +13,33 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: set up Docker buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: login to GHCR - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: login to Docker hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: login to Axual Open Source Registry + uses: docker/login-action@v3 + with: + registry: registry.axual.io + username: ${{ secrets.AXUAL_REGISTRY_USER }} + password: ${{ secrets.AXUAL_REGISTRY_TOKEN }} + - name: build and push KSML Runner - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: file: Dockerfile target: ksml @@ -41,3 +48,12 @@ jobs: tags: | ghcr.io/axual/ksml:snapshot axual/ksml:snapshot + registry.axual.io/opensource/images/axual/ksml:snapshot + + helm-chart-release: + needs: build-push-docker + uses: './.github/workflows/package-push-helm.yml' + secrets: inherit + with: + app-version: snapshot + version: 0.0.0-snapshot diff --git a/.github/workflows/package-push-helm.yml b/.github/workflows/package-push-helm.yml index ceede48d..2ef43744 100644 --- a/.github/workflows/package-push-helm.yml +++ b/.github/workflows/package-push-helm.yml @@ -1,12 +1,22 @@ -name: build and push Helm Charts +name: Package and Push Helm Chart on: workflow_dispatch: + inputs: + app-version: + required: true + type: string + description: "The app version that should be set in the chart" + version: + required: false + default: 0.0.0-snapshot + type: string + description: "The version of the chart to be released" workflow_call: inputs: - tag-name: + app-version: required: true type: string - target-version: + version: required: false default: 0.0.0-snapshot type: string @@ -21,23 +31,27 @@ jobs: run: | curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + - name: Checkout + uses: actions/checkout@v4 + - name: Package Helm shell: bash run: | - helm package --version '${{ inputs.target-version }}' --app-version '${{ inputs.tag-name }}' packaging/helm-charts/ksml + helm package --version '${{ inputs.version }}' --app-version '${{ inputs.app-version }}' ./packaging/helm-charts/ksml && ls ksml*.tgz - name: Login to registry shell: bash run: | - helm registry login '${{ env.AXUAL_REGISTRY }}' -u '${{ secrets.AXUAL_REGISTRY_USER }}' -p '${{ secrets.AXUAL_REGISTRY_TOKEN }}' + helm registry login registry.axual.io/opensource/charts -u '${{ secrets.AXUAL_REGISTRY_USER }}' -p '${{ secrets.AXUAL_REGISTRY_TOKEN }}' - name: Push to registry shell: bash run: | - helm push 'ksml-${{ inputs.target-version }}.tgz' + helm push 'ksml-${{ inputs.version }}.tgz' oci://registry.axual.io/opensource/charts - name: Log out of registry shell: bash if: ${{ always() }} + run: | - helm registry logout '${{ env.AXUAL_REGISTRY }}' & + helm registry logout registry.axual.io/opensource/charts & diff --git a/.github/workflows/release-push-docker.yml b/.github/workflows/release-push-docker.yml index 19c7f899..e9062232 100644 --- a/.github/workflows/release-push-docker.yml +++ b/.github/workflows/release-push-docker.yml @@ -1,10 +1,16 @@ name: build and push release Docker images on: + workflow_dispatch: + inputs: + tag: + required: true + type: string + description: "The image tag to create" release: types: [published] env: - RELEASE_TAG: ${{ github.event.release.tag_name }} + RELEASE_TAG: ${{ inputs.tag || github.event.release.tag_name }} jobs: release-push-docker: @@ -33,6 +39,13 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: login to Axual Open Source Registry + uses: docker/login-action@v2 + with: + registry: registry.axual.io + username: ${{ secrets.AXUAL_REGISTRY_USER }} + password: ${{ secrets.AXUAL_REGISTRY_TOKEN }} + - name: build and push KSML Runner uses: docker/build-push-action@v4 with: @@ -42,5 +55,16 @@ jobs: push: true tags: | axual/ksml:${{ github.event.release.tag_name }} + axual/ksml:latest ghcr.io/axual/ksml:${{ github.event.release.tag_name }} ghcr.io/axual/ksml:latest + registry.axual.io/opensource/images/axual/ksml:${{ github.event.release.tag_name }} + registry.axual.io/opensource/images/axual/ksml:latest + + helm-chart-release: + needs: release-push-docker + uses: './.github/workflows/package-push-helm.yml' + secrets: inherit + with: + app-version: ${{ inputs.tag || github.event.release.tag_name }} + version: ${{ inputs.tag || github.event.release.tag_name }} diff --git a/.gitignore b/.gitignore index 4637da44..63d8e962 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ examples/io.axual* examples/io.ksml* workspace/* +/packaging/helm-charts/**/*.tgz diff --git a/Dockerfile b/Dockerfile index 9dc85c1c..bd82b7d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ # Step 1: Create the common base image with the ksml user and group and the required packages -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9-1137 AS base +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.4-1134 AS base ENV LANG=en_US.UTF-8 # Environment variable for Connect Build and Runtime @@ -20,7 +20,8 @@ ENV PATH=/opt/graal/bin:$PATH \ KSML_INSTALL="/opt/ksml" RUN set -eux \ - && microdnf install -y procps curl tar unzip gzip zlib openssl-devel gcc gcc-c++ make patch glibc-langpack-en libxcrypt shadow-utils \ + && microdnf upgrade -y --nodocs \ + && microdnf install -y --nodocs procps tar unzip gzip zlib openssl-devel gcc gcc-c++ make patch glibc-langpack-en libxcrypt shadow-utils \ && groupadd --gid 1024 ksml \ && useradd -g ksml -u 1024 -d "$KSML_HOME" -ms /bin/sh -f -1 ksml \ && chown -R ksml:0 "$KSML_HOME" /opt \ @@ -50,7 +51,6 @@ RUN set -eux \ && mkdir -p /opt/maven \ && MVN_PKG=https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ && curl --fail --silent --location --retry 3 ${MVN_PKG} | gunzip | tar x -C /opt/maven --strip-components=1 \ -# && GRAALVM_PKG=https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-21.0.1/graalvm-community-jdk-21.0.1_linux-x64_bin.tar.gz && GRAALVM_PKG=https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-${GRAALVM_JDK_VERSION}/graalvm-community-jdk-${GRAALVM_JDK_VERSION}_linux-${JAVA_ARCH}_bin.tar.gz \ && mkdir -p /opt/graal \ && curl --fail --silent --location --retry 3 ${GRAALVM_PKG} | gunzip | tar x -C /opt/graal --strip-components=1 @@ -73,7 +73,7 @@ ENV JAVA_HOME=/opt/graalvm COPY --chown=ksml:0 --from=graal-builder /opt/graal/ /opt/graal/ WORKDIR /home/ksml -USER ksml +USER 1024 #There is no more GraalPy command here and no venv #RUN graalpy -m venv graalenv && \ # echo "source $HOME/graalenv/bin/activate" >> ~/.bashrc @@ -81,10 +81,9 @@ USER ksml # Step 5: Create the KSML Runner image FROM ksml-graal AS ksml -COPY --chown=ksml:0 --from=builder /project_dir/ksml-runner/NOTICE.txt /licences/THIRD-PARTY-LICENSES.txt -COPY --chown=ksml:0 --from=builder /project_dir/LICENSE.txt /licences/LICENSE.txt +COPY --chown=ksml:0 --from=builder /project_dir/ksml-runner/NOTICE.txt /licenses/THIRD-PARTY-LICENSES.txt +COPY --chown=ksml:0 --from=builder /project_dir/LICENSE.txt /licenses/LICENSE.txt COPY --chown=ksml:0 --from=builder /project_dir/ksml-runner/target/libs/ /opt/ksml/libs/ -COPY --chown=ksml:0 --from=builder /project_dir/graalpy-module-collection/target/modules/ /opt/ksml/modules/ COPY --chown=ksml:0 --from=builder /project_dir/ksml-runner/target/ksml-runner*.jar /opt/ksml/ksml.jar -ENTRYPOINT ["java", "--upgrade-module-path", "/opt/ksml/modules", "-jar", "/opt/ksml/ksml.jar"] +ENTRYPOINT ["java", "-jar", "/opt/ksml/ksml.jar"] diff --git a/README.md b/README.md index 77350ce8..d27c734e 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ The submodules are as follows: | Module | Description | |-----------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------| -| [`graalpy-module-collection`](graalpy-module-collection/) | contains dependencies to all GraalVM runtime modules for Python. | | [`ksml-data`](ksml-data/) | contains core data type logic, including BINARY and JSON types, used in Kafka Streams topologies and by Producers. | | [`ksml-data-avro`](ksml-data-avro/) | extension to the data library for AVRO support. | | [`ksml-data-csv`](ksml-data-csv/) | extension to the data library for CSV support. | diff --git a/graalpy-module-collection/NOTICE.txt b/graalpy-module-collection/NOTICE.txt deleted file mode 100644 index fd79689a..00000000 --- a/graalpy-module-collection/NOTICE.txt +++ /dev/null @@ -1,23 +0,0 @@ - -Lists of 21 third-party dependencies. - (Bouncy Castle Licence) Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (org.bouncycastle:bcpkix-jdk18on:1.78.1 - https://www.bouncycastle.org/java.html) - (Bouncy Castle Licence) Bouncy Castle Provider (org.bouncycastle:bcprov-jdk18on:1.78.1 - https://www.bouncycastle.org/java.html) - (Bouncy Castle Licence) Bouncy Castle ASN.1 Extension and Utility APIs (org.bouncycastle:bcutil-jdk18on:1.78.1 - https://www.bouncycastle.org/java.html) - (New BSD License (3-clause BSD license)) Sulong API (org.graalvm.llvm:llvm-api:23.1.2 - http://www.graalvm.org/) - (Universal Permissive License, Version 1.0) Polyglot (org.graalvm.polyglot:polyglot:23.1.2 - https://github.com/oracle/graal) - (MIT License) (Python Software Foundation License) (Universal Permissive License, Version 1.0) Graalpython (org.graalvm.python:python-language:23.1.2 - http://www.graalvm.org/) - (MIT License) (Python Software Foundation License) (Universal Permissive License, Version 1.0) Graalpython Resources (org.graalvm.python:python-resources:23.1.2 - http://www.graalvm.org/) - (Universal Permissive License, Version 1.0) Tregex (org.graalvm.regex:regex:23.1.2 - http://www.graalvm.org/) - (Universal Permissive License, Version 1.0) Collections (org.graalvm.sdk:collections:23.1.2 - https://github.com/oracle/graal) - (Universal Permissive License, Version 1.0) Jniutils (org.graalvm.sdk:jniutils:23.1.2 - https://github.com/oracle/graal) - (Universal Permissive License, Version 1.0) Nativeimage (org.graalvm.sdk:nativeimage:23.1.2 - https://github.com/oracle/graal) - (Universal Permissive License, Version 1.0) Word (org.graalvm.sdk:word:23.1.2 - https://github.com/oracle/graal) - (Unicode/ICU License) Truffle Icu4j (org.graalvm.shadowed:icu4j:23.1.2 - http://openjdk.java.net/projects/graal) - (Universal Permissive License, Version 1.0) Truffle Json (org.graalvm.shadowed:json:23.1.2 - http://openjdk.java.net/projects/graal) - (GNU General Public License, version 2, with the Classpath Exception) Truffle Profiler (org.graalvm.tools:profiler-tool:23.1.2 - http://openjdk.java.net/projects/graal) - (Universal Permissive License, Version 1.0) Truffle API (org.graalvm.truffle:truffle-api:23.1.2 - http://openjdk.java.net/projects/graal) - (Universal Permissive License, Version 1.0) Truffle Compiler (org.graalvm.truffle:truffle-compiler:23.1.2 - http://openjdk.java.net/projects/graal) - (Universal Permissive License, Version 1.0) Truffle Nfi (org.graalvm.truffle:truffle-nfi:23.1.2 - http://openjdk.java.net/projects/graal) - (Universal Permissive License, Version 1.0) Truffle Nfi Libffi (org.graalvm.truffle:truffle-nfi-libffi:23.1.2 - http://openjdk.java.net/projects/graal) - (Universal Permissive License, Version 1.0) Truffle Runtime (org.graalvm.truffle:truffle-runtime:23.1.2 - http://openjdk.java.net/projects/graal) - (Public Domain) XZ for Java (org.tukaani:xz:1.9 - https://tukaani.org/xz/java.html) diff --git a/graalpy-module-collection/pom.xml b/graalpy-module-collection/pom.xml deleted file mode 100644 index fafcf6f4..00000000 --- a/graalpy-module-collection/pom.xml +++ /dev/null @@ -1,102 +0,0 @@ - - - 4.0.0 - - io.axual.ksml - ksml-parent - 0.9.2-SNAPSHOT - - - graalpy-module-collection - - - 1.78.1 - 21 - 21 - UTF-8 - - - - - org.graalvm.polyglot - polyglot - provided - - - org.graalvm.truffle - truffle-api - provided - - - - org.bouncycastle - bcpkix-jdk18on - ${bouncycastle.override.version} - - - org.bouncycastle - bcutil-jdk18on - ${bouncycastle.override.version} - - - org.bouncycastle - bcprov-jdk18on - ${bouncycastle.override.version} - - - org.graalvm.python - python-language - provided - - - - org.bouncycastle - bcpkix-jdk18on - - - org.bouncycastle - bcutil-jdk18on - - - org.bouncycastle - bcprov-jdk18on - - - - - org.graalvm.python - python-resources - provided - - - org.graalvm.truffle - truffle-runtime - provided - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-dependencies - prepare-package - - copy-dependencies - - - - ${project.build.directory}/modules - - - - - - - - \ No newline at end of file diff --git a/graalpy-module-collection/src/main/java/module-info.java b/graalpy-module-collection/src/main/java/module-info.java deleted file mode 100644 index 34274007..00000000 --- a/graalpy-module-collection/src/main/java/module-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/*- - * ========================LICENSE_START================================= - * graalpy-module-collection - * %% - * Copyright (C) 2021 - 2024 Axual B.V. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================LICENSE_END================================== - */ -module graalpy.module.collection { - requires org.graalvm.truffle; - requires org.graalvm.polyglot; -} diff --git a/ksml-runner/src/main/resources/prometheus/default_config.yaml b/ksml-runner/src/main/resources/prometheus/default_config.yaml index 942c2ec1..e965e969 100644 --- a/ksml-runner/src/main/resources/prometheus/default_config.yaml +++ b/ksml-runner/src/main/resources/prometheus/default_config.yaml @@ -2,7 +2,7 @@ lowercaseOutputName: true lowercaseOutputLabelNames: true rules: - pattern: 'ksmlValue:' - name: ksml_app_info + name: ksml_app labels: app_id: "$1" name: "$2" @@ -31,4 +31,3 @@ rules: name: os_$1 type: GAUGE attrNameSnakeCase: true - - pattern: .* diff --git a/packaging/helm-charts/ksml/.helmignore b/packaging/helm-charts/ksml/.helmignore new file mode 100644 index 00000000..73543e59 --- /dev/null +++ b/packaging/helm-charts/ksml/.helmignore @@ -0,0 +1,27 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +*.tar +*.tgz +*.tbz + diff --git a/packaging/helm-charts/ksml/Chart.yaml b/packaging/helm-charts/ksml/Chart.yaml new file mode 100644 index 00000000..3caf0208 --- /dev/null +++ b/packaging/helm-charts/ksml/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +name: ksml +description: "KSML: Kafka Streaming for Low Code Environments" +type: application +version: 1.0.0-SNAPSHOT +appVersion: "1.0.0-snapshot" +icon: https://avatars.githubusercontent.com/u/26816390?s=48&v=4 +keywords: + - kafka + - streams + - streaming + - messaging + - event + - axual +home: https://ksml.io +sources: + - https://github.com/Axual/ksml +maintainers: + - name: jeroenvandisseldorp + - name: richard-axual + - name: xsreality + - name: tonvanbart diff --git a/packaging/helm-charts/ksml/LICENSE.txt b/packaging/helm-charts/ksml/LICENSE.txt new file mode 100644 index 00000000..97ff4280 --- /dev/null +++ b/packaging/helm-charts/ksml/LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021 Axual B.V. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packaging/helm-charts/ksml/README.md b/packaging/helm-charts/ksml/README.md new file mode 100644 index 00000000..548019b2 --- /dev/null +++ b/packaging/helm-charts/ksml/README.md @@ -0,0 +1,370 @@ +# ksml + +[KSML](https://ksml.io) - Kafka Streaming for Low Code Environments + +* [Introduction](#introduction) +* [Prerequisites](#prerequisites) +* [Installing the Chart](#installing-the-chart) +* [Uninstalling the Chart](#uninstalling-the-chart) +* [Configuration](#configuration) +* [Examples](#examples) + * [Plain Kafka SASL Connection](#plain-kafka-sasl-connection) + * [Axual Kafka SASL Connection](#axual-kafka-sasl-connection) + +## Introduction + +This chart deploys an easy way to define, run and deploy KSML applications on +a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes v1.21+ + +## Installing the Chart + +To install/upgrade the chart with the release name `ksml`: + +```bash +$ helm upgrade -i ksml oci://registry.axual.io/opensource/charts/ksml -n streaming --create-namespace --version=| 0.0.0-snapshot +``` + +The command deploys An easy way to define, run and deploy Kafka streaming applications on the +Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists +the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall the `ksml`: + +```bash +$ helm uninstall ksml -n streaming +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following table lists the configurable parameters of the `ksml` chart and their default values. + +| Parameter | Description | Default | +|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------| +| replicaCount | | 1 | +| image.repository | Registry to pull the image from and the name of the image. | docker.io/axual/ksml | +| image.pullPolicy | One of `Always`, `IfNotPresent`, or `Never`. | IfNotPresent | +| image.tag | Override the image tag whose default is the chart `appVersion`. | "" | +| nameOverride | Override the app name generated by the chart. | "" | +| imagePullSecrets | Override the list of ImagePullSecrets provided | [] | +| fullnameOverride | Override the fully qualified app name generated by the chart. | "" | +| applicationServer.enabled | Start the application server for the component | false | +| applicationServer.port | | "8080" | +| ksmlDefinitions | Use this dictionary to specify one or more KSML definitions and filenames | | +| ksmlRunnerConfig.definitionDirectory | Directory containing KSML Definition files. This should only be changed if an external volume mount is used. | '/ksml' | +| ksmlRunnerConfig.definitions | A dictionary specifying a namespace, and the KSML definition file that should be executed in that namespace | '/ksml' | +| ksmlRunnerConfig.schemaDirectory | Directory containing the Schema files. This should only be changed if an external volume mount is used. | '/ksml' | +| ksmlRunnerConfig.errorHandling.consume.log | log a message when consume errors occur | true | +| ksmlRunnerConfig.errorHandling.consume.logPayload | Add the record payload to the log message | false | +| ksmlRunnerConfig.errorHandling.consume.loggerName | Specify the name of the logger to use | ConsumeError | +| ksmlRunnerConfig.errorHandling.consume.stopOnError | Should the KSML application stop when a consume error occurs | true | +| ksmlRunnerConfig.errorHandling.produce.log | log a message when produce errors occur | true | +| ksmlRunnerConfig.errorHandling.produce.logPayload | Add the record payload to the log message | false | +| ksmlRunnerConfig.errorHandling.produce.loggerName | Specify the name of the logger to use | ProduceErrpr | +| ksmlRunnerConfig.errorHandling.produce.stopOnError | Should the KSML application stop when a consume error occurs | true | +| ksmlRunnerConfig.errorHandling.process.log | log a message when processing errors occur | true | +| ksmlRunnerConfig.errorHandling.process.logPayload | Add the record payload to the log message | false | +| ksmlRunnerConfig.errorHandling.process.loggerName | Specify the name of the logger to use | ProcessError | +| ksmlRunnerConfig.errorHandling.process.stopOnError | Should the KSML application stop when a consume error occurs | true | +| ksmlRunnerConfig.producersEnabled | Data generation can be disabled for deployments to prevent accidental test data in non test environments | true | +| ksmlRunnerConfig.pipelinesEnabled | Pipelines can be disabled allow for data generation only deployments. | true | +| ksmlRunnerConfig.kafka | Dictionary of Kafka connection properties and application id | | +| ksmlRunnerConfig.kafka.application.id | The application id for the app. Also used in resources and metrics | | +| ksmlRunnerConfig.kafka.bootstrap.servers | The hostname and port number for connecting to the Kafka cluster | | +| schemaDefinitions | Use this dictionary to specify one or more schema definitions and filenames | | +| serviceAccount.create | Specifies whether a service account should be created | true | +| serviceAccount.automount | Automatically mount a ServiceAccount's API credentials? | true | +| serviceAccount.name | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | "" | +| service.type | | ClusterIP | +| service.port | | 80 | +| ingress.enabled | Enable creation of the Ingress resource to expose this service. | false | +| ingress.className | The name of the IngressClass cluster resource. The associated IngressClass defines which controller will implement the resource. | "" | +| ingress.hosts | The Ingress Host definitions | | +| ingress.tls | The Ingress TLS configuration | | +| applicationServer.enabled | Start an application server for the component | false | +| applicationServer.port | Port on which Application server should be initialized | "8080" | +| logging.configFile | Location of the logging configuration file | '/ksml-logging/logback.xml' | +| logging.jsonEnabled | Enable JSON logging | false | +| logging.patterns.stdout | Pattern for the log message to standard out. See the [Logback](https://logback.qos.ch/manual/layouts.html#conversionWord) documentation for the syntax | '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} [%t] %-5level %logger{36} - %msg%n' | +| logging.patterns.stderr | Pattern for the log message to standard error. See the [Logback](https://logback.qos.ch/manual/layouts.html#conversionWord) documentation for the syntax | '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} [%t] %-5level %logger{36} - %msg%n' | +| logging.rootLogLevel | The default log level to use for all loggers | INFO | +| logging.loggers | A map of the logger name and corresponding loglevel | | +| store.spec | Specify the VolumeClaimTemplate used by the StatefulSet. Default it uses access mode "ReadWriteOnce" and requests 1Gi | | +| prometheus.enabled | Enable Prometheus Metrics export | false | +| prometheus.port | Port on which Prometheus metrics endpoint should be initialized | "9999" | +| prometheus.config | The configuration for the [Prometheus JMX Exporter](https://github.com/prometheus/jmx_exporter/blob/1.0.1/README.md). The 1.x.x config is used | | +| serviceMonitor.enabled | Enables creation of Prometheus Operator [ServiceMonitor](https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.ServiceMonitor). Ignored if API `monitoring.coreos.com/v1` is not available. | true | +| serviceMonitor.interval | Interval at which metrics should be scraped. | 30s | +| serviceMonitor.scrapeTimeout | Timeout after which the scrape is ended. | 10s | +| serviceMonitor.labels | A dictionary with additional labels for the service monitor | | +| affinity | The pod's scheduling constraints. See the Kubernetes documentation on [Affinity and Anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) | | +| topologySpreadConstraints | Describes how a group of pods ought to spread across topology domains. See the Kubernetes documentation on [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/). | [] | +| tolerations | The tolerations on this pod. See the Kubernetes documentation on [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/). | [] | +| nodeSelector | Selector which must match a node's labels for the pod to be scheduled on that node. | | +| volumeMounts | Additional volumeMounts on the output Deployment definition | [] | +| volumes | Additional volumes on the output Deployment definition | [] | +| resources | Resource requests and limits for KSML. | | +| securityContext | Defines the security options the container should be run with. If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext | | +| podSecurityContext | Pod-level security attributes and common container settings | | +| podLabels | Extra labels to add to the Pods | | +| podAnnotations | Extra annotations to add to the Pods | | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm upgrade -i`. For +example: + +```bash +$ helm upgrade -i ksml oci://registry.axual.io/opensource/charts/ksml -n streaming --create-namespace --version=| 0.0.0-snapshot --set replicaCount=1 +``` + +Alternatively, a YAML file that specifies the values for the parameters can be provided while +installing the chart. For example: + +```bash +$ helm upgrade -i ksml oci://registry.axual.io/opensource/charts/ksml -n streaming --create-namespace --version=| 0.0.0-snapshot --values values.yaml +``` +## Examples + +### Plain Kafka SASL Connection +
+ Example configuration plain Kafka SASL + +```YAML +image: + tag: snapshot + +replicaCount: 3 + +resources: + requests: + memory: "128Mi" + cpu: "250m" + limits: + memory: "768Mi" + cpu: "500m" + +ksmlRunnerConfig: + definitions: +# generate: generator.yaml + inspect: peek.yaml + kafka: + application.id: example.datagen + + bootstrap.servers: 'testing-sasl-broker:9095' + security.protocol: SASL_SSL + sasl.mechanism: SCRAM-SHA-512 + sasl.jaas.config: 'org.apache.kafka.common.security.scram.ScramLoginModule required username=example_user password=12345678 ;' + +ksmlDefinitions: + generator.yaml: | + # This example shows how to generate data and have it sent to a target topic in a given format. + + functions: + generate_sensordata_message: + type: generator + globalCode: | + import time + import random + sensorCounter = 0 + code: | + global sensorCounter + + key = "sensor"+str(sensorCounter) # Set the key to return ("sensor0" to "sensor9") + sensorCounter = (sensorCounter+1) % 10 # Increase the counter for next iteration + + # Generate some random sensor measurement data + types = { 0: { "type": "AREA", "unit": random.choice([ "m2", "ft2" ]), "value": str(random.randrange(1000)) }, + 1: { "type": "HUMIDITY", "unit": random.choice([ "g/m3", "%" ]), "value": str(random.randrange(100)) }, + 2: { "type": "LENGTH", "unit": random.choice([ "m", "ft" ]), "value": str(random.randrange(1000)) }, + 3: { "type": "STATE", "unit": "state", "value": random.choice([ "off", "on" ]) }, + 4: { "type": "TEMPERATURE", "unit": random.choice([ "C", "F" ]), "value": str(random.randrange(-100, 100)) } + } + + # Build the result value using any of the above measurement types + value = { "name": key, "timestamp": str(round(time.time()*1000)), **random.choice(types) } + value["color"] = random.choice([ "black", "blue", "red", "yellow", "white" ]) + value["owner"] = random.choice([ "Alice", "Bob", "Charlie", "Dave", "Evan" ]) + value["city"] = random.choice([ "Amsterdam", "Xanten", "Utrecht", "Alkmaar", "Leiden" ]) + + if random.randrange(10) == 0: + value = None + expression: (key, value) # Return a message tuple with the key and value + resultType: (string, json) # Indicate the type of key and value + + producers: + sensordata_json_producer: + generator: generate_sensordata_message + interval: 444 + to: + topic: ksml_sensordata_json + keyType: string + valueType: json + +applicationServer: + enabled: true + port: "8080" + +prometheus: + enabled: true + port: "9993" + config: + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + - pattern: 'ksmlValue:' + name: ksml_app_info + labels: + app_id: "$1" + name: "$2" + version: "$3" + build_time: "$4" + type: GAUGE + value: 1 + cached: true + - pattern: .* + +logging: + configFile: '/ksml-logging/logback.xml' + jsonEnabled: false + patterns: + stdout: '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} %-5level %logger{36} - %msg%n' + stderr: '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} %-5level %logger{36} - %msg%n' + loggers: + org.apache.kafka: INFO + org.apache.kafka.streams.StreamsConfig: INFO + org.apache.kafka.clients.consumer.ConsumerConfig: INFO + +``` +
+ +### Axual Kafka SASL Connection +
+ Example configuration to connect to an Axual Kafka cluster +See the ksmlRunnerConfig.kafka section for the special additions to automatically resolve resource names to the Axual namespaces + +```YAML +image: + tag: snapshot + +replicaCount: 3 + +resources: + requests: + memory: "128Mi" + cpu: "250m" + limits: + memory: "768Mi" + cpu: "500m" + +ksmlRunnerConfig: + definitions: +# generate: generator.yaml + inspect: peek.yaml + kafka: + application.id: example.datagen + + bootstrap.servers: 'testing-sasl-broker.axual.cloud:9095' + security.protocol: SASL_SSL + sasl.mechanism: SCRAM-SHA-512 + sasl.jaas.config: 'org.apache.kafka.common.security.scram.ScramLoginModule required username=example_user password=12345678 ;' + + # Use these configuration properties when connecting to a cluster using the Axual naming patterns. + # These patterns are resolved into the actual name used on Kafka using the values in this configuration map + # and the topic names specified in the definition YAML files + + tenant: 'custom' + instance: 'dta' + environment: 'test' + group.id.pattern: '{tenant}-{instance}-{environment}-{group.id}' + topic.pattern: '{tenant}-{instance}-{environment}-{topic}' + transactional.id.pattern: '{tenant}-{instance}-{environment}-{transactional.id}' + +ksmlDefinitions: + generator.yaml: | + # This example shows how to generate data and have it sent to a target topic in a given format. + + functions: + generate_sensordata_message: + type: generator + globalCode: | + import time + import random + sensorCounter = 0 + code: | + global sensorCounter + + key = "sensor"+str(sensorCounter) # Set the key to return ("sensor0" to "sensor9") + sensorCounter = (sensorCounter+1) % 10 # Increase the counter for next iteration + + # Generate some random sensor measurement data + types = { 0: { "type": "AREA", "unit": random.choice([ "m2", "ft2" ]), "value": str(random.randrange(1000)) }, + 1: { "type": "HUMIDITY", "unit": random.choice([ "g/m3", "%" ]), "value": str(random.randrange(100)) }, + 2: { "type": "LENGTH", "unit": random.choice([ "m", "ft" ]), "value": str(random.randrange(1000)) }, + 3: { "type": "STATE", "unit": "state", "value": random.choice([ "off", "on" ]) }, + 4: { "type": "TEMPERATURE", "unit": random.choice([ "C", "F" ]), "value": str(random.randrange(-100, 100)) } + } + + # Build the result value using any of the above measurement types + value = { "name": key, "timestamp": str(round(time.time()*1000)), **random.choice(types) } + value["color"] = random.choice([ "black", "blue", "red", "yellow", "white" ]) + value["owner"] = random.choice([ "Alice", "Bob", "Charlie", "Dave", "Evan" ]) + value["city"] = random.choice([ "Amsterdam", "Xanten", "Utrecht", "Alkmaar", "Leiden" ]) + + if random.randrange(10) == 0: + value = None + expression: (key, value) # Return a message tuple with the key and value + resultType: (string, json) # Indicate the type of key and value + + producers: + sensordata_json_producer: + generator: generate_sensordata_message + interval: 444 + to: + topic: ksml_sensordata_json + keyType: string + valueType: json + +applicationServer: + enabled: true + port: "8080" + +prometheus: + enabled: true + port: "9993" + config: + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + - pattern: 'ksmlValue:' + name: ksml_app_info + labels: + app_id: "$1" + name: "$2" + version: "$3" + build_time: "$4" + type: GAUGE + value: 1 + cached: true + - pattern: .* + +logging: + configFile: '/ksml-logging/logback.xml' + jsonEnabled: false + patterns: + stdout: '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} %-5level %logger{36} - %msg%n' + stderr: '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} %-5level %logger{36} - %msg%n' + loggers: + org.apache.kafka: INFO + org.apache.kafka.streams.StreamsConfig: INFO + org.apache.kafka.clients.consumer.ConsumerConfig: INFO +``` +
+ diff --git a/packaging/helm-charts/ksml/templates/NOTES.txt b/packaging/helm-charts/ksml/templates/NOTES.txt new file mode 100644 index 00000000..9cf433c5 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/NOTES.txt @@ -0,0 +1 @@ +Thank you for installing {{ .Chart.Name }}-{{ .Chart.Version }} \ No newline at end of file diff --git a/packaging/helm-charts/ksml/templates/_helpers.tpl b/packaging/helm-charts/ksml/templates/_helpers.tpl new file mode 100644 index 00000000..5a0bf225 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ksml.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ksml.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ksml.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ksml.labels" -}} +helm.sh/chart: {{ include "ksml.chart" . }} +{{ include "ksml.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ksml.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ksml.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ksml.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ksml.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/packaging/helm-charts/ksml/templates/configmap-logging.yaml b/packaging/helm-charts/ksml/templates/configmap-logging.yaml new file mode 100644 index 00000000..f2d7b13f --- /dev/null +++ b/packaging/helm-charts/ksml/templates/configmap-logging.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "ksml.fullname" . }}-logging + labels: + {{- include "ksml.labels" . | nindent 4 }} +data: + logback.xml: | + + + + + {{- if .Values.logging.jsonEnabled }} + + {{- else }} + + {{ .Values.logging.patterns.stderr }} + + {{- end }} + + System.err + + WARN + + + + {{- if .Values.logging.jsonEnabled }} + + {{- else }} + + {{ .Values.logging.patterns.stdout }} + + {{- end }} + + System.out + + TRACE + ACCEPT + + + DEBUG + ACCEPT + + + INFO + ACCEPT + + + WARN + DENY + + + ERROR + DENY + + + + + + + + + + + {{- range $key, $value := (mergeOverwrite .Values.logging.defaultLoggers .Values.logging.loggers ) }} + + {{- end }} + + + diff --git a/packaging/helm-charts/ksml/templates/configmap-runner.yaml b/packaging/helm-charts/ksml/templates/configmap-runner.yaml new file mode 100644 index 00000000..dd84cc65 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/configmap-runner.yaml @@ -0,0 +1,60 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "ksml.fullname" . }}-runner + labels: + {{- include "ksml.labels" . | nindent 4 }} +data: + ksml-runner.yaml: | + ksml: + configDirectory: {{ .Values.ksmlRunnerConfig.definitionDirectory | quote }} + schemaDirectory: {{ .Values.ksmlRunnerConfig.schemaDirectory | quote }} + storageDirectory: "/ksml-store" + enableProducers: {{ .Values.ksmlRunnerConfig.producersEnabled }} + enablePipelines: {{ .Values.ksmlRunnerConfig.pipelinesEnabled }} + errorHandling: + consume: + log: {{ .Values.ksmlRunnerConfig.errorHandling.consume.log }} + logPayload: {{ .Values.ksmlRunnerConfig.errorHandling.consume.logPayload }} + loggerName: {{ .Values.ksmlRunnerConfig.errorHandling.consume.loggerName }} + handler: {{ ternary "stopOnFail" "continueOnFail" .Values.ksmlRunnerConfig.errorHandling.consume.stopOnError }} + produce: + log: {{ .Values.ksmlRunnerConfig.errorHandling.produce.log }} + logPayload: {{ .Values.ksmlRunnerConfig.errorHandling.produce.logPayload }} + loggerName: {{ .Values.ksmlRunnerConfig.errorHandling.produce.loggerName }} + handler: {{ ternary "stopOnFail" "continueOnFail" .Values.ksmlRunnerConfig.errorHandling.produce.stopOnError }} + process: + log: {{ .Values.ksmlRunnerConfig.errorHandling.process.log }} + logPayload: {{ .Values.ksmlRunnerConfig.errorHandling.process.logPayload }} + loggerName: {{ .Values.ksmlRunnerConfig.errorHandling.process.loggerName }} + handler: {{ ternary "stopOnFail" "continueOnFail" .Values.ksmlRunnerConfig.errorHandling.process.stopOnError }} + applicationServer: + enabled: {{ .Values.applicationServer.enabled }} + port: {{ .Values.applicationServer.port }} + prometheus: + enabled: {{ .Values.prometheus.enabled }} + port: {{ .Values.prometheus.port }} + configFile: "/ksml/ksml-metrics-config.yml" + definitions: +{{- with .Values.ksmlRunnerConfig.definitions }} + {{- toYaml . | nindent 8 }} +{{- end }} + kafka: + config.providers: env + config.providers.env.class: 'org.apache.kafka.common.config.provider.EnvVarConfigProvider' + group.instance.id: '${env:KSML_GROUP_INSTANCE_ID}' +{{- with .Values.ksmlRunnerConfig.kafka }} + {{- toYaml . | nindent 6 }} +{{- end }} + +{{- with .Values.ksmlDefinitions }} + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.schemaDefinitions }} + {{- toYaml . | nindent 2 }} +{{- end }} + + ksml-metrics-config.yml: | + {{- with .Values.prometheus.config }} + {{- toYaml . | nindent 4 }} + {{- end }} diff --git a/packaging/helm-charts/ksml/templates/headless-service.yaml b/packaging/helm-charts/ksml/templates/headless-service.yaml new file mode 100644 index 00000000..fae0fdbd --- /dev/null +++ b/packaging/helm-charts/ksml/templates/headless-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ksml.fullname" . }}-headless + labels: + {{- include "ksml.labels" . | nindent 4 }} +spec: + type: ClusterIP + publishNotReadyAddresses: true + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "ksml.selectorLabels" . | nindent 4 }} diff --git a/packaging/helm-charts/ksml/templates/ingress.yaml b/packaging/helm-charts/ksml/templates/ingress.yaml new file mode 100644 index 00000000..3b1bb237 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "ksml.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "ksml.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/packaging/helm-charts/ksml/templates/service.yaml b/packaging/helm-charts/ksml/templates/service.yaml new file mode 100644 index 00000000..c0d20f54 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ksml.fullname" . }} + labels: + {{- include "ksml.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + - port: {{ .Values.prometheus.port }} + targetPort: metrics + protocol: TCP + name: metrics + selector: + {{- include "ksml.selectorLabels" . | nindent 4 }} diff --git a/packaging/helm-charts/ksml/templates/serviceaccount.yaml b/packaging/helm-charts/ksml/templates/serviceaccount.yaml new file mode 100644 index 00000000..a8049936 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ksml.serviceAccountName" . }} + labels: + {{- include "ksml.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/packaging/helm-charts/ksml/templates/servicemonitor.yaml b/packaging/helm-charts/ksml/templates/servicemonitor.yaml new file mode 100644 index 00000000..7baefd10 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/servicemonitor.yaml @@ -0,0 +1,20 @@ +{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "ksml.fullname" . }} + labels: + {{- include "ksml.labels" . | nindent 4 }} + {{- if (.Values.serviceMonitor.labels) }} + {{- toYaml .Values.serviceMonitor.labels | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "ksml.selectorLabels" . | nindent 6 }} + endpoints: + - port: metrics + path: /metrics + interval: {{ .Values.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} +{{- end }} diff --git a/packaging/helm-charts/ksml/templates/statefulset.yaml b/packaging/helm-charts/ksml/templates/statefulset.yaml new file mode 100644 index 00000000..6b1a3394 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/statefulset.yaml @@ -0,0 +1,104 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "ksml.fullname" . }} + labels: + {{- include "ksml.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "ksml.selectorLabels" . | nindent 6 }} + serviceName: {{ include "ksml.fullname" . }}-headless + podManagementPolicy: Parallel + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap-runner.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "ksml.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "ksml.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + workingDir: /ksml + ports: + - name: http + containerPort: {{ .Values.applicationServer.port }} + protocol: TCP + - name: metrics + containerPort: {{ .Values.prometheus.port }} + protocol: TCP + env: + - name: "KSML_K8S_POD_NAME" + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KSML_GROUP_INSTANCE_ID + value: "$(KSML_K8S_POD_NAME)" + - name: LOGBACK_CONFIGURATION_FILE + value: {{ .Values.logging.configFile | quote }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: ksml-config + mountPath: /ksml + - name: ksml-logging + mountPath: /ksml-logging + - name: ksml-store + mountPath: /ksml-store + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: ksml-config + configMap: + name: {{ include "ksml.fullname" . }}-runner + - name: ksml-logging + configMap: + name: {{ include "ksml.fullname" . }}-logging + {{- with .Values.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologySpreadConstraints}} + topologySpreadConstraints: + {{- range . }} + - labelSelector: + matchLabels: + {{- include "ksml.selectorLabels" $ | nindent 12 }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: ksml-store + spec: + {{- toYaml .Values.store.spec | nindent 8 }} diff --git a/packaging/helm-charts/ksml/values.schema.json b/packaging/helm-charts/ksml/values.schema.json new file mode 100644 index 00000000..fc30393d --- /dev/null +++ b/packaging/helm-charts/ksml/values.schema.json @@ -0,0 +1,330 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "affinity": { + "type": "object" + }, + "applicationServer": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "port": { + "type": "string" + } + } + }, + "fullnameOverride": { + "type": "string" + }, + "image": { + "type": "object", + "properties": { + "pullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "imagePullSecrets": { + "type": "array" + }, + "ingress": { + "type": "object", + "properties": { + "annotations": { + "type": "object" + }, + "className": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "hosts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "host": { + "type": "string" + }, + "paths": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "pathType": { + "type": "string" + } + } + } + } + } + } + }, + "tls": { + "type": "array" + } + } + }, + "ksmlDefinitions": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "ksmlRunnerConfig": { + "type": "object", + "properties": { + "definitionDirectory": { + "type": "string" + }, + "schemaDirectory": { + "type": "string" + }, + "producersEnabled": { + "type": "boolean" + }, + "pipelinesEnabled": { + "type": "boolean" + }, + "errorHandling": { + "type": "object", + "properties": { + "consume" : {"$ref": "#/definitions/ErrorHandling"}, + "produce" : {"$ref": "#/definitions/ErrorHandling"}, + "process" : {"$ref": "#/definitions/ErrorHandling"} + } + }, + "definitions": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + }, + "kafka": { + "type": "object", + "properties": { + "application.id": { + "type": "string" + }, + "bootstrap.servers": { + "type": "string" + } + }, + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + }, + "required": [ + "application.id", + "bootstrap.servers" + ] + } + }, + "required": [ + "definitionDirectory", + "schemaDirectory", + "definitions", + "kafka" + ] + }, + "logging": { + "type": "object", + "properties": { + "configFile": { + "type": "string" + }, + "jsonEnabled": { + "type": "boolean" + }, + "patters": { + "type": "object", + "properties": { + "stdout": { + "type": "string" + }, + "stderr": { + "type": "string" + } + } + }, + "rootLogLevel": { + "$ref": "#/definitions/LogLevel" + }, + "loggers": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/LogLevel" + } + }, + "defaultLoggers": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/LogLevel" + } + } + } + }, + "nameOverride": { + "type": "string" + }, + "nodeSelector": { + "type": "object" + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "podSecurityContext": { + "type": "object" + }, + "prometheus": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "port": { + "type": "string" + }, + "config": { + "type": "object" + } + } + }, + "replicaCount": { + "type": "integer", + "minimum": 0 + }, + "resources": { + "type": "object" + }, + "schemaDefinitions": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "securityContext": { + "type": "object" + }, + "service": { + "type": "object", + "properties": { + "port": { + "type": "integer" + }, + "type": { + "type": "string" + } + } + }, + "serviceAccount": { + "type": "object", + "properties": { + "annotations": { + "type": "object" + }, + "automount": { + "type": "boolean" + }, + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + } + }, + "store": { + "type": "object" + }, + "serviceMonitor": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "interval": { + "type": "string" + }, + "labels": { + "type": "object" + }, + "scrapeTimeout": { + "type": "string" + } + } + }, + "tolerations": { + "type": "array" + }, + "topologySpreadConstraints": { + "type": "array" + }, + "volumeMounts": { + "type": "array" + }, + "volumes": { + "type": "array" + } + }, + "definitions": { + "LogLevel": { + "enum": [ + "TRACE", + "DEBUG", + "INFO", + "WARN", + "ERROR", + "trace", + "debug", + "info", + "warn", + "error" + ] + }, + "ErrorHandling": { + "type": "object", + "properties": { + "log": { + "type": "boolean" + }, + "logPayload": { + "type": "boolean" + }, + "stopOnError": { + "type": "boolean" + }, + "logger": { + "type": "string", + "minLength": 1 + } + } + } + } +} diff --git a/packaging/helm-charts/ksml/values.yaml b/packaging/helm-charts/ksml/values.yaml new file mode 100644 index 00000000..0abba731 --- /dev/null +++ b/packaging/helm-charts/ksml/values.yaml @@ -0,0 +1,270 @@ +# Default values for ksml. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + # -- Registry to pull the image from and the name of the image. + repository: registry.axual.io/opensource/images/axual/ksml + # -- One of `Always`, `IfNotPresent`, or `Never`. + pullPolicy: IfNotPresent + # -- Override the image tag whose default is the chart `appVersion`. + tag: "" + +# -- Override the list of ImagePullSecrets provided. +imagePullSecrets: [] +# -- Override the app name generated by the chart. +nameOverride: "" +# -- Override the fully qualified app name generated by the chart. +fullnameOverride: "" + +# -- KSML runner configuration. See https://axual.github.io/ksml/runners.html +ksmlRunnerConfig: + # -- Directory containing KSML Definition files. This should only be changed if an external volume mount is used. + definitionDirectory: '/ksml' + # -- Directory containing the Schema files. This should only be changed if an external volume mount is used. + schemaDirectory: '/ksml' + + # -- KSML Error handling settings. Error Handling can be defined for consume, produce and processing errors. + errorHandling: + # -- Error handling settings for consume related errors + consume: + # -- log a message when consume errors occur + log: true + # -- Add the record payload to the log message + logPayload: false + # -- Specify the name of the logger to use + loggerName: ConsumeError + # -- Should the KSML application stop when a consume error occurs + stopOnError: true + # -- Error handling settings for produce related errors + produce: + # -- log a message when produce errors occur + log: true + # -- Add the record payload to the log message + logPayload: false + # -- Specify the name of the logger to use + loggerName: ProduceErrpr + # -- Should the KSML application stop when a consume error occurs + stopOnError: true + # -- Error handling settings for processing related errors + process: + # -- log a message when processing errors occur + log: true + # -- Add the record payload to the log message + logPayload: false + # -- Specify the name of the logger to use + loggerName: ProcessError + # -- Should the KSML application stop when a consume error occurs + stopOnError: true + + # -- Data generation can be disabled for deployments to prevent accidental test data in non test environments + producersEnabled: true + # -- Pipelines can be disabled allow for data generation only deployments. + pipelinesEnabled: true + + # -- Map of the namespace and the KSML definition file executed in that namespace. + definitions: {} + # -- Map of Kafka connection properties and application id + kafka: + # kafka: + # application.id: my-streaming-app + # bootstrap.servers: "localhost:9092" + +# KSML Pipeline or Producer definition files. See https://axual.github.io/ksml/introduction.html +ksmlDefinitions: {} + # my-definition.yaml: | + # ... + +# Schema definitions files. See https://github.com/Axual/ksml/tree/main/examples +schemaDefinitions: {} + # my-schema.avsc: | + # ... + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# -- Extra annotations to add to the Pods. +podAnnotations: {} + +# -- Extra labels to add to the Pods. +podLabels: {} + +# -- Pod-level security attributes and common container settings. +podSecurityContext: {} + # fsGroup: 2000 + +# -- Defines the security options the container should be run with. +# If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. +# @default -- See `values.yaml` file. +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + # -- Enable creation of the Ingress resource to expose this service. + enabled: false + # -- The name of the IngressClass cluster resource. + # The associated IngressClass defines which controller will implement the resource. + className: "" + # -- Annotations to add to the Ingress resource. + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + # -- TLS configuration for this Ingress. + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: "1000m" + # memory: "768Mi" + # requests: + # cpu: "500m" + # memory: "128Mi" + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +# -- Selector which must match a node's labels for the pod to be scheduled on that node. +nodeSelector: {} + +# -- The tolerations on this pod. See the Kubernetes documentation on [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/). +tolerations: [] + +# -- The pod's scheduling constraints. See the Kubernetes documentation on [Affinity and Anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). +affinity: {} + +# -- Describes how a group of pods ought to spread across topology domains. See the Kubernetes documentation on [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/). +topologySpreadConstraints: [] + +# -- KSML application server port defined in the statefulset. +applicationServer: + # -- Start an application server for the component + enabled: false + # -- Port on which Application server should be initialized + port: "8080" + +# -- Logging configuration settings for KSML. +logging: + # -- Location of the logging configuration file + configFile: '/ksml-logging/logback.xml' + # -- Enable JSON logging + jsonEnabled: false + # -- The logback log pattern to use + patterns: + stdout: '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} [%t] %-5level %logger{36} - %msg%n' + stderr: '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} [%t] %-5level %logger{36} - %msg%n' + # -- The default log level to use for all loggers + rootLogLevel: INFO + # -- Specify log levels for specific loggers. + loggers: {} + # -- Specify default loggers set for all deployments to minimize excessive logging + defaultLoggers: + io.axual.ksml: INFO + org.apache.kafka: WARN + org.apache.kafka.clients.consumer.ConsumerConfig: WARN + org.apache.kafka.clients.producer.ProducerConfig: WARN + org.apache.kafka.clients.admin.AdminClientConfig: WARN + org.apache.kafka.streams.StreamsConfig: WARN + io.confluent: WARN + +# -- Configure the storage specification for the volumeClaimTemplates +store: + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + +# -- KSML prometheus server port defined in the statefulset. Note that this must match the port defined in ksmlRunnerConfig. +prometheus: + enabled: false + # -- Port on which Prometheus metrics endpoint should be initialized + port: "9999" + # -- Contents of the JMX exporter config file + config: + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + - pattern: 'ksmlValue:' + name: ksml_app + labels: + app_id: "$1" + name: "$2" + version: "$3" + build_time: "$4" + type: GAUGE + value: 1 + cached: true + - pattern: 'ksml' + type: COUNTER + - pattern: 'ksml' + type: GAUGE + - pattern: 'kafka.streams' + type: GAUGE + - pattern: 'kafka.producer' + type: GAUGE + - pattern: 'kafka.consumer' + type: GAUGE + - pattern: 'kafka.admin.client' + type: GAUGE + - pattern: 'java.lang<>(committed_virtual_memory|free_physical_memory|free_swap_space|total_physical_memory|total_swap_space)_size:' + name: os_$1_bytes + type: GAUGE + attrNameSnakeCase: true + - pattern: 'java.lang<>((?!process_cpu_time)\w+):' + name: os_$1 + type: GAUGE + attrNameSnakeCase: true + + + +serviceMonitor: + # -- Enables creation of Prometheus Operator [ServiceMonitor](https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.ServiceMonitor). + # Ignored if API `monitoring.coreos.com/v1` is not available. + enabled: true + # -- Interval at which metrics should be scraped. + interval: 30s + # -- Timeout after which the scrape is ended. + scrapeTimeout: 10s + # -- Additional labels for the ServiceMonitor + labels: {} diff --git a/pom.xml b/pom.xml index 0508bc0b..16e98925 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,6 @@ ksml-kafka-clients ksml-runner ksml-reporting - graalpy-module-collection