diff --git a/.github/workflows/build-push-docker.yml b/.github/workflows/build-push-docker.yml index 11c89a8f..3b518041 100644 --- a/.github/workflows/build-push-docker.yml +++ b/.github/workflows/build-push-docker.yml @@ -13,26 +13,33 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: set up Docker buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: login to GHCR - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: login to Docker hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: login to Axual Open Source Registry + uses: docker/login-action@v3 + with: + registry: registry.axual.io + username: ${{ secrets.AXUAL_REGISTRY_USER }} + password: ${{ secrets.AXUAL_REGISTRY_TOKEN }} + - name: build and push KSML Runner - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: file: Dockerfile target: ksml @@ -41,3 +48,12 @@ jobs: tags: | ghcr.io/axual/ksml:snapshot axual/ksml:snapshot + registry.axual.io/opensource/images/axual/ksml:snapshot + + helm-chart-release: + needs: build-push-docker + uses: './.github/workflows/package-push-helm.yml' + secrets: inherit + with: + app-version: snapshot + version: 0.0.0-snapshot diff --git a/.github/workflows/package-push-helm.yml b/.github/workflows/package-push-helm.yml index ceede48d..2ef43744 100644 --- a/.github/workflows/package-push-helm.yml +++ b/.github/workflows/package-push-helm.yml @@ -1,12 +1,22 @@ -name: build and push Helm Charts +name: Package and Push Helm Chart on: workflow_dispatch: + inputs: + app-version: + required: true + type: string + description: "The app version that should be set in the chart" + version: + required: false + default: 0.0.0-snapshot + type: string + description: "The version of the chart to be released" workflow_call: inputs: - tag-name: + app-version: required: true type: string - target-version: + version: required: false default: 0.0.0-snapshot type: string @@ -21,23 +31,27 @@ jobs: run: | curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + - name: Checkout + uses: actions/checkout@v4 + - name: Package Helm shell: bash run: | - helm package --version '${{ inputs.target-version }}' --app-version '${{ inputs.tag-name }}' packaging/helm-charts/ksml + helm package --version '${{ inputs.version }}' --app-version '${{ inputs.app-version }}' ./packaging/helm-charts/ksml && ls ksml*.tgz - name: Login to registry shell: bash run: | - helm registry login '${{ env.AXUAL_REGISTRY }}' -u '${{ secrets.AXUAL_REGISTRY_USER }}' -p '${{ secrets.AXUAL_REGISTRY_TOKEN }}' + helm registry login registry.axual.io/opensource/charts -u '${{ secrets.AXUAL_REGISTRY_USER }}' -p '${{ secrets.AXUAL_REGISTRY_TOKEN }}' - name: Push to registry shell: bash run: | - helm push 'ksml-${{ inputs.target-version }}.tgz' + helm push 'ksml-${{ inputs.version }}.tgz' oci://registry.axual.io/opensource/charts - name: Log out of registry shell: bash if: ${{ always() }} + run: | - helm registry logout '${{ env.AXUAL_REGISTRY }}' & + helm registry logout registry.axual.io/opensource/charts & diff --git a/.github/workflows/release-push-docker.yml b/.github/workflows/release-push-docker.yml index 19c7f899..e9062232 100644 --- a/.github/workflows/release-push-docker.yml +++ b/.github/workflows/release-push-docker.yml @@ -1,10 +1,16 @@ name: build and push release Docker images on: + workflow_dispatch: + inputs: + tag: + required: true + type: string + description: "The image tag to create" release: types: [published] env: - RELEASE_TAG: ${{ github.event.release.tag_name }} + RELEASE_TAG: ${{ inputs.tag || github.event.release.tag_name }} jobs: release-push-docker: @@ -33,6 +39,13 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: login to Axual Open Source Registry + uses: docker/login-action@v2 + with: + registry: registry.axual.io + username: ${{ secrets.AXUAL_REGISTRY_USER }} + password: ${{ secrets.AXUAL_REGISTRY_TOKEN }} + - name: build and push KSML Runner uses: docker/build-push-action@v4 with: @@ -42,5 +55,16 @@ jobs: push: true tags: | axual/ksml:${{ github.event.release.tag_name }} + axual/ksml:latest ghcr.io/axual/ksml:${{ github.event.release.tag_name }} ghcr.io/axual/ksml:latest + registry.axual.io/opensource/images/axual/ksml:${{ github.event.release.tag_name }} + registry.axual.io/opensource/images/axual/ksml:latest + + helm-chart-release: + needs: release-push-docker + uses: './.github/workflows/package-push-helm.yml' + secrets: inherit + with: + app-version: ${{ inputs.tag || github.event.release.tag_name }} + version: ${{ inputs.tag || github.event.release.tag_name }} diff --git a/.gitignore b/.gitignore index 4637da44..63d8e962 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ examples/io.axual* examples/io.ksml* workspace/* +/packaging/helm-charts/**/*.tgz diff --git a/Dockerfile b/Dockerfile index 9dc85c1c..bd82b7d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ # Step 1: Create the common base image with the ksml user and group and the required packages -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9-1137 AS base +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.4-1134 AS base ENV LANG=en_US.UTF-8 # Environment variable for Connect Build and Runtime @@ -20,7 +20,8 @@ ENV PATH=/opt/graal/bin:$PATH \ KSML_INSTALL="/opt/ksml" RUN set -eux \ - && microdnf install -y procps curl tar unzip gzip zlib openssl-devel gcc gcc-c++ make patch glibc-langpack-en libxcrypt shadow-utils \ + && microdnf upgrade -y --nodocs \ + && microdnf install -y --nodocs procps tar unzip gzip zlib openssl-devel gcc gcc-c++ make patch glibc-langpack-en libxcrypt shadow-utils \ && groupadd --gid 1024 ksml \ && useradd -g ksml -u 1024 -d "$KSML_HOME" -ms /bin/sh -f -1 ksml \ && chown -R ksml:0 "$KSML_HOME" /opt \ @@ -50,7 +51,6 @@ RUN set -eux \ && mkdir -p /opt/maven \ && MVN_PKG=https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ && curl --fail --silent --location --retry 3 ${MVN_PKG} | gunzip | tar x -C /opt/maven --strip-components=1 \ -# && GRAALVM_PKG=https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-21.0.1/graalvm-community-jdk-21.0.1_linux-x64_bin.tar.gz && GRAALVM_PKG=https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-${GRAALVM_JDK_VERSION}/graalvm-community-jdk-${GRAALVM_JDK_VERSION}_linux-${JAVA_ARCH}_bin.tar.gz \ && mkdir -p /opt/graal \ && curl --fail --silent --location --retry 3 ${GRAALVM_PKG} | gunzip | tar x -C /opt/graal --strip-components=1 @@ -73,7 +73,7 @@ ENV JAVA_HOME=/opt/graalvm COPY --chown=ksml:0 --from=graal-builder /opt/graal/ /opt/graal/ WORKDIR /home/ksml -USER ksml +USER 1024 #There is no more GraalPy command here and no venv #RUN graalpy -m venv graalenv && \ # echo "source $HOME/graalenv/bin/activate" >> ~/.bashrc @@ -81,10 +81,9 @@ USER ksml # Step 5: Create the KSML Runner image FROM ksml-graal AS ksml -COPY --chown=ksml:0 --from=builder /project_dir/ksml-runner/NOTICE.txt /licences/THIRD-PARTY-LICENSES.txt -COPY --chown=ksml:0 --from=builder /project_dir/LICENSE.txt /licences/LICENSE.txt +COPY --chown=ksml:0 --from=builder /project_dir/ksml-runner/NOTICE.txt /licenses/THIRD-PARTY-LICENSES.txt +COPY --chown=ksml:0 --from=builder /project_dir/LICENSE.txt /licenses/LICENSE.txt COPY --chown=ksml:0 --from=builder /project_dir/ksml-runner/target/libs/ /opt/ksml/libs/ -COPY --chown=ksml:0 --from=builder /project_dir/graalpy-module-collection/target/modules/ /opt/ksml/modules/ COPY --chown=ksml:0 --from=builder /project_dir/ksml-runner/target/ksml-runner*.jar /opt/ksml/ksml.jar -ENTRYPOINT ["java", "--upgrade-module-path", "/opt/ksml/modules", "-jar", "/opt/ksml/ksml.jar"] +ENTRYPOINT ["java", "-jar", "/opt/ksml/ksml.jar"] diff --git a/README.md b/README.md index 77350ce8..d27c734e 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ The submodules are as follows: | Module | Description | |-----------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------| -| [`graalpy-module-collection`](graalpy-module-collection/) | contains dependencies to all GraalVM runtime modules for Python. | | [`ksml-data`](ksml-data/) | contains core data type logic, including BINARY and JSON types, used in Kafka Streams topologies and by Producers. | | [`ksml-data-avro`](ksml-data-avro/) | extension to the data library for AVRO support. | | [`ksml-data-csv`](ksml-data-csv/) | extension to the data library for CSV support. | diff --git a/graalpy-module-collection/NOTICE.txt b/graalpy-module-collection/NOTICE.txt deleted file mode 100644 index fd79689a..00000000 --- a/graalpy-module-collection/NOTICE.txt +++ /dev/null @@ -1,23 +0,0 @@ - -Lists of 21 third-party dependencies. - (Bouncy Castle Licence) Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (org.bouncycastle:bcpkix-jdk18on:1.78.1 - https://www.bouncycastle.org/java.html) - (Bouncy Castle Licence) Bouncy Castle Provider (org.bouncycastle:bcprov-jdk18on:1.78.1 - https://www.bouncycastle.org/java.html) - (Bouncy Castle Licence) Bouncy Castle ASN.1 Extension and Utility APIs (org.bouncycastle:bcutil-jdk18on:1.78.1 - https://www.bouncycastle.org/java.html) - (New BSD License (3-clause BSD license)) Sulong API (org.graalvm.llvm:llvm-api:23.1.2 - http://www.graalvm.org/) - (Universal Permissive License, Version 1.0) Polyglot (org.graalvm.polyglot:polyglot:23.1.2 - https://github.com/oracle/graal) - (MIT License) (Python Software Foundation License) (Universal Permissive License, Version 1.0) Graalpython (org.graalvm.python:python-language:23.1.2 - http://www.graalvm.org/) - (MIT License) (Python Software Foundation License) (Universal Permissive License, Version 1.0) Graalpython Resources (org.graalvm.python:python-resources:23.1.2 - http://www.graalvm.org/) - (Universal Permissive License, Version 1.0) Tregex (org.graalvm.regex:regex:23.1.2 - http://www.graalvm.org/) - (Universal Permissive License, Version 1.0) Collections (org.graalvm.sdk:collections:23.1.2 - https://github.com/oracle/graal) - (Universal Permissive License, Version 1.0) Jniutils (org.graalvm.sdk:jniutils:23.1.2 - https://github.com/oracle/graal) - (Universal Permissive License, Version 1.0) Nativeimage (org.graalvm.sdk:nativeimage:23.1.2 - https://github.com/oracle/graal) - (Universal Permissive License, Version 1.0) Word (org.graalvm.sdk:word:23.1.2 - https://github.com/oracle/graal) - (Unicode/ICU License) Truffle Icu4j (org.graalvm.shadowed:icu4j:23.1.2 - http://openjdk.java.net/projects/graal) - (Universal Permissive License, Version 1.0) Truffle Json (org.graalvm.shadowed:json:23.1.2 - http://openjdk.java.net/projects/graal) - (GNU General Public License, version 2, with the Classpath Exception) Truffle Profiler (org.graalvm.tools:profiler-tool:23.1.2 - http://openjdk.java.net/projects/graal) - (Universal Permissive License, Version 1.0) Truffle API (org.graalvm.truffle:truffle-api:23.1.2 - http://openjdk.java.net/projects/graal) - (Universal Permissive License, Version 1.0) Truffle Compiler (org.graalvm.truffle:truffle-compiler:23.1.2 - http://openjdk.java.net/projects/graal) - (Universal Permissive License, Version 1.0) Truffle Nfi (org.graalvm.truffle:truffle-nfi:23.1.2 - http://openjdk.java.net/projects/graal) - (Universal Permissive License, Version 1.0) Truffle Nfi Libffi (org.graalvm.truffle:truffle-nfi-libffi:23.1.2 - http://openjdk.java.net/projects/graal) - (Universal Permissive License, Version 1.0) Truffle Runtime (org.graalvm.truffle:truffle-runtime:23.1.2 - http://openjdk.java.net/projects/graal) - (Public Domain) XZ for Java (org.tukaani:xz:1.9 - https://tukaani.org/xz/java.html) diff --git a/graalpy-module-collection/pom.xml b/graalpy-module-collection/pom.xml deleted file mode 100644 index fafcf6f4..00000000 --- a/graalpy-module-collection/pom.xml +++ /dev/null @@ -1,102 +0,0 @@ - - - 4.0.0 - - io.axual.ksml - ksml-parent - 0.9.2-SNAPSHOT - - - graalpy-module-collection - - - 1.78.1 - 21 - 21 - UTF-8 - - - - - org.graalvm.polyglot - polyglot - provided - - - org.graalvm.truffle - truffle-api - provided - - - - org.bouncycastle - bcpkix-jdk18on - ${bouncycastle.override.version} - - - org.bouncycastle - bcutil-jdk18on - ${bouncycastle.override.version} - - - org.bouncycastle - bcprov-jdk18on - ${bouncycastle.override.version} - - - org.graalvm.python - python-language - provided - - - - org.bouncycastle - bcpkix-jdk18on - - - org.bouncycastle - bcutil-jdk18on - - - org.bouncycastle - bcprov-jdk18on - - - - - org.graalvm.python - python-resources - provided - - - org.graalvm.truffle - truffle-runtime - provided - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-dependencies - prepare-package - - copy-dependencies - - - - ${project.build.directory}/modules - - - - - - - - \ No newline at end of file diff --git a/graalpy-module-collection/src/main/java/module-info.java b/graalpy-module-collection/src/main/java/module-info.java deleted file mode 100644 index 34274007..00000000 --- a/graalpy-module-collection/src/main/java/module-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/*- - * ========================LICENSE_START================================= - * graalpy-module-collection - * %% - * Copyright (C) 2021 - 2024 Axual B.V. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================LICENSE_END================================== - */ -module graalpy.module.collection { - requires org.graalvm.truffle; - requires org.graalvm.polyglot; -} diff --git a/ksml-runner/src/main/resources/prometheus/default_config.yaml b/ksml-runner/src/main/resources/prometheus/default_config.yaml index 942c2ec1..e965e969 100644 --- a/ksml-runner/src/main/resources/prometheus/default_config.yaml +++ b/ksml-runner/src/main/resources/prometheus/default_config.yaml @@ -2,7 +2,7 @@ lowercaseOutputName: true lowercaseOutputLabelNames: true rules: - pattern: 'ksmlValue:' - name: ksml_app_info + name: ksml_app labels: app_id: "$1" name: "$2" @@ -31,4 +31,3 @@ rules: name: os_$1 type: GAUGE attrNameSnakeCase: true - - pattern: .* diff --git a/packaging/helm-charts/ksml/.helmignore b/packaging/helm-charts/ksml/.helmignore new file mode 100644 index 00000000..73543e59 --- /dev/null +++ b/packaging/helm-charts/ksml/.helmignore @@ -0,0 +1,27 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +*.tar +*.tgz +*.tbz + diff --git a/packaging/helm-charts/ksml/Chart.yaml b/packaging/helm-charts/ksml/Chart.yaml new file mode 100644 index 00000000..3caf0208 --- /dev/null +++ b/packaging/helm-charts/ksml/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +name: ksml +description: "KSML: Kafka Streaming for Low Code Environments" +type: application +version: 1.0.0-SNAPSHOT +appVersion: "1.0.0-snapshot" +icon: https://avatars.githubusercontent.com/u/26816390?s=48&v=4 +keywords: + - kafka + - streams + - streaming + - messaging + - event + - axual +home: https://ksml.io +sources: + - https://github.com/Axual/ksml +maintainers: + - name: jeroenvandisseldorp + - name: richard-axual + - name: xsreality + - name: tonvanbart diff --git a/packaging/helm-charts/ksml/LICENSE.txt b/packaging/helm-charts/ksml/LICENSE.txt new file mode 100644 index 00000000..97ff4280 --- /dev/null +++ b/packaging/helm-charts/ksml/LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021 Axual B.V. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packaging/helm-charts/ksml/README.md b/packaging/helm-charts/ksml/README.md new file mode 100644 index 00000000..548019b2 --- /dev/null +++ b/packaging/helm-charts/ksml/README.md @@ -0,0 +1,370 @@ +# ksml + +[KSML](https://ksml.io) - Kafka Streaming for Low Code Environments + +* [Introduction](#introduction) +* [Prerequisites](#prerequisites) +* [Installing the Chart](#installing-the-chart) +* [Uninstalling the Chart](#uninstalling-the-chart) +* [Configuration](#configuration) +* [Examples](#examples) + * [Plain Kafka SASL Connection](#plain-kafka-sasl-connection) + * [Axual Kafka SASL Connection](#axual-kafka-sasl-connection) + +## Introduction + +This chart deploys an easy way to define, run and deploy KSML applications on +a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes v1.21+ + +## Installing the Chart + +To install/upgrade the chart with the release name `ksml`: + +```bash +$ helm upgrade -i ksml oci://registry.axual.io/opensource/charts/ksml -n streaming --create-namespace --version=| 0.0.0-snapshot +``` + +The command deploys An easy way to define, run and deploy Kafka streaming applications on the +Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists +the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall the `ksml`: + +```bash +$ helm uninstall ksml -n streaming +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following table lists the configurable parameters of the `ksml` chart and their default values. + +| Parameter | Description | Default | +|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------| +| replicaCount | | 1 | +| image.repository | Registry to pull the image from and the name of the image. | docker.io/axual/ksml | +| image.pullPolicy | One of `Always`, `IfNotPresent`, or `Never`. | IfNotPresent | +| image.tag | Override the image tag whose default is the chart `appVersion`. | "" | +| nameOverride | Override the app name generated by the chart. | "" | +| imagePullSecrets | Override the list of ImagePullSecrets provided | [] | +| fullnameOverride | Override the fully qualified app name generated by the chart. | "" | +| applicationServer.enabled | Start the application server for the component | false | +| applicationServer.port | | "8080" | +| ksmlDefinitions | Use this dictionary to specify one or more KSML definitions and filenames | | +| ksmlRunnerConfig.definitionDirectory | Directory containing KSML Definition files. This should only be changed if an external volume mount is used. | '/ksml' | +| ksmlRunnerConfig.definitions | A dictionary specifying a namespace, and the KSML definition file that should be executed in that namespace | '/ksml' | +| ksmlRunnerConfig.schemaDirectory | Directory containing the Schema files. This should only be changed if an external volume mount is used. | '/ksml' | +| ksmlRunnerConfig.errorHandling.consume.log | log a message when consume errors occur | true | +| ksmlRunnerConfig.errorHandling.consume.logPayload | Add the record payload to the log message | false | +| ksmlRunnerConfig.errorHandling.consume.loggerName | Specify the name of the logger to use | ConsumeError | +| ksmlRunnerConfig.errorHandling.consume.stopOnError | Should the KSML application stop when a consume error occurs | true | +| ksmlRunnerConfig.errorHandling.produce.log | log a message when produce errors occur | true | +| ksmlRunnerConfig.errorHandling.produce.logPayload | Add the record payload to the log message | false | +| ksmlRunnerConfig.errorHandling.produce.loggerName | Specify the name of the logger to use | ProduceErrpr | +| ksmlRunnerConfig.errorHandling.produce.stopOnError | Should the KSML application stop when a consume error occurs | true | +| ksmlRunnerConfig.errorHandling.process.log | log a message when processing errors occur | true | +| ksmlRunnerConfig.errorHandling.process.logPayload | Add the record payload to the log message | false | +| ksmlRunnerConfig.errorHandling.process.loggerName | Specify the name of the logger to use | ProcessError | +| ksmlRunnerConfig.errorHandling.process.stopOnError | Should the KSML application stop when a consume error occurs | true | +| ksmlRunnerConfig.producersEnabled | Data generation can be disabled for deployments to prevent accidental test data in non test environments | true | +| ksmlRunnerConfig.pipelinesEnabled | Pipelines can be disabled allow for data generation only deployments. | true | +| ksmlRunnerConfig.kafka | Dictionary of Kafka connection properties and application id | | +| ksmlRunnerConfig.kafka.application.id | The application id for the app. Also used in resources and metrics | | +| ksmlRunnerConfig.kafka.bootstrap.servers | The hostname and port number for connecting to the Kafka cluster | | +| schemaDefinitions | Use this dictionary to specify one or more schema definitions and filenames | | +| serviceAccount.create | Specifies whether a service account should be created | true | +| serviceAccount.automount | Automatically mount a ServiceAccount's API credentials? | true | +| serviceAccount.name | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | "" | +| service.type | | ClusterIP | +| service.port | | 80 | +| ingress.enabled | Enable creation of the Ingress resource to expose this service. | false | +| ingress.className | The name of the IngressClass cluster resource. The associated IngressClass defines which controller will implement the resource. | "" | +| ingress.hosts | The Ingress Host definitions | | +| ingress.tls | The Ingress TLS configuration | | +| applicationServer.enabled | Start an application server for the component | false | +| applicationServer.port | Port on which Application server should be initialized | "8080" | +| logging.configFile | Location of the logging configuration file | '/ksml-logging/logback.xml' | +| logging.jsonEnabled | Enable JSON logging | false | +| logging.patterns.stdout | Pattern for the log message to standard out. See the [Logback](https://logback.qos.ch/manual/layouts.html#conversionWord) documentation for the syntax | '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} [%t] %-5level %logger{36} - %msg%n' | +| logging.patterns.stderr | Pattern for the log message to standard error. See the [Logback](https://logback.qos.ch/manual/layouts.html#conversionWord) documentation for the syntax | '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} [%t] %-5level %logger{36} - %msg%n' | +| logging.rootLogLevel | The default log level to use for all loggers | INFO | +| logging.loggers | A map of the logger name and corresponding loglevel | | +| store.spec | Specify the VolumeClaimTemplate used by the StatefulSet. Default it uses access mode "ReadWriteOnce" and requests 1Gi | | +| prometheus.enabled | Enable Prometheus Metrics export | false | +| prometheus.port | Port on which Prometheus metrics endpoint should be initialized | "9999" | +| prometheus.config | The configuration for the [Prometheus JMX Exporter](https://github.com/prometheus/jmx_exporter/blob/1.0.1/README.md). The 1.x.x config is used | | +| serviceMonitor.enabled | Enables creation of Prometheus Operator [ServiceMonitor](https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.ServiceMonitor). Ignored if API `monitoring.coreos.com/v1` is not available. | true | +| serviceMonitor.interval | Interval at which metrics should be scraped. | 30s | +| serviceMonitor.scrapeTimeout | Timeout after which the scrape is ended. | 10s | +| serviceMonitor.labels | A dictionary with additional labels for the service monitor | | +| affinity | The pod's scheduling constraints. See the Kubernetes documentation on [Affinity and Anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) | | +| topologySpreadConstraints | Describes how a group of pods ought to spread across topology domains. See the Kubernetes documentation on [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/). | [] | +| tolerations | The tolerations on this pod. See the Kubernetes documentation on [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/). | [] | +| nodeSelector | Selector which must match a node's labels for the pod to be scheduled on that node. | | +| volumeMounts | Additional volumeMounts on the output Deployment definition | [] | +| volumes | Additional volumes on the output Deployment definition | [] | +| resources | Resource requests and limits for KSML. | | +| securityContext | Defines the security options the container should be run with. If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext | | +| podSecurityContext | Pod-level security attributes and common container settings | | +| podLabels | Extra labels to add to the Pods | | +| podAnnotations | Extra annotations to add to the Pods | | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm upgrade -i`. For +example: + +```bash +$ helm upgrade -i ksml oci://registry.axual.io/opensource/charts/ksml -n streaming --create-namespace --version=| 0.0.0-snapshot --set replicaCount=1 +``` + +Alternatively, a YAML file that specifies the values for the parameters can be provided while +installing the chart. For example: + +```bash +$ helm upgrade -i ksml oci://registry.axual.io/opensource/charts/ksml -n streaming --create-namespace --version=| 0.0.0-snapshot --values values.yaml +``` +## Examples + +### Plain Kafka SASL Connection +
+ Example configuration plain Kafka SASL + +```YAML +image: + tag: snapshot + +replicaCount: 3 + +resources: + requests: + memory: "128Mi" + cpu: "250m" + limits: + memory: "768Mi" + cpu: "500m" + +ksmlRunnerConfig: + definitions: +# generate: generator.yaml + inspect: peek.yaml + kafka: + application.id: example.datagen + + bootstrap.servers: 'testing-sasl-broker:9095' + security.protocol: SASL_SSL + sasl.mechanism: SCRAM-SHA-512 + sasl.jaas.config: 'org.apache.kafka.common.security.scram.ScramLoginModule required username=example_user password=12345678 ;' + +ksmlDefinitions: + generator.yaml: | + # This example shows how to generate data and have it sent to a target topic in a given format. + + functions: + generate_sensordata_message: + type: generator + globalCode: | + import time + import random + sensorCounter = 0 + code: | + global sensorCounter + + key = "sensor"+str(sensorCounter) # Set the key to return ("sensor0" to "sensor9") + sensorCounter = (sensorCounter+1) % 10 # Increase the counter for next iteration + + # Generate some random sensor measurement data + types = { 0: { "type": "AREA", "unit": random.choice([ "m2", "ft2" ]), "value": str(random.randrange(1000)) }, + 1: { "type": "HUMIDITY", "unit": random.choice([ "g/m3", "%" ]), "value": str(random.randrange(100)) }, + 2: { "type": "LENGTH", "unit": random.choice([ "m", "ft" ]), "value": str(random.randrange(1000)) }, + 3: { "type": "STATE", "unit": "state", "value": random.choice([ "off", "on" ]) }, + 4: { "type": "TEMPERATURE", "unit": random.choice([ "C", "F" ]), "value": str(random.randrange(-100, 100)) } + } + + # Build the result value using any of the above measurement types + value = { "name": key, "timestamp": str(round(time.time()*1000)), **random.choice(types) } + value["color"] = random.choice([ "black", "blue", "red", "yellow", "white" ]) + value["owner"] = random.choice([ "Alice", "Bob", "Charlie", "Dave", "Evan" ]) + value["city"] = random.choice([ "Amsterdam", "Xanten", "Utrecht", "Alkmaar", "Leiden" ]) + + if random.randrange(10) == 0: + value = None + expression: (key, value) # Return a message tuple with the key and value + resultType: (string, json) # Indicate the type of key and value + + producers: + sensordata_json_producer: + generator: generate_sensordata_message + interval: 444 + to: + topic: ksml_sensordata_json + keyType: string + valueType: json + +applicationServer: + enabled: true + port: "8080" + +prometheus: + enabled: true + port: "9993" + config: + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + - pattern: 'ksmlValue:' + name: ksml_app_info + labels: + app_id: "$1" + name: "$2" + version: "$3" + build_time: "$4" + type: GAUGE + value: 1 + cached: true + - pattern: .* + +logging: + configFile: '/ksml-logging/logback.xml' + jsonEnabled: false + patterns: + stdout: '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} %-5level %logger{36} - %msg%n' + stderr: '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} %-5level %logger{36} - %msg%n' + loggers: + org.apache.kafka: INFO + org.apache.kafka.streams.StreamsConfig: INFO + org.apache.kafka.clients.consumer.ConsumerConfig: INFO + +``` +
+ +### Axual Kafka SASL Connection +
+ Example configuration to connect to an Axual Kafka cluster +See the ksmlRunnerConfig.kafka section for the special additions to automatically resolve resource names to the Axual namespaces + +```YAML +image: + tag: snapshot + +replicaCount: 3 + +resources: + requests: + memory: "128Mi" + cpu: "250m" + limits: + memory: "768Mi" + cpu: "500m" + +ksmlRunnerConfig: + definitions: +# generate: generator.yaml + inspect: peek.yaml + kafka: + application.id: example.datagen + + bootstrap.servers: 'testing-sasl-broker.axual.cloud:9095' + security.protocol: SASL_SSL + sasl.mechanism: SCRAM-SHA-512 + sasl.jaas.config: 'org.apache.kafka.common.security.scram.ScramLoginModule required username=example_user password=12345678 ;' + + # Use these configuration properties when connecting to a cluster using the Axual naming patterns. + # These patterns are resolved into the actual name used on Kafka using the values in this configuration map + # and the topic names specified in the definition YAML files + + tenant: 'custom' + instance: 'dta' + environment: 'test' + group.id.pattern: '{tenant}-{instance}-{environment}-{group.id}' + topic.pattern: '{tenant}-{instance}-{environment}-{topic}' + transactional.id.pattern: '{tenant}-{instance}-{environment}-{transactional.id}' + +ksmlDefinitions: + generator.yaml: | + # This example shows how to generate data and have it sent to a target topic in a given format. + + functions: + generate_sensordata_message: + type: generator + globalCode: | + import time + import random + sensorCounter = 0 + code: | + global sensorCounter + + key = "sensor"+str(sensorCounter) # Set the key to return ("sensor0" to "sensor9") + sensorCounter = (sensorCounter+1) % 10 # Increase the counter for next iteration + + # Generate some random sensor measurement data + types = { 0: { "type": "AREA", "unit": random.choice([ "m2", "ft2" ]), "value": str(random.randrange(1000)) }, + 1: { "type": "HUMIDITY", "unit": random.choice([ "g/m3", "%" ]), "value": str(random.randrange(100)) }, + 2: { "type": "LENGTH", "unit": random.choice([ "m", "ft" ]), "value": str(random.randrange(1000)) }, + 3: { "type": "STATE", "unit": "state", "value": random.choice([ "off", "on" ]) }, + 4: { "type": "TEMPERATURE", "unit": random.choice([ "C", "F" ]), "value": str(random.randrange(-100, 100)) } + } + + # Build the result value using any of the above measurement types + value = { "name": key, "timestamp": str(round(time.time()*1000)), **random.choice(types) } + value["color"] = random.choice([ "black", "blue", "red", "yellow", "white" ]) + value["owner"] = random.choice([ "Alice", "Bob", "Charlie", "Dave", "Evan" ]) + value["city"] = random.choice([ "Amsterdam", "Xanten", "Utrecht", "Alkmaar", "Leiden" ]) + + if random.randrange(10) == 0: + value = None + expression: (key, value) # Return a message tuple with the key and value + resultType: (string, json) # Indicate the type of key and value + + producers: + sensordata_json_producer: + generator: generate_sensordata_message + interval: 444 + to: + topic: ksml_sensordata_json + keyType: string + valueType: json + +applicationServer: + enabled: true + port: "8080" + +prometheus: + enabled: true + port: "9993" + config: + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + - pattern: 'ksmlValue:' + name: ksml_app_info + labels: + app_id: "$1" + name: "$2" + version: "$3" + build_time: "$4" + type: GAUGE + value: 1 + cached: true + - pattern: .* + +logging: + configFile: '/ksml-logging/logback.xml' + jsonEnabled: false + patterns: + stdout: '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} %-5level %logger{36} - %msg%n' + stderr: '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} %-5level %logger{36} - %msg%n' + loggers: + org.apache.kafka: INFO + org.apache.kafka.streams.StreamsConfig: INFO + org.apache.kafka.clients.consumer.ConsumerConfig: INFO +``` +
+ diff --git a/packaging/helm-charts/ksml/templates/NOTES.txt b/packaging/helm-charts/ksml/templates/NOTES.txt new file mode 100644 index 00000000..9cf433c5 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/NOTES.txt @@ -0,0 +1 @@ +Thank you for installing {{ .Chart.Name }}-{{ .Chart.Version }} \ No newline at end of file diff --git a/packaging/helm-charts/ksml/templates/_helpers.tpl b/packaging/helm-charts/ksml/templates/_helpers.tpl new file mode 100644 index 00000000..5a0bf225 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ksml.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ksml.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ksml.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ksml.labels" -}} +helm.sh/chart: {{ include "ksml.chart" . }} +{{ include "ksml.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ksml.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ksml.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ksml.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ksml.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/packaging/helm-charts/ksml/templates/configmap-logging.yaml b/packaging/helm-charts/ksml/templates/configmap-logging.yaml new file mode 100644 index 00000000..f2d7b13f --- /dev/null +++ b/packaging/helm-charts/ksml/templates/configmap-logging.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "ksml.fullname" . }}-logging + labels: + {{- include "ksml.labels" . | nindent 4 }} +data: + logback.xml: | + + + + + {{- if .Values.logging.jsonEnabled }} + + {{- else }} + + {{ .Values.logging.patterns.stderr }} + + {{- end }} + + System.err + + WARN + + + + {{- if .Values.logging.jsonEnabled }} + + {{- else }} + + {{ .Values.logging.patterns.stdout }} + + {{- end }} + + System.out + + TRACE + ACCEPT + + + DEBUG + ACCEPT + + + INFO + ACCEPT + + + WARN + DENY + + + ERROR + DENY + + + + + + + + + + + {{- range $key, $value := (mergeOverwrite .Values.logging.defaultLoggers .Values.logging.loggers ) }} + + {{- end }} + + + diff --git a/packaging/helm-charts/ksml/templates/configmap-runner.yaml b/packaging/helm-charts/ksml/templates/configmap-runner.yaml new file mode 100644 index 00000000..dd84cc65 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/configmap-runner.yaml @@ -0,0 +1,60 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "ksml.fullname" . }}-runner + labels: + {{- include "ksml.labels" . | nindent 4 }} +data: + ksml-runner.yaml: | + ksml: + configDirectory: {{ .Values.ksmlRunnerConfig.definitionDirectory | quote }} + schemaDirectory: {{ .Values.ksmlRunnerConfig.schemaDirectory | quote }} + storageDirectory: "/ksml-store" + enableProducers: {{ .Values.ksmlRunnerConfig.producersEnabled }} + enablePipelines: {{ .Values.ksmlRunnerConfig.pipelinesEnabled }} + errorHandling: + consume: + log: {{ .Values.ksmlRunnerConfig.errorHandling.consume.log }} + logPayload: {{ .Values.ksmlRunnerConfig.errorHandling.consume.logPayload }} + loggerName: {{ .Values.ksmlRunnerConfig.errorHandling.consume.loggerName }} + handler: {{ ternary "stopOnFail" "continueOnFail" .Values.ksmlRunnerConfig.errorHandling.consume.stopOnError }} + produce: + log: {{ .Values.ksmlRunnerConfig.errorHandling.produce.log }} + logPayload: {{ .Values.ksmlRunnerConfig.errorHandling.produce.logPayload }} + loggerName: {{ .Values.ksmlRunnerConfig.errorHandling.produce.loggerName }} + handler: {{ ternary "stopOnFail" "continueOnFail" .Values.ksmlRunnerConfig.errorHandling.produce.stopOnError }} + process: + log: {{ .Values.ksmlRunnerConfig.errorHandling.process.log }} + logPayload: {{ .Values.ksmlRunnerConfig.errorHandling.process.logPayload }} + loggerName: {{ .Values.ksmlRunnerConfig.errorHandling.process.loggerName }} + handler: {{ ternary "stopOnFail" "continueOnFail" .Values.ksmlRunnerConfig.errorHandling.process.stopOnError }} + applicationServer: + enabled: {{ .Values.applicationServer.enabled }} + port: {{ .Values.applicationServer.port }} + prometheus: + enabled: {{ .Values.prometheus.enabled }} + port: {{ .Values.prometheus.port }} + configFile: "/ksml/ksml-metrics-config.yml" + definitions: +{{- with .Values.ksmlRunnerConfig.definitions }} + {{- toYaml . | nindent 8 }} +{{- end }} + kafka: + config.providers: env + config.providers.env.class: 'org.apache.kafka.common.config.provider.EnvVarConfigProvider' + group.instance.id: '${env:KSML_GROUP_INSTANCE_ID}' +{{- with .Values.ksmlRunnerConfig.kafka }} + {{- toYaml . | nindent 6 }} +{{- end }} + +{{- with .Values.ksmlDefinitions }} + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.schemaDefinitions }} + {{- toYaml . | nindent 2 }} +{{- end }} + + ksml-metrics-config.yml: | + {{- with .Values.prometheus.config }} + {{- toYaml . | nindent 4 }} + {{- end }} diff --git a/packaging/helm-charts/ksml/templates/headless-service.yaml b/packaging/helm-charts/ksml/templates/headless-service.yaml new file mode 100644 index 00000000..fae0fdbd --- /dev/null +++ b/packaging/helm-charts/ksml/templates/headless-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ksml.fullname" . }}-headless + labels: + {{- include "ksml.labels" . | nindent 4 }} +spec: + type: ClusterIP + publishNotReadyAddresses: true + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "ksml.selectorLabels" . | nindent 4 }} diff --git a/packaging/helm-charts/ksml/templates/ingress.yaml b/packaging/helm-charts/ksml/templates/ingress.yaml new file mode 100644 index 00000000..3b1bb237 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "ksml.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "ksml.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/packaging/helm-charts/ksml/templates/service.yaml b/packaging/helm-charts/ksml/templates/service.yaml new file mode 100644 index 00000000..c0d20f54 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ksml.fullname" . }} + labels: + {{- include "ksml.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + - port: {{ .Values.prometheus.port }} + targetPort: metrics + protocol: TCP + name: metrics + selector: + {{- include "ksml.selectorLabels" . | nindent 4 }} diff --git a/packaging/helm-charts/ksml/templates/serviceaccount.yaml b/packaging/helm-charts/ksml/templates/serviceaccount.yaml new file mode 100644 index 00000000..a8049936 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ksml.serviceAccountName" . }} + labels: + {{- include "ksml.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/packaging/helm-charts/ksml/templates/servicemonitor.yaml b/packaging/helm-charts/ksml/templates/servicemonitor.yaml new file mode 100644 index 00000000..7baefd10 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/servicemonitor.yaml @@ -0,0 +1,20 @@ +{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "ksml.fullname" . }} + labels: + {{- include "ksml.labels" . | nindent 4 }} + {{- if (.Values.serviceMonitor.labels) }} + {{- toYaml .Values.serviceMonitor.labels | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "ksml.selectorLabels" . | nindent 6 }} + endpoints: + - port: metrics + path: /metrics + interval: {{ .Values.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} +{{- end }} diff --git a/packaging/helm-charts/ksml/templates/statefulset.yaml b/packaging/helm-charts/ksml/templates/statefulset.yaml new file mode 100644 index 00000000..6b1a3394 --- /dev/null +++ b/packaging/helm-charts/ksml/templates/statefulset.yaml @@ -0,0 +1,104 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "ksml.fullname" . }} + labels: + {{- include "ksml.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "ksml.selectorLabels" . | nindent 6 }} + serviceName: {{ include "ksml.fullname" . }}-headless + podManagementPolicy: Parallel + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap-runner.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "ksml.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "ksml.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + workingDir: /ksml + ports: + - name: http + containerPort: {{ .Values.applicationServer.port }} + protocol: TCP + - name: metrics + containerPort: {{ .Values.prometheus.port }} + protocol: TCP + env: + - name: "KSML_K8S_POD_NAME" + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KSML_GROUP_INSTANCE_ID + value: "$(KSML_K8S_POD_NAME)" + - name: LOGBACK_CONFIGURATION_FILE + value: {{ .Values.logging.configFile | quote }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: ksml-config + mountPath: /ksml + - name: ksml-logging + mountPath: /ksml-logging + - name: ksml-store + mountPath: /ksml-store + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: ksml-config + configMap: + name: {{ include "ksml.fullname" . }}-runner + - name: ksml-logging + configMap: + name: {{ include "ksml.fullname" . }}-logging + {{- with .Values.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologySpreadConstraints}} + topologySpreadConstraints: + {{- range . }} + - labelSelector: + matchLabels: + {{- include "ksml.selectorLabels" $ | nindent 12 }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: ksml-store + spec: + {{- toYaml .Values.store.spec | nindent 8 }} diff --git a/packaging/helm-charts/ksml/values.schema.json b/packaging/helm-charts/ksml/values.schema.json new file mode 100644 index 00000000..fc30393d --- /dev/null +++ b/packaging/helm-charts/ksml/values.schema.json @@ -0,0 +1,330 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "affinity": { + "type": "object" + }, + "applicationServer": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "port": { + "type": "string" + } + } + }, + "fullnameOverride": { + "type": "string" + }, + "image": { + "type": "object", + "properties": { + "pullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "imagePullSecrets": { + "type": "array" + }, + "ingress": { + "type": "object", + "properties": { + "annotations": { + "type": "object" + }, + "className": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "hosts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "host": { + "type": "string" + }, + "paths": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "pathType": { + "type": "string" + } + } + } + } + } + } + }, + "tls": { + "type": "array" + } + } + }, + "ksmlDefinitions": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "ksmlRunnerConfig": { + "type": "object", + "properties": { + "definitionDirectory": { + "type": "string" + }, + "schemaDirectory": { + "type": "string" + }, + "producersEnabled": { + "type": "boolean" + }, + "pipelinesEnabled": { + "type": "boolean" + }, + "errorHandling": { + "type": "object", + "properties": { + "consume" : {"$ref": "#/definitions/ErrorHandling"}, + "produce" : {"$ref": "#/definitions/ErrorHandling"}, + "process" : {"$ref": "#/definitions/ErrorHandling"} + } + }, + "definitions": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + }, + "kafka": { + "type": "object", + "properties": { + "application.id": { + "type": "string" + }, + "bootstrap.servers": { + "type": "string" + } + }, + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + }, + "required": [ + "application.id", + "bootstrap.servers" + ] + } + }, + "required": [ + "definitionDirectory", + "schemaDirectory", + "definitions", + "kafka" + ] + }, + "logging": { + "type": "object", + "properties": { + "configFile": { + "type": "string" + }, + "jsonEnabled": { + "type": "boolean" + }, + "patters": { + "type": "object", + "properties": { + "stdout": { + "type": "string" + }, + "stderr": { + "type": "string" + } + } + }, + "rootLogLevel": { + "$ref": "#/definitions/LogLevel" + }, + "loggers": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/LogLevel" + } + }, + "defaultLoggers": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/LogLevel" + } + } + } + }, + "nameOverride": { + "type": "string" + }, + "nodeSelector": { + "type": "object" + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "podSecurityContext": { + "type": "object" + }, + "prometheus": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "port": { + "type": "string" + }, + "config": { + "type": "object" + } + } + }, + "replicaCount": { + "type": "integer", + "minimum": 0 + }, + "resources": { + "type": "object" + }, + "schemaDefinitions": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "securityContext": { + "type": "object" + }, + "service": { + "type": "object", + "properties": { + "port": { + "type": "integer" + }, + "type": { + "type": "string" + } + } + }, + "serviceAccount": { + "type": "object", + "properties": { + "annotations": { + "type": "object" + }, + "automount": { + "type": "boolean" + }, + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + } + }, + "store": { + "type": "object" + }, + "serviceMonitor": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "interval": { + "type": "string" + }, + "labels": { + "type": "object" + }, + "scrapeTimeout": { + "type": "string" + } + } + }, + "tolerations": { + "type": "array" + }, + "topologySpreadConstraints": { + "type": "array" + }, + "volumeMounts": { + "type": "array" + }, + "volumes": { + "type": "array" + } + }, + "definitions": { + "LogLevel": { + "enum": [ + "TRACE", + "DEBUG", + "INFO", + "WARN", + "ERROR", + "trace", + "debug", + "info", + "warn", + "error" + ] + }, + "ErrorHandling": { + "type": "object", + "properties": { + "log": { + "type": "boolean" + }, + "logPayload": { + "type": "boolean" + }, + "stopOnError": { + "type": "boolean" + }, + "logger": { + "type": "string", + "minLength": 1 + } + } + } + } +} diff --git a/packaging/helm-charts/ksml/values.yaml b/packaging/helm-charts/ksml/values.yaml new file mode 100644 index 00000000..0abba731 --- /dev/null +++ b/packaging/helm-charts/ksml/values.yaml @@ -0,0 +1,270 @@ +# Default values for ksml. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + # -- Registry to pull the image from and the name of the image. + repository: registry.axual.io/opensource/images/axual/ksml + # -- One of `Always`, `IfNotPresent`, or `Never`. + pullPolicy: IfNotPresent + # -- Override the image tag whose default is the chart `appVersion`. + tag: "" + +# -- Override the list of ImagePullSecrets provided. +imagePullSecrets: [] +# -- Override the app name generated by the chart. +nameOverride: "" +# -- Override the fully qualified app name generated by the chart. +fullnameOverride: "" + +# -- KSML runner configuration. See https://axual.github.io/ksml/runners.html +ksmlRunnerConfig: + # -- Directory containing KSML Definition files. This should only be changed if an external volume mount is used. + definitionDirectory: '/ksml' + # -- Directory containing the Schema files. This should only be changed if an external volume mount is used. + schemaDirectory: '/ksml' + + # -- KSML Error handling settings. Error Handling can be defined for consume, produce and processing errors. + errorHandling: + # -- Error handling settings for consume related errors + consume: + # -- log a message when consume errors occur + log: true + # -- Add the record payload to the log message + logPayload: false + # -- Specify the name of the logger to use + loggerName: ConsumeError + # -- Should the KSML application stop when a consume error occurs + stopOnError: true + # -- Error handling settings for produce related errors + produce: + # -- log a message when produce errors occur + log: true + # -- Add the record payload to the log message + logPayload: false + # -- Specify the name of the logger to use + loggerName: ProduceErrpr + # -- Should the KSML application stop when a consume error occurs + stopOnError: true + # -- Error handling settings for processing related errors + process: + # -- log a message when processing errors occur + log: true + # -- Add the record payload to the log message + logPayload: false + # -- Specify the name of the logger to use + loggerName: ProcessError + # -- Should the KSML application stop when a consume error occurs + stopOnError: true + + # -- Data generation can be disabled for deployments to prevent accidental test data in non test environments + producersEnabled: true + # -- Pipelines can be disabled allow for data generation only deployments. + pipelinesEnabled: true + + # -- Map of the namespace and the KSML definition file executed in that namespace. + definitions: {} + # -- Map of Kafka connection properties and application id + kafka: + # kafka: + # application.id: my-streaming-app + # bootstrap.servers: "localhost:9092" + +# KSML Pipeline or Producer definition files. See https://axual.github.io/ksml/introduction.html +ksmlDefinitions: {} + # my-definition.yaml: | + # ... + +# Schema definitions files. See https://github.com/Axual/ksml/tree/main/examples +schemaDefinitions: {} + # my-schema.avsc: | + # ... + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# -- Extra annotations to add to the Pods. +podAnnotations: {} + +# -- Extra labels to add to the Pods. +podLabels: {} + +# -- Pod-level security attributes and common container settings. +podSecurityContext: {} + # fsGroup: 2000 + +# -- Defines the security options the container should be run with. +# If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. +# @default -- See `values.yaml` file. +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + # -- Enable creation of the Ingress resource to expose this service. + enabled: false + # -- The name of the IngressClass cluster resource. + # The associated IngressClass defines which controller will implement the resource. + className: "" + # -- Annotations to add to the Ingress resource. + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + # -- TLS configuration for this Ingress. + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: "1000m" + # memory: "768Mi" + # requests: + # cpu: "500m" + # memory: "128Mi" + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +# -- Selector which must match a node's labels for the pod to be scheduled on that node. +nodeSelector: {} + +# -- The tolerations on this pod. See the Kubernetes documentation on [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/). +tolerations: [] + +# -- The pod's scheduling constraints. See the Kubernetes documentation on [Affinity and Anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). +affinity: {} + +# -- Describes how a group of pods ought to spread across topology domains. See the Kubernetes documentation on [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/). +topologySpreadConstraints: [] + +# -- KSML application server port defined in the statefulset. +applicationServer: + # -- Start an application server for the component + enabled: false + # -- Port on which Application server should be initialized + port: "8080" + +# -- Logging configuration settings for KSML. +logging: + # -- Location of the logging configuration file + configFile: '/ksml-logging/logback.xml' + # -- Enable JSON logging + jsonEnabled: false + # -- The logback log pattern to use + patterns: + stdout: '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} [%t] %-5level %logger{36} - %msg%n' + stderr: '%date{"yyyy-MM-dd''T''HH:mm:ss,SSSXXX", UTC} [%t] %-5level %logger{36} - %msg%n' + # -- The default log level to use for all loggers + rootLogLevel: INFO + # -- Specify log levels for specific loggers. + loggers: {} + # -- Specify default loggers set for all deployments to minimize excessive logging + defaultLoggers: + io.axual.ksml: INFO + org.apache.kafka: WARN + org.apache.kafka.clients.consumer.ConsumerConfig: WARN + org.apache.kafka.clients.producer.ProducerConfig: WARN + org.apache.kafka.clients.admin.AdminClientConfig: WARN + org.apache.kafka.streams.StreamsConfig: WARN + io.confluent: WARN + +# -- Configure the storage specification for the volumeClaimTemplates +store: + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + +# -- KSML prometheus server port defined in the statefulset. Note that this must match the port defined in ksmlRunnerConfig. +prometheus: + enabled: false + # -- Port on which Prometheus metrics endpoint should be initialized + port: "9999" + # -- Contents of the JMX exporter config file + config: + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + - pattern: 'ksmlValue:' + name: ksml_app + labels: + app_id: "$1" + name: "$2" + version: "$3" + build_time: "$4" + type: GAUGE + value: 1 + cached: true + - pattern: 'ksml' + type: COUNTER + - pattern: 'ksml' + type: GAUGE + - pattern: 'kafka.streams' + type: GAUGE + - pattern: 'kafka.producer' + type: GAUGE + - pattern: 'kafka.consumer' + type: GAUGE + - pattern: 'kafka.admin.client' + type: GAUGE + - pattern: 'java.lang<>(committed_virtual_memory|free_physical_memory|free_swap_space|total_physical_memory|total_swap_space)_size:' + name: os_$1_bytes + type: GAUGE + attrNameSnakeCase: true + - pattern: 'java.lang<>((?!process_cpu_time)\w+):' + name: os_$1 + type: GAUGE + attrNameSnakeCase: true + + + +serviceMonitor: + # -- Enables creation of Prometheus Operator [ServiceMonitor](https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.ServiceMonitor). + # Ignored if API `monitoring.coreos.com/v1` is not available. + enabled: true + # -- Interval at which metrics should be scraped. + interval: 30s + # -- Timeout after which the scrape is ended. + scrapeTimeout: 10s + # -- Additional labels for the ServiceMonitor + labels: {} diff --git a/pom.xml b/pom.xml index 0508bc0b..16e98925 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,6 @@ ksml-kafka-clients ksml-runner ksml-reporting - graalpy-module-collection