diff --git a/.github/workflows/clean.yml b/.github/workflows/clean.yml new file mode 100644 index 0000000..f9efbe0 --- /dev/null +++ b/.github/workflows/clean.yml @@ -0,0 +1,38 @@ +name: Branch Deleted +on: delete + +env: + TAG_NAME: ${{ github.event.ref }} + +jobs: + delete: + strategy: + fail-fast: false + matrix: + component: + - name: qubership-hive-metastore-transfer + - name: qubership-hive-metastore + if: github.event.ref_type == 'branch' + runs-on: ubuntu-latest + steps: + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${GITHUB_ACTOR} + password: ${{secrets.GITHUB_TOKEN}} + - name: Prepare Tag + run: echo "TAG_NAME=$(echo ${TAG_NAME} | sed 's@refs/heads/@@;s@/@_@g')" >> $GITHUB_ENV + - name: Get package IDs for delete + id: get-ids-for-delete + uses: Netcracker/get-package-ids@v0.0.1 + with: + component-name: ${{ matrix.component.name }} + component-tag: ${{ env.TAG_NAME }} + access-token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/delete-package-versions@v5 + with: + package-name: ${{ matrix.component.name }} + package-type: 'container' + package-version-ids: ${{ steps.get-ids-for-delete.outputs.ids-for-delete }} + if: ${{ steps.get-ids-for-delete.outputs.ids-for-delete != '' }} \ No newline at end of file diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..f2ba431 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,63 @@ +name: Build Artifacts +on: + release: + types: [created] + push: + branches: + - '**' +env: + TAG_NAME: ${{ github.event.release.tag_name || github.ref }} + +jobs: + multiplatform_build: + strategy: + fail-fast: false + matrix: + component: + - name: qubership-hive-metastore-transfer + file: docker-transfer/Dockerfile + context: "" + - name: qubership-hive-metastore + file: docker/Dockerfile + context: "" + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${GITHUB_ACTOR} + password: ${{secrets.GITHUB_TOKEN}} + - name: Prepare Tag + run: echo "TAG_NAME=$(echo ${TAG_NAME} | sed 's@refs/tags/@@;s@refs/heads/@@;s@/@_@g')" >> $GITHUB_ENV + - name: Get package IDs for delete + id: get-ids-for-delete + uses: Netcracker/get-package-ids@v0.0.1 + with: + component-name: ${{ matrix.component.name }} + component-tag: ${{ env.TAG_NAME }} + access-token: ${{ secrets.GH_ACCESS_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + no-cache: true + context: ${{ matrix.component.context }} + file: ${{ matrix.component.file }} + platforms: linux/amd64,linux/arm64 + push: true + tags: ghcr.io/netcracker/${{ matrix.component.name }}:${{ env.TAG_NAME }} + provenance: false + build-args: | + GH_ACCESS_TOKEN=${{ secrets.GH_ACCESS_TOKEN }} + - uses: actions/delete-package-versions@v5 + with: + package-name: ${{ matrix.component.name }} + package-type: 'container' + package-version-ids: ${{ steps.get-ids-for-delete.outputs.ids-for-delete }} + if: ${{ steps.get-ids-for-delete.outputs.ids-for-delete != '' }} \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d4b0c5a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.iml +.idea \ No newline at end of file diff --git a/CODE-OF-CONDUCT.md b/CODE-OF-CONDUCT.md new file mode 100644 index 0000000..aec27d2 --- /dev/null +++ b/CODE-OF-CONDUCT.md @@ -0,0 +1,73 @@ +# Code of Conduct + +This repository is governed by following code of conduct guidelines. + +We put collaboration, trust, respect and transparency as core values for our community. +Our community welcomes participants from all over the world with different experience, +opinion and ideas to share. + +We have adopted this code of conduct and require all contributors to agree with that to build a healthy, +safe and productive community for all. + +The guideline is aimed to support a community where all people should feel safe to participate, +introduce new ideas and inspire others, regardless of: + +* Age +* Gender +* Gender identity or expression +* Family status +* Marital status +* Ability +* Ethnicity +* Race +* Sex characteristics +* Sexual identity and orientation +* Education +* Native language +* Background +* Caste +* Religion +* Geographic location +* Socioeconomic status +* Personal appearance +* Any other dimension of diversity + +## Our Standards + +We are welcoming the following behavior: + +* Be respectful for different ideas, opinions and points of view +* Be constructive and professional +* Use inclusive language +* Be collaborative and show the empathy +* Focus on the best results for the community + +The following behavior is unacceptable: + +* Violence, threats of violence, or inciting others to commit self-harm +* Personal attacks, trolling, intentionally spreading misinformation, insulting/derogatory comments +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Derogatory language +* Encouraging unacceptable behavior +* Other conduct which could reasonably be considered inappropriate in a professional community + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of the Code of Conduct +and are expected to take appropriate actions in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, +commits, code, wiki edits, issues, and other contributions that are not aligned +to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors +that they deem inappropriate, threatening, offensive, or harmful. + +## Reporting + +If you believe you’re experiencing unacceptable behavior that will not be tolerated as outlined above, +please report to `opensourcegroup@netcracker.com`. All complaints will be reviewed and investigated and will result in a response +that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality +with regard to the reporter of an incident. + +Please also report if you observe a potentially dangerous situation, someone in distress, or violations of these guidelines, +even if the situation is not happening to you. \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4bfc0ea --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contribution Guide + +We'd love to accept patches and contributions to this project. +Please, follow these guidelines to make the contribution process easy and effective for everyone involved. + +## Contributor License Agreement + +You must sign the [Contributor License Agreement](https://pages.netcracker.com/cla-main.html) in order to contribute. + +## Code of Conduct + +Please make sure to read and follow the [Code of Conduct](CODE-OF-CONDUCT.md). \ No newline at end of file diff --git a/LICENSE b/LICENSE index 261eeb9..7a4a3ea 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -198,4 +199,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. + limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md index ac20b44..1fe24ef 100644 --- a/README.md +++ b/README.md @@ -1 +1,55 @@ -# qubership-hive-metastore \ No newline at end of file +## Table of Contents + +* [Overview](#overview) +* [Architecture](docs/public/architecture.md) +* [Installation Guide](docs/public/installation.md) + +## Overview + +Qubership Hive-Metastore is a comprehensive solution for deploying [Hive-Metastore](https://hive.apache.org/) in Kubernetes. + +In Kubernetes Qubership Hive-Metastore uses Minio S3 storage to use data and PostgreSQL to store metadata. + +The table below shows services in K8s that can potentially replaces services in Hadoop cluster. + +**Note** [Apache Hive](https://github.com/apache/hive) contains two large parts, hive-metastore and hiveServer2. This project contains only hive-metastore and does not hiveServer2 libraries. Hive-metastore libraries can be found at https://repo1.maven.org/maven2/org/apache/hive/hive-standalone-metastore-server . + +## Repository structure + +* `chart` - helm charts for Qubership Hive-Metastore. +* `docker` - files for building Qubership Hive-Metastore docker image +* `docs` - Qubership Hive-Metastore documentation. + +### How to debug and troubleshoot + +After deploying to K8s, `log4j2-properties` configmap is created, where it is possible to change hive logging level. + +#### Connecting to hive-metastore + +It is possible to connect to deployed Qubership Hive-Metastore using [Trino](https://github.com/trinodb/trino) or [Spark](https://github.com/apache/spark). Example PySpark configuration can be found below + +```python +from pyspark.sql import SparkSession +... +spark = SparkSession \ + .builder.master("local") \ + .appName("MyApp.com") \ + .config("spark.sql.warehouse.dir", "s3a://hive/warehouse") \ + .config("spark.sql.hive.metastore.version", "3.1.3") \ + .config("spark.sql.hive.metastore.jars", "maven") \ + .config("spark.hadoop.hive.metastore.uris", "thrift://hive_address") \ + .config("spark.hadoop.hive.metastore.schema.verification", "false") \ + .config("spark.hadoop.hive.metastore.schema.verification.record.version", "false") \ + .config("spark.hadoop.hive.metastore.use.SSL", "false") \ + .config('spark.hadoop.fs.s3.buckets.create.enabled', 'true') \ + .config('spark.hadoop.fs.s3a.endpoint', 'https://s3.endpoint.address.com') \ + .config('spark.hadoop.fs.s3a.access.key', 's3accesskey') \ + .config('spark.hadoop.fs.s3a.secret.key', 's3secretkey') \ + .config('spark.hadoop.fs.s3a.connection.ssl.enabled', 'false') \ + .config('spark.hadoop.fs.s3a.impl', 'org.apache.hadoop.fs.s3a.S3AFileSystem') \ + .config('spark.hadoop.fs.s3a.path.style.access', 'true') \ + .config('spark.driver.extraJavaOptions', '-Dcom.amazonaws.sdk.disableCertChecking') \ + .config('spark.executor.extraJavaOptions', '-Dcom.amazonaws.sdk.disableCertChecking') \ + .enableHiveSupport() \ + .getOrCreate() +``` \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..5ca1afc --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,15 @@ +# Security Reporting Process + +Please, report any security issue to `opensourcegroup@netcracker.com` where the issue will be triaged appropriately. + +If you know of a publicly disclosed security vulnerability please IMMEDIATELY email `opensourcegroup@netcracker.com` +to inform the team about the vulnerability, so we may start the patch, release, and communication process. + +# Security Release Process + +If the vulnerability is found in the latest stable release, then it would be fixed in patch version for that release. +E.g., issue is found in 2.5.0 release, then 2.5.1 version with a fix will be released. +By default, older versions will not have security releases. + +If the issue doesn't affect any existing public releases, the fix for medium and high issues is performed +in a main branch before releasing a new version. For low priority issues the fix can be planned for future releases. \ No newline at end of file diff --git a/chart/helm/hivemetastore/.helmignore b/chart/helm/hivemetastore/.helmignore new file mode 100644 index 0000000..8f0966a --- /dev/null +++ b/chart/helm/hivemetastore/.helmignore @@ -0,0 +1,37 @@ +# Copyright 2024-2025 NetCracker Technology Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/chart/helm/hivemetastore/Chart.yaml b/chart/helm/hivemetastore/Chart.yaml new file mode 100644 index 0000000..81a8171 --- /dev/null +++ b/chart/helm/hivemetastore/Chart.yaml @@ -0,0 +1,38 @@ +# Copyright 2024-2025 NetCracker Technology Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: hive-metastore +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/chart/helm/hivemetastore/monitoring/hive-metastore-overview.json b/chart/helm/hivemetastore/monitoring/hive-metastore-overview.json new file mode 100644 index 0000000..ac7d03a --- /dev/null +++ b/chart/helm/hivemetastore/monitoring/hive-metastore-overview.json @@ -0,0 +1,2245 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "panels": [], + "title": "Hive-Metastore Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Health status of Hive-Metastore", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 2, + "text": "DOWN" + } + }, + "type": "value" + }, + { + "options": { + "from": -1000000, + "result": { + "color": "yellow", + "index": 0, + "text": "DEGRADED" + }, + "to": -0.00001 + }, + "type": "range" + }, + { + "options": { + "from": 0.01, + "result": { + "color": "green", + "index": 1, + "text": "UP" + }, + "to": 1000000 + }, + "type": "range" + } + ], + "min": 0, + "noValue": "DOWN", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(kube_deployment_status_replicas_available{cluster=\"$cluster\", namespace=~\"$namespace\", deployment=~\"hive-metastore\"})*( sum(kube_deployment_status_replicas_available{cluster=\"$cluster\", namespace=~\"$namespace\", deployment=~\"hive-metastore\"} OR on() vector(1)) - 0.5) ", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Hive-Metastore Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of Hive-Metastore running pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "color": "green", + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(kube_pod_status_phase{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"hive-metastore.*\", phase=\"Running\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active Replicas", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Shows the number of pods related to the hive-metastore in the namespace", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(kube_pod_info{cluster=\"$cluster\",exported_namespace=~\"$namespace\",exported_pod=~\"hive-metastore.*\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Pods Count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PC3E95692D54ABCC0" + }, + "description": "Shows pod state", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepBefore", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + { + "options": { + "0": { + "color": "dark-red", + "index": 0 + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "color": "green", + "index": 1 + }, + "to": 100 + }, + "type": "range" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Succeeded" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "transparent", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "transparent", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PC3E95692D54ABCC0" + }, + "editorMode": "code", + "expr": "sum(avg(kube_pod_status_phase{cluster=\"$cluster\",exported_namespace=~\"$namespace\",exported_pod=~\"hive-metastore.*\"}) by(exported_namespace, exported_pod, phase)) by(phase)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 2, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Shows CPU usage by pods in the namespace", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Millicores", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/limit/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\",container!=\"POD\", namespace=\"$namespace\",name!=\"\"}[1m])) by (pod)", + "instant": false, + "interval": "", + "legendFormat": "{{pod}} usage", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(sum(kube_pod_container_resource_limits_cpu_cores{namespace=\"$namespace\", cluster=\"$cluster\", container!=\"POD\"}) by (pod)) * 1", + "hide": false, + "instant": false, + "legendFormat": "{{pod}} limit", + "range": true, + "refId": "C" + } + ], + "title": "CPU Usage", + "type": "timeseries" + } + ], + "title": "CPU", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 11, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Shows memory usage by pods in the namespace", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/limit/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\",namespace=\"$namespace\", container!=\"POD\", name!=\"\", image!=\"\"}) by (pod)", + "instant": false, + "interval": "", + "legendFormat": "{{pod}}: usage", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", cluster=\"$cluster\", container!=\"POD\"}) by (pod)", + "hide": false, + "instant": false, + "legendFormat": "{{pod}}: limit", + "range": true, + "refId": "B" + } + ], + "title": "Memory Usage", + "type": "timeseries" + } + ], + "title": "Memory", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 23, + "panels": [], + "title": "JVM", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "metrics_heap_used{namespace=\"$namespace\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "JVM Heap Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": -1, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/peak/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "jvm_threads_current{namespace=\"$namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}: current", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "jvm_threads_peak{namespace=\"$namespace\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}: peak", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(jvm_threads_started_total{namespace=\"$namespace\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{pod}}: started/sec", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "Threads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PC3E95692D54ABCC0" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PC3E95692D54ABCC0" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "marksweepcompact_collectiontime{namespace=\"$namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "GC Time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 1, + "panels": [], + "title": "Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "hive-metastore - reads bytes" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "hive-metastore - written bytes" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum (rate(container_fs_reads_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\", container!~\"POD|\"}[5m])) by (namespace)", + "instant": false, + "legendFormat": "{{ namespace }} - reads bytes", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum (rate(container_fs_writes_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\", container!=\"\"}[5m])) by (namespace)", + "hide": false, + "instant": false, + "legendFormat": "{{ namespace }} - written bytes", + "range": true, + "refId": "B" + } + ], + "title": "Disk I/O utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PC3E95692D54ABCC0" + }, + "description": "Count of writes/reads completed", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "hive-metastore reading" + }, + "properties": [ + { + "id": "unit", + "value": "iops" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "hive-metastore writing" + }, + "properties": [ + { + "id": "unit", + "value": "iops" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PC3E95692D54ABCC0" + }, + "editorMode": "code", + "expr": "sum (rate(container_fs_reads_total{cluster=\"$cluster\",namespace=~\"$namespace\",container!~\"POD|\"}[5m])) by (namespace)", + "instant": false, + "legendFormat": "{{ namespace }} reading", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PC3E95692D54ABCC0" + }, + "editorMode": "code", + "expr": "sum (rate(container_fs_writes_total{cluster=\"$cluster\",namespace=~\"$namespace\",container!~\"POD|\"}[5m])) by (namespace)", + "hide": false, + "instant": false, + "legendFormat": "{{ namespace }} writing", + "range": true, + "refId": "B" + } + ], + "title": "Conteiner IOps", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "panels": [], + "title": "Network", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PC3E95692D54ABCC0" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "open_connections_count{namespace=\"$namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Open connection ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Shows network traffic in bytes per second for the pod", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "binBps" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", container!=\"POD\"}[1m])) by (pod)", + "instant": false, + "interval": "$interval", + "legendFormat": "receive: {{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", container!=\"POD\"}[1m])) by (pod)", + "hide": false, + "instant": false, + "interval": "$interval", + "legendFormat": "transmit: {{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "Receive/Transmit Bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Shows network packets for pod", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", container!=\"POD\"}[1m])) by (pod)", + "instant": false, + "interval": "$interval", + "legendFormat": "received: {{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", container!=\"POD\"}[1m])) by (pod) ", + "hide": false, + "instant": false, + "interval": "$interval", + "legendFormat": "transmitted: {{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "Rate of Received/Transmitted Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Shows dropped packets for pod", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit:/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", container!=\"POD\"}[1m])) by (pod)", + "instant": false, + "interval": "$interval", + "legendFormat": "receive: {{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", container!=\"POD\"}[1m])) by (pod)", + "hide": false, + "instant": false, + "interval": "$interval", + "legendFormat": "transmit: {{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "Rate of Received/Transmitted Packets Dropped", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 3, + "panels": [], + "title": "Data", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 73 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "count_db{namespace=\"$namespace\", pod=\"$pod\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "Count DB", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "delete_total_count_db{namespace=\"$namespace\", pod=\"$pod\"}", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "Delete DB", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "hive-metastore-dashboard", + "value": "hive-metastore-dashboard" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_pod_info{cluster=\"$cluster\"},exported_namespace)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_pod_info{cluster=\"$cluster\"},exported_namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(up,cluster)", + "hide": 0, + "includeAll": false, + "label": "", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(up,cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "type": "query" + }, + { + "auto": true, + "auto_count": 30, + "auto_min": "60s", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "label": "", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": false, + "text": "hive-metastore-6f4c449967-dngnw", + "value": "hive-metastore-6f4c449967-dngnw" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"},exported_pod)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "pod", + "options": [], + "query": { + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"},exported_pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "hive-metastore.*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Hive-metastore-overview", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/chart/helm/hivemetastore/templates/_helpers.tpl b/chart/helm/hivemetastore/templates/_helpers.tpl new file mode 100644 index 0000000..2171b47 --- /dev/null +++ b/chart/helm/hivemetastore/templates/_helpers.tpl @@ -0,0 +1,199 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "hive-metastore.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "hive-metastore.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "hive-metastore.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "hive-metastore.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "hive-metastore.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{ define "hivemetastore_image" -}} +{{ printf "%s:%v" (.Values.image.repository) (.Values.image.tag) }} +{{- end }} + +{{/* +MinIO S3 Endpoint +*/}} +{{- define "s3.endpoint" -}} + {{- .Values.s3.endpoint -}} +{{- end -}} + +{{/* +MinIO S3 accesskey +*/}} +{{- define "s3.accessKey" -}} + {{- .Values.s3.accessKey -}} +{{- end -}} + +{{/* +MinIO S3 secretkey +*/}} +{{- define "s3.secretKey" -}} + {{- .Values.s3.secretKey -}} +{{- end -}} + +{{/* +Postgres Host +*/}} +{{- define "postgres.host" -}} + {{- .Values.postgres.host -}} +{{- end -}} + +{{/* +Postgres Port +*/}} +{{- define "postgres.port" -}} + {{- .Values.postgres.port -}} +{{- end -}} + +{{/* +Postgres Admin User +*/}} +{{- define "postgres.adminUser" -}} + {{- .Values.postgres.adminUser -}} +{{- end -}} + +{{/* +Postgres Admin Password +*/}} +{{- define "postgres.adminPassword" -}} + {{- .Values.postgres.adminPassword -}} +{{- end -}} + + +{{/* +Hive User +*/}} +{{- define "postgres.hive.user" -}} + {{- .Values.hive.user -}} +{{- end -}} + +{{/* +Hive Password +*/}} +{{- define "postgres.hive.password" -}} + {{- .Values.hive.password -}} +{{- end -}} + +{{/* + Postgres Connection URL + */}} + {{- define "postgres.psql.connection.url" -}} + {{- if and .Values.tls.enabled .Values.postgres.psqlParams -}} + {{ printf "postgresql://%s:%s@%s:%s?%s" (include "postgres.adminUser" .) (include "postgres.adminPassword" .) (include "postgres.host" .) (include "postgres.port" .) ( .Values.postgres.psqlParams ) }} + {{- else -}} + {{ printf "postgresql://%s:%s@%s:%s" (include "postgres.adminUser" .) (include "postgres.adminPassword" .) (include "postgres.host" .) (include "postgres.port" .)}} + {{- end -}} +{{- end -}} + +{{- define "jdbcParams" -}} +{{- if .Values.postgres.jdbcParams -}} +{{- $params := default "" (.Values.postgres.jdbcParams | replace "," .separator) -}} +{{ printf "?%s" $params }} +{{- end -}} +{{- end -}} + +{{/* +Selector labels for cloud release +*/}} +{{- define "selector_labels" -}} +app.kubernetes.io/instance: {{ cat .Release.Name "-" .Release.Namespace | nospace | trunc 63 | trimSuffix "-" }} +app.kubernetes.io/name: {{ include "hive-metastore.name" . }} +{{- end }} + +{{/* +Deployment only labels for cloud release +*/}} +{{- define "deployment_only_labels" -}} +app.kubernetes.io/instance: {{ cat .Release.Name "-" .Release.Namespace | nospace | trunc 63 | trimSuffix "-" }} +app.kubernetes.io/component: hivemetastore +app.kubernetes.io/version: {{ splitList ":" ( include "hivemetastore_image" . ) | last }} +app.kubernetes.io/technology: java-others +{{- end }} + + +{{/* +Deployment and service only labels for cloud release +*/}} +{{- define "deployment_and_service_only_labels" -}} +name: {{ include "hive-metastore.name" . }} +app.kubernetes.io/name: {{ include "hive-metastore.name" . }} +{{- end }} + +{{/* +All object labels for cloud release +*/}} +{{- define "all_objects_labels" -}} +app.kubernetes.io/part-of: hivemetastore +{{- end }} + +{{/* +Processed by grafana operator label for cloud release +*/}} +{{- define "grafana_operator_label" -}} +app.kubernetes.io/processed-by-operator: grafana-operator +{{- end }} + +{{/* +Processed by prometheus operator label for cloud release +*/}} +{{- define "prometheus_operator_label" -}} +app.kubernetes.io/processed-by-operator: prometheus-operator +{{- end }} + +{{/* +Processed by cert-manager label for cloud release +*/}} +{{- define "cert_manager_label" -}} +app.kubernetes.io/processed-by-operator: cert-manager +{{- end }} \ No newline at end of file diff --git a/chart/helm/hivemetastore/templates/configmap.yaml b/chart/helm/hivemetastore/templates/configmap.yaml new file mode 100644 index 0000000..b498b46 --- /dev/null +++ b/chart/helm/hivemetastore/templates/configmap.yaml @@ -0,0 +1,26 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +kind: ConfigMap +apiVersion: v1 +metadata: + name: metastore-cfg + namespace: {{ .Release.Namespace }} + labels: + {{- include "all_objects_labels" . | nindent 4 }} +data: + metastore-site.xml: |- + {{ tpl .Values.metastoreConfig.metastoreSiteProperties . | nindent 4 }} diff --git a/chart/helm/hivemetastore/templates/configmap_jmx.yaml b/chart/helm/hivemetastore/templates/configmap_jmx.yaml new file mode 100644 index 0000000..c0301c0 --- /dev/null +++ b/chart/helm/hivemetastore/templates/configmap_jmx.yaml @@ -0,0 +1,26 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +kind: ConfigMap +apiVersion: v1 +metadata: + name: jmx-cfg + labels: + {{- include "all_objects_labels" . | nindent 4 }} +data: + jmx-exporter-config.yaml: |- + {{ tpl .Values.jmxConfig.jmxExporterConfig . | nindent 4 }} + diff --git a/chart/helm/hivemetastore/templates/credentials-secret.yaml b/chart/helm/hivemetastore/templates/credentials-secret.yaml new file mode 100644 index 0000000..e9bf52c --- /dev/null +++ b/chart/helm/hivemetastore/templates/credentials-secret.yaml @@ -0,0 +1,30 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-credentials-secret + namespace: {{ .Release.Namespace }} + labels: + {{- include "all_objects_labels" . | nindent 4 }} +data: + pg_password: {{ include "postgres.adminPassword" . | b64enc }} + pg_username: {{ include "postgres.adminUser" . | b64enc }} + hive_password: {{ include "postgres.hive.password" . | b64enc }} + hive_username: {{ include "postgres.hive.user" . | b64enc }} + s3_accessKey: {{ include "s3.accessKey" . | b64enc }} + s3_secretKey: {{ include "s3.secretKey" . | b64enc }} \ No newline at end of file diff --git a/chart/helm/hivemetastore/templates/deployment.yaml b/chart/helm/hivemetastore/templates/deployment.yaml new file mode 100644 index 0000000..9ff68ae --- /dev/null +++ b/chart/helm/hivemetastore/templates/deployment.yaml @@ -0,0 +1,177 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hive-metastore.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "all_objects_labels" . | nindent 4 }} + {{- include "deployment_and_service_only_labels" . | nindent 4 }} + {{- include "deployment_only_labels" . | nindent 4 }} + {{- with .Values.labels }} + {{ toYaml . | indent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "selector_labels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "all_objects_labels" . | nindent 8 }} + {{- include "deployment_and_service_only_labels" . | nindent 8 }} + {{- include "deployment_only_labels" . | nindent 8 }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + automountServiceAccountToken: false + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + image: {{ template "hivemetastore_image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + env: + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- end }} + - name: "HIVE_SERVICE_PORT" + value: "{{ .Values.service.port }}" + - name: "PG_USERNAME" + valueFrom: + secretKeyRef: + key: pg_username + name: {{ .Release.Name }}-credentials-secret + - name: "PG_PASSWORD" + valueFrom: + secretKeyRef: + key: pg_password + name: {{ .Release.Name }}-credentials-secret + - name: "HIVE_USERNAME" + valueFrom: + secretKeyRef: + key: hive_username + name: {{ .Release.Name }}-credentials-secret + - name: "HIVE_PASSWORD" + valueFrom: + secretKeyRef: + key: hive_password + name: {{ .Release.Name }}-credentials-secret + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + - name: metrics + containerPort: 9028 + protocol: TCP + volumeMounts: + - name: log4j2-properties + mountPath: /opt/hive-metastore/conf/metastore-log4j2.properties + subPath: log4j2.properties + - name: metastore-cfg-vol + mountPath: /opt/hive-metastore/conf/metastore-site.xml + subPath: metastore-site.xml + - name: metastore-cfg-secret + mountPath: /opt/hive-metastore/conf/hivemetastore-site.xml + subPath: hivemetastore-site.xml + - name: metastore-cfg-secret + mountPath: /opt/hadoop/etc/hadoop/core-site.xml + subPath: core-site.xml + {{- range .Values.secretMounts }} + - name: {{ .name }} + mountPath: {{ .path }} + subPath: {{ .subPath }} + {{- end }} + - name: jmx-cfg-vol + mountPath: /opt/hive-metastore/conf/jmx-exporter-config.yaml + subPath: jmx-exporter-config.yaml + {{- if .Values.extraVolumeMounts }} + {{- tpl (toYaml .Values.extraVolumeMounts) . | nindent 12 }} + {{- end }} + readinessProbe: + exec: + command: + - /bin/sh + - -c + - |- + /opt/hive-metastore/bin/schematool -verbose -info -dbType postgres -userName $PG_USERNAME -passWord $PG_PASSWORD -url jdbc:postgresql://{{ include "postgres.host" . }}:{{ include "postgres.port" . }}/{{ .Values.hive.db }} {{ if and .Values.tls.enabled .Values.postgres.jdbcParams }} -dbOpts {{ .Values.postgres.jdbcParams }} {{ end }} -driver {{ .Values.postgres.driver }} -userName $HIVE_USERNAME -passWord $HIVE_PASSWORD + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds | default 15 }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds | default 15 }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds | default 30 }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold | default 6 }} + successThreshold: {{ .Values.readinessProbe.successThreshold | default 1 }} + livenessProbe: + tcpSocket: + port: 9028 + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds | default 30 }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds | default 60 }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds | default 10 }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold | default 6 }} + successThreshold: {{ .Values.livenessProbe.successThreshold | default 1 }} + resources: + {{- toYaml .Values.hiveMetastore.resources | nindent 12 }} + volumes: + - name: log4j2-properties + configMap: + name: log4j2-properties + - name: metastore-cfg-vol + configMap: + name: metastore-cfg + - name: metastore-cfg-secret + secret: + secretName: metastore-cfg-secret + items: + - key: hivemetastore-site.xml + path: hivemetastore-site.xml + - key: core-site.xml + path: core-site.xml + {{- range .Values.secretMounts }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + {{- end }} + - name: jmx-cfg-vol + configMap: + name: jmx-cfg + {{- if .Values.extraVolumes }} + {{- tpl (toYaml .Values.extraVolumes) . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/chart/helm/hivemetastore/templates/extra-secrets.yaml b/chart/helm/hivemetastore/templates/extra-secrets.yaml new file mode 100644 index 0000000..5d89f3f --- /dev/null +++ b/chart/helm/hivemetastore/templates/extra-secrets.yaml @@ -0,0 +1,59 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +################################################# +## Extra Secrets provisioned via the chart values, reused from Helm Chart for Apache Airflow +################################################# +{{- $Global := . }} +{{- range $secretName, $secretContent := .Values.extraSecrets }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ tpl $secretName $Global | quote }} + labels: + release: {{ $Global.Release.Name }} + chart: "{{ $Global.Chart.Name }}-{{ $Global.Chart.Version }}" + heritage: {{ $Global.Release.Service }} + {{- with $Global.Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if $secretContent.labels }} + {{- toYaml $secretContent.labels | nindent 4 }} + {{- end }} + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-delete-policy": "before-hook-creation" + "helm.sh/hook-weight": "-11" + {{- if $secretContent.annotations }} + {{- toYaml $secretContent.annotations | nindent 4 }} + {{- end }} +{{- if $secretContent.type }} +type: {{ $secretContent.type }} +{{- end }} +{{- if $secretContent.data }} +data: + {{- with $secretContent.data }} + {{- tpl . $Global | nindent 2 }} + {{- end }} +{{- end }} +{{- if $secretContent.stringData }} +stringData: + {{- with $secretContent.stringData }} + {{- tpl . $Global | nindent 2 }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/chart/helm/hivemetastore/templates/hive-create-db-job.yaml b/chart/helm/hivemetastore/templates/hive-create-db-job.yaml new file mode 100644 index 0000000..766e9ee --- /dev/null +++ b/chart/helm/hivemetastore/templates/hive-create-db-job.yaml @@ -0,0 +1,229 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- if .Values.hiveInitJob.enabled -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: hive-create-db + namespace: {{ .Release.Namespace }} + labels: + {{- include "all_objects_labels" . | nindent 4 }} + annotations: + {{- toYaml .Values.hiveInitJob.initAnnotations | nindent 4 }} +spec: + template: + metadata: + labels: + {{- include "all_objects_labels" . | nindent 8 }} + spec: + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.extraVolumes }} + {{- tpl (toYaml .Values.extraVolumes) . | nindent 8 }} + {{- end }} + {{- range .Values.secretMounts}} + - name: {{.name}} + secret: + secretName: {{.secretName}} + {{- end}} + containers: + - name: hivemeta + image: {{ template "hivemetastore_image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + command: ["/bin/bash"] + resources: {{- toYaml .Values.hiveInitJob.resources | nindent 12 }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{- tpl (toYaml .Values.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- range .Values.secretMounts}} + - name: {{.name}} + mountPath: {{.path}} + subPath: {{.subPath}} + {{- end }} + + args: + - "-c" + - |- + + {{ if .Values.tls.enabled }} + if [[ "$(ls ${TRUST_CERTS_DIR})" ]]; then + for filename in ${TRUST_CERTS_DIR}/*; do + echo "Import $filename certificate to Java cacerts" + ${JAVA_HOME}/bin/keytool -import -trustcacerts -keystore ${JAVA_HOME}/lib/security/cacerts -storepass changeit -noprompt -alias ${filename} -file ${filename} + done; + fi + {{- end }} + + {{ if .Values.hiveInitJob.cleanupDB }} + + echo -e "-----Clean Up DB-------" + + OUTPUT=$(psql {{ include "postgres.psql.connection.url" . }} -c "drop database if exists {{ .Values.hive.db }} with (force);" 2>&1) + EXIT_CODE=$? + echo "Command output "$OUTPUT"" + + if [ $EXIT_CODE -ne 0 ] + then + echo "ERROR: Could not drop the database in PG." + exit 1 + else + if [[ "$OUTPUT" == *"DROP DATABASE"* ]] + then + echo "Hive database has been successfully dropped" + fi + fi + + {{ end }} + + echo -e "\n-----Creating Hive user in PG-----" + + CREATE_USER_OUTPUT=$(psql {{ include "postgres.psql.connection.url" . }} -c "create user {{ include "postgres.hive.user" . }} with encrypted password '{{ include "postgres.hive.password" . }}';" 2>&1) + EXIT_CODE=$? + echo "Command output $CREATE_USER_OUTPUT" + + if [ $EXIT_CODE -ne 0 ] && [[ "$CREATE_USER_OUTPUT" != *"role"*"already exists"* ]] + then + echo "ERROR: Could not create user in PG." + exit 1 + else + if [[ "$CREATE_USER_OUTPUT" == *"role"*"already exists"* ]] + then + echo "Hive user already exists." + else + if [[ "$CREATE_USER_OUTPUT" == *"CREATE ROLE"* ]] + then + echo "Hive user has been successfully created" + fi + fi + fi + + echo -e " \n-----Grant user in schema-----" + + GRANT_USER=$(psql {{ include "postgres.psql.connection.url" . }} -c "grant usage on schema pg_catalog to {{ include "postgres.hive.user" . }};" 2>&1) + EXIT_CODE=$? + echo "Command output $GRANT_USER" + + echo -e " \n-----Creating Hive DB in PG-----" + + CREATE_DB_OUTPUT=$(psql {{ include "postgres.psql.connection.url" . }} -c "create database {{ .Values.hive.db }} owner {{ include "postgres.hive.user" . }};" 2>&1) + EXIT_CODE=$? + echo "Command output $CREATE_DB_OUTPUT" + + if [ $EXIT_CODE -ne 0 ] && [[ "$CREATE_DB_OUTPUT" != *"database"*"already exists"* ]] + then + echo "ERROR: Could not create hive database in PG." + printf %s "$CREATE_DB_OUTPUT" + exit 1 + else + if [[ "$CREATE_DB_OUTPUT" == *"database"*"already exists"* ]] + then + echo "Hive database already exists." + else + if [[ "$CREATE_DB_OUTPUT" == *"CREATE DATABASE"* ]] + then + echo "Hive database has been successfully created" + fi + fi + fi + + + echo -e "\n-----Permissions Hive DB in PG-----" + + PERMISSIONS_DB_OUTPUT=$(psql {{ include "postgres.psql.connection.url" . }} -c "grant all privileges on database {{ .Values.hive.db }} to {{ include "postgres.hive.user" . }};" 2>&1) + EXIT_CODE=$? + echo -e "\n-----Permissions Hive DB updated-----" + + echo -e "\n-----Grants Hive DB in PG-----" + + GRANTS_DB_OUTPUT=$(psql {{ include "postgres.psql.connection.url" . }} -c "alter database {{ .Values.hive.db }} owner to {{ include "postgres.hive.user" . }};" 2>&1) + EXIT_CODE=$? + echo -e "\n-----Grants Hive DB updated-----" + + {{ if .Values.hiveInitJob.upgradeSchema }} + + echo -e " \n----- Updating Hive Schema-----" + + HIVE_UPDATE_ERROR=$(/opt/hive-metastore/bin/schematool -verbose -upgradeSchema -dbType postgres -userName {{ include "postgres.hive.user" . }} -passWord {{ include "postgres.hive.password" . }} -url 'jdbc:postgresql://{{ include "postgres.host" . }}:{{ include "postgres.port" . }}/{{ .Values.hive.db }}{{ include "jdbcParams" (dict "Values" .Values "separator" "&")}}' -driver {{ .Values.postgres.driver }} 2>&1 > /dev/null) + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ] + then + echo "ERROR: Hive schema init failed." + printf %s "$HIVE_INIT_ERROR" + exit 1 + else + echo "Hive schema has been successfully updated" + fi + + {{ else }} + + echo -e " \n-----Initializing Hive Schema-----" + + HIVE_INIT_ERROR=$(/opt/hive-metastore/bin/schematool -verbose -initSchema -dbType postgres -userName {{ include "postgres.hive.user" . }} -passWord {{ include "postgres.hive.password" . }} -url 'jdbc:postgresql://{{ include "postgres.host" . }}:{{ include "postgres.port" . }}/{{ .Values.hive.db }}{{ include "jdbcParams" (dict "Values" .Values "separator" "&")}}' -driver {{ .Values.postgres.driver }} 2>&1 > /dev/null) + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ] + then + if [[ "$HIVE_INIT_ERROR" =~ .*"already exists".* ]] + then + echo "Hive schema is already initialized." + else + echo "ERROR: Hive schema init failed." + printf %s "$HIVE_INIT_ERROR" + exit 1 + fi + else + echo "Hive schema has been successfully initialized" + fi + + {{ end }} + + echo -e " \n-----Validating Hive Schema-----" + + HIVE_VALID_ERROR=$(/opt/hive-metastore/bin/schematool -verbose -validate -dbType postgres -url 'jdbc:postgresql://{{ include "postgres.host" . }}:{{ include "postgres.port" . }}/{{ .Values.hive.db }}{{ include "jdbcParams" (dict "Values" .Values "separator" "&")}}' -driver {{ .Values.postgres.driver }} -userName {{ include "postgres.hive.user" . }} -passWord {{ include "postgres.hive.password" . }} 2>&1) + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ] + then + echo "ERROR: Hive schema validation failed." + printf %s "$HIVE_VALID_ERROR" + if [[ "$HIVE_VALID_ERROR" =~ .*"Failed in schema version validation".* ]] + then + echo -e " \n-----Upgrading Hive Schema-----" + HIVE_UPGRADE_SCHEMA=$(/opt/hive-metastore/bin/schematool -verbose -upgradeSchema -dbType postgres -url 'jdbc:postgresql://{{ include "postgres.host" . }}:{{ include "postgres.port" . }}/{{ .Values.hive.db }}{{ include "jdbcParams" (dict "Values" .Values "separator" "&")}}' -driver {{ .Values.postgres.driver }} -userName {{ include "postgres.hive.user" . }} -passWord {{ include "postgres.hive.password" . }} 2>&1 > /dev/null) + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ] + then + echo "ERROR: Hive schema upgrade failed." + printf %s "$HIVE_UPGRADE_SCHEMA" + exit 1 + else + echo "Hive Schema successfully upgraded." + fi + else + echo "ERROR: Hive schema validation failed." + printf %s "$HIVE_VALID_ERROR" + exit 1 + fi + else + echo "Hive schema is valid" + fi + + restartPolicy: Never + backoffLimit: 5 +{{- end }} diff --git a/chart/helm/hivemetastore/templates/init-job-credentials-secret.yaml b/chart/helm/hivemetastore/templates/init-job-credentials-secret.yaml new file mode 100644 index 0000000..ff04297 --- /dev/null +++ b/chart/helm/hivemetastore/templates/init-job-credentials-secret.yaml @@ -0,0 +1,30 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- if .Values.s3InitJob.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-init-job-credentials + namespace: {{ .Release.Namespace }} + labels: + {{- include "all_objects_labels" . | nindent 4 }} + annotations: + {{- toYaml .Values.initJobCredentialsSecret.annotations | nindent 4}} +data: + s3_accessKey: {{ include "s3.accessKey" . | b64enc }} + s3_secretKey: {{ include "s3.secretKey" . | b64enc }} +{{- end }} \ No newline at end of file diff --git a/chart/helm/hivemetastore/templates/jks-secret.yaml b/chart/helm/hivemetastore/templates/jks-secret.yaml new file mode 100644 index 0000000..8a08043 --- /dev/null +++ b/chart/helm/hivemetastore/templates/jks-secret.yaml @@ -0,0 +1,32 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- if and .Values.tls.enabled .Values.tls.generateCerts.enabled .Values.tls.generateCerts.keystores.jks.create }} +apiVersion: v1 +kind: Secret +metadata: + name: jks-password-secret + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-delete-policy": "before-hook-creation" + "helm.sh/hook-weight": "-11" + labels: + {{- include "all_objects_labels" . | nindent 4 }} +type: Opaque +data: + password-key: {{ tpl .Values.tls.certificates.jks_key . | b64enc }} +{{- end }} \ No newline at end of file diff --git a/chart/helm/hivemetastore/templates/log4j2-properties.yaml b/chart/helm/hivemetastore/templates/log4j2-properties.yaml new file mode 100644 index 0000000..4f58876 --- /dev/null +++ b/chart/helm/hivemetastore/templates/log4j2-properties.yaml @@ -0,0 +1,30 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: log4j2-properties + namespace: {{ .Release.Namespace }} + labels: + {{- include "all_objects_labels" . | nindent 4 }} + app: {{ .Chart.Name }} + release: {{ .Release.Name }} +data: + log4j2.properties: | #log4j properties + {{- if .Values.log4j2Properties }} + {{- .Values.log4j2Properties | nindent 4 }} + {{- end }} diff --git a/chart/helm/hivemetastore/templates/qbmonitoring/hive_metastore_overview.yaml b/chart/helm/hivemetastore/templates/qbmonitoring/hive_metastore_overview.yaml new file mode 100644 index 0000000..b0578d5 --- /dev/null +++ b/chart/helm/hivemetastore/templates/qbmonitoring/hive_metastore_overview.yaml @@ -0,0 +1,30 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- if .Values.monitoring.enabled }} +apiVersion: integreatly.org/v1alpha1 +kind: GrafanaDashboard +metadata: + name: hive-metastore-overview.json + labels: + app: grafana + {{- include "all_objects_labels" . | nindent 4 }} + {{- include "grafana_operator_label" . | nindent 4 }} +spec: + name: hive-metastore-overview.json + json: > +{{ .Files.Get "monitoring/hive-metastore-overview.json" | indent 4 }} +{{ end }} diff --git a/chart/helm/hivemetastore/templates/qbmonitoring/prometeus-rule.yaml b/chart/helm/hivemetastore/templates/qbmonitoring/prometeus-rule.yaml new file mode 100644 index 0000000..0796241 --- /dev/null +++ b/chart/helm/hivemetastore/templates/qbmonitoring/prometeus-rule.yaml @@ -0,0 +1,68 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{ if .Values.prometheusRules.alert.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/component: monitoring + prometheus: hive-metastore-rules + role: alert-rules + {{- include "all_objects_labels" . | nindent 4 }} + {{- include "prometheus_operator_label" . | nindent 4 }} + name: hive-metastore-prometheus-rules +spec: + groups: + - name: {{ .Release.Namespace }} + rules: + - alert: HiveMetastoreCPUusage + annotations: + summary: Some of Hive Metastore pods CPU load is higher than specified threshold + description: 'Hive Metastore CPU load is higher than {{ default 90 .Values.prometheusRules.alert.cpuThreshold }} percent on {{ .Release.Namespace }}' + expr: max(rate(container_cpu_usage_seconds_total{image!="", namespace="{{ .Release.Namespace }}", container!="POD", pod=~"hive-metastore.*"}[1m])) / max(kube_pod_container_resource_limits_cpu_cores{namespace="{{ .Release.Namespace }}", container="hive-metastore"}) * 100 > {{ default 90 .Values.prometheusRules.alert.cpuThreshold }} + labels: + severity: warning + namespace: {{ .Release.Namespace }} + service: {{ .Release.Name }} + - alert: HiveMetastoreMemoryUsage + annotations: + summary: Some of Hive Metastore pods memory usage is higher than specified threshold + description: 'Hive Metastore memory usage is higher than {{ default 90 .Values.prometheusRules.alert.memoryThreshold }} percent on {{ .Release.Namespace }}' + expr: max(container_memory_working_set_bytes{image!="", namespace="{{ .Release.Namespace }}", container!="POD", pod=~"hive-metastore.*"}) / max(kube_pod_container_resource_limits_memory_bytes{namespace="{{ .Release.Namespace }}", container="hive-metastore"}) * 100 > {{ default 90 .Values.prometheusRules.alert.memoryThreshold }} + labels: + severity: warning + namespace: {{ .Release.Namespace }} + service: {{ .Release.Name }} + - alert: HiveMetastoreIsDegraded + annotations: + summary: Hive Metastore Is Degraded + description: 'Some of Hive Metastore pods went down on {{ .Release.Namespace }}' + expr: kube_replicaset_status_ready_replicas{namespace="{{ .Release.Namespace }}", replicaset=~"hive-metastore.*"} < kube_replicaset_spec_replicas{namespace="{{ .Release.Namespace }}", replicaset=~"hive-metastore.*"} + labels: + severity: high + namespace: {{ .Release.Namespace }} + service: {{ .Release.Name }} + - alert: HiveMetastoreIsDown + annotations: + summary: Hive Metastore is down + description: 'Hive Metastore is down on {{ .Release.Namespace }}' + expr: absent(kube_pod_status_phase{namespace="{{ .Release.Namespace }}", exported_pod=~"hive-metastore.*", pod=~"hive-metastore.*", phase="Running"} == 1) + labels: + severity: critical + namespace: {{ .Release.Namespace }} + service: {{ .Release.Name }} +{{ end }} \ No newline at end of file diff --git a/chart/helm/hivemetastore/templates/s3-init-job.yaml b/chart/helm/hivemetastore/templates/s3-init-job.yaml new file mode 100644 index 0000000..74c39b0 --- /dev/null +++ b/chart/helm/hivemetastore/templates/s3-init-job.yaml @@ -0,0 +1,205 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- if .Values.s3InitJob.enabled -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: hive-s3-init + namespace: {{ .Release.Namespace }} + labels: + {{- include "all_objects_labels" . | nindent 4 }} + annotations: + {{- toYaml .Values.s3InitJob.initAnnotations | nindent 4 }} +spec: + template: + metadata: + labels: + {{- include "all_objects_labels" . | nindent 8 }} + spec: + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.extraVolumes }} + {{- tpl (toYaml .Values.extraVolumes) . | nindent 8 }} + {{- end }} + {{- range .Values.secretMounts}} + - name: {{.name}} + secret: + secretName: {{.secretName}} + {{- end }} + containers: + - name: s3init + image: {{ template "hivemetastore_image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + command: ["/bin/bash"] + resources: {{- toYaml .Values.s3InitJob.resources | nindent 12 }} + env: + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 10}} + {{- end }} + - name: "ACCESS_KEY" + valueFrom: + secretKeyRef: + key: s3_accessKey + name: {{ .Release.Name }}-init-job-credentials + - name: "SECRET_KEY" + valueFrom: + secretKeyRef: + key: s3_secretKey + name: {{ .Release.Name }}-init-job-credentials + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{- tpl (toYaml .Values.extraVolumeMounts) . | nindent 12 }} + {{- end }} + {{- range .Values.secretMounts}} + - name: {{.name}} + mountPath: {{.path}} + subPath: {{.subPath}} + {{- end }} + + args: + - "-c" + - |- + s3_url="{{ include "s3.endpoint" . }}" + s3_uri="{{ .Values.s3.warehouseDir }}" + aws_sigv4="{{ .Values.s3InitJob.awsSigV4 }}" + curlOpts="--retry 5" + + echo "s3_url: $s3_url" + echo "s3_uri: $s3_uri" + + {{- if .Values.s3InitJob.disableTLSValidation }} + curlOpts="${curlOpts} --insecure" + echo "TLS certificate validation is disabled." + {{- end }} + + # remove last '/' + s3_uri=$(echo "$s3_uri" | sed 's/\/$//') + + # get bucket dir from S3 URI + bucket_dir=$(echo "$s3_uri" | cut -d/ -f4-) + + # get bucket name from S3 URI + bucket_name=$(echo "$s3_uri" | cut -d/ -f3) + + get_http_code_from_response() { + response="$1" + status_code=$(echo "$response" | grep -E "^HTTP/[0-9\.]*\s[0-9]{3}(\s[a-zA-Z]*)?" | grep -oE "[0-9]{3}") + echo "${status_code}" + } + + head_bucket() { + response=$(curl -v -s -I ${curlOpts} --head ${s3_url}/${bucket_name} \ + -H "Date: $(date --utc +'%Y%m%dT%H%M%SZ')" \ + --user ${ACCESS_KEY}:${SECRET_KEY} \ + --aws-sigv4 "${aws_sigv4}" 2>&1) + + echo "${response}" + } + + create_bucket() { + response=$(curl -v -s -I ${curlOpts} -X PUT \ + -H "Date: $(date --utc +'%Y%m%dT%H%M%SZ')" \ + -H "Content-Length: 0" \ + --user ${ACCESS_KEY}:${SECRET_KEY} \ + --aws-sigv4 "${aws_sigv4}" \ + ${s3_url}/${bucket_name} 2>&1) + + echo "${response}" + } + + put_file_to_bucket_dir() { + s3_path="" + mock_file="mock.txt" + + if [[ ${bucket_dir} != "" ]]; then + s3_path=${bucket_name}/${bucket_dir}/${mock_file} + else + s3_path=${bucket_name}/${mock_file} + fi + + response=$(curl -v -s -I ${curlOpts} -X PUT \ + -H "Date: $(date --utc +'%Y%m%dT%H%M%SZ')" \ + -H "Content-Type: text/plain" \ + --user ${ACCESS_KEY}:${SECRET_KEY} \ + --aws-sigv4 "${aws_sigv4}" \ + ${s3_url}/${s3_path} 2>&1) + + echo "${response}" + } + + echo -e "\n\n\n\n-----Check if bucket '${bucket_name}' exists-------" + head_response=$(head_bucket) + head_http_status_code="$(get_http_code_from_response "$head_response")" + echo "head_http_status: $head_http_status_code" + + if [[ "$head_http_status_code" = 200 ]]; then + echo "Bucket '$bucket_name' already exists." + echo -e "\n\n\n\n------Create\Update bucket path '$bucket_dir' and upload mock file-------" + put_response=$(put_file_to_bucket_dir) + put_http_status_code="$(get_http_code_from_response "$put_response")" + echo "put_http_status: $put_http_status_code" + + if [[ "$put_http_status_code" = 200 ]]; then + echo "Bucket path '$bucket_dir' has been successfully created\updated with mock file." + else + echo "Could not create\update a bucket path '$bucket_dir' with mock file." + echo "$put_response" + exit 1 + fi + + else + if [[ "$head_http_status_code" = 404 ]]; then + echo "Bucket '$bucket_name' does not exist" + echo -e "\n\n\n\n------Create bucket '$bucket_name'-------" + create_response=$(create_bucket) + create_http_status_code="$(get_http_code_from_response "$create_response")" + echo "create_http_status: $create_http_status_code" + + if [[ "$create_http_status_code" = 200 ]]; then + echo "Bucket '$bucket_name' has been successfully created." + else + echo "Could not create a bucket." + echo "$create_response" + exit 1 + fi + + echo -e "\n\n\n\n------Create bucket path '$bucket_dir' upload mock file-------" + put_response=$(put_file_to_bucket_dir) + put_http_status_code="$(get_http_code_from_response "$put_response")" + echo "put_http_status: $put_http_status_code" + + if [[ "$put_http_status_code" = 200 ]]; then + echo "Bucket path has been successfully created with mock file." + else + echo "Could not create a bucket path with mock file." + echo "$put_response" + exit 1 + fi + + else + echo "Could not check if bucket '$bucket_name' exists". + echo "$head_response" + exit 1 + fi + fi + + restartPolicy: Never + backoffLimit: 5 +{{- end }} diff --git a/chart/helm/hivemetastore/templates/secret.yaml b/chart/helm/hivemetastore/templates/secret.yaml new file mode 100644 index 0000000..77e2557 --- /dev/null +++ b/chart/helm/hivemetastore/templates/secret.yaml @@ -0,0 +1,29 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +apiVersion: v1 +kind: Secret +metadata: + name: metastore-cfg-secret + namespace: {{ .Release.Namespace }} + labels: + {{- include "all_objects_labels" . | nindent 4 }} +type: Opaque +data: + hivemetastore-site.xml: |- + {{ tpl .Values.metastoreConfigsecret.metastoreSitePropertiesSecret . | b64enc }} + core-site.xml: |- + {{ tpl .Values.metastoreConfigsecret.coreSitePropertiesSecret . | b64enc }} \ No newline at end of file diff --git a/chart/helm/hivemetastore/templates/service-monitor.yaml b/chart/helm/hivemetastore/templates/service-monitor.yaml new file mode 100644 index 0000000..4745dd7 --- /dev/null +++ b/chart/helm/hivemetastore/templates/service-monitor.yaml @@ -0,0 +1,40 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- if .Values.monitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: hive-metastore-monitor + labels: + app.kubernetes.io/component: monitoring + app: hive-metastore + release: prometheus + {{- include "all_objects_labels" . | nindent 4 }} + {{- include "prometheus_operator_label" . | nindent 4 }} +spec: + endpoints: + - honorLabels: false + interval: {{ .Values.monitoring.interval }} + path: /metrics + scrapeTimeout: 30s + targetPort: 9028 + jobLabel: hive-metastore + selector: + matchLabels: + {{- include "all_objects_labels" . | nindent 6 }} + {{- include "deployment_and_service_only_labels" . | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/chart/helm/hivemetastore/templates/service.yaml b/chart/helm/hivemetastore/templates/service.yaml new file mode 100644 index 0000000..0fc99f9 --- /dev/null +++ b/chart/helm/hivemetastore/templates/service.yaml @@ -0,0 +1,39 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hive-metastore.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "all_objects_labels" . | nindent 4 }} + {{- include "deployment_and_service_only_labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + {{- if .Values.monitoring.enabled }} + - port: 9028 + targetPort: 9028 + protocol: TCP + name: metrics + {{- end }} + selector: + {{- include "selector_labels" . | nindent 4 }} \ No newline at end of file diff --git a/chart/helm/hivemetastore/templates/serviceaccount.yaml b/chart/helm/hivemetastore/templates/serviceaccount.yaml new file mode 100644 index 0000000..8db7dbb --- /dev/null +++ b/chart/helm/hivemetastore/templates/serviceaccount.yaml @@ -0,0 +1,30 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hive-metastore.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "all_objects_labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: false +{{- end }} diff --git a/chart/helm/hivemetastore/templates/tls-certificate.yaml b/chart/helm/hivemetastore/templates/tls-certificate.yaml new file mode 100644 index 0000000..c36bc21 --- /dev/null +++ b/chart/helm/hivemetastore/templates/tls-certificate.yaml @@ -0,0 +1,72 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- define "certDnsNames" -}} + {{- $dnsNames := list "localhost" "hive-metastore" (printf "%s.%s" "hive-metastore" .Release.Namespace) (printf "%s.%s.svc" "hive-metastore" .Release.Namespace) (printf "%s.%s.svc.cluster.local" "hive-metastore" .Release.Namespace) -}} + {{- $dnsNames = concat $dnsNames .Values.tls.generateCerts.subjectAlternativeName.additionalDnsNames -}} + {{- $dnsNames | toYaml -}} +{{- end -}} +{{- define "certIpAddresses" -}} + {{- $ipAddresses := list "127.0.0.1" -}} + {{- $ipAddresses = concat $ipAddresses .Values.tls.generateCerts.subjectAlternativeName.additionalIpAddresses -}} + {{- $ipAddresses | toYaml -}} +{{- end -}} + + +{{- if and (or .Values.tls.enabled .Values.tls.serverSideTls ) .Values.tls.generateCerts.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: hive-metastore-certificate + namespace: {{ .Release.Namespace }} + labels: + {{- include "all_objects_labels" . | nindent 4 }} + {{- include "cert_manager_label" . | nindent 4 }} + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-delete-policy": before-hook-creation,hook-failed + "helm.sh/hook-weight": "-11" +spec: + secretName: {{ .Values.tls.generateCerts.secretName }} + duration: {{ default 365 .Values.tls.generateCerts.duration | mul 24 }}h + renewBefore: 360h + commonName: hive-metastore.svc + isCA: false + privateKey: + algorithm: RSA + encoding: PKCS1 + size: 2048 + keystores: + jks: + create: {{ default false .Values.tls.generateCerts.keystores.jks.create }} + passwordSecretRef: + key: password-key + name: jks-password-secret + dnsNames: +{{ ( include "certDnsNames" . | indent 4 ) }} + ipAddresses: +{{ ( include "certIpAddresses" . | indent 4) }} + issuerRef: + group: cert-manager.io + {{- if .Values.tls.generateCerts.clusterIssuerName }} + name: {{ .Values.tls.generateCerts.clusterIssuerName }} + kind: ClusterIssuer + {{- else }} + name: hive-metastore-tls-issuer + kind: Issuer + {{- end }} + group: cert-manager.io +{{- end }} diff --git a/chart/helm/hivemetastore/templates/tls-secret.yaml b/chart/helm/hivemetastore/templates/tls-secret.yaml new file mode 100644 index 0000000..a0c6e22 --- /dev/null +++ b/chart/helm/hivemetastore/templates/tls-secret.yaml @@ -0,0 +1,31 @@ +{{/* + Copyright 2024-2025 NetCracker Technology Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- if and .Values.tls.enabled .Values.tls.serverSideTls }} +{{- if not .Values.tls.generateCerts.enabled }} +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: hive-metastore-certificate + namespace: {{ .Release.Namespace }} + labels: + {{- include "all_objects_labels" . | nindent 4 }} +data: + keystore.jks: {{ .Values.tls.certificates.keystore_jks | quote }} + truststore.jks: {{ .Values.tls.certificates.truststore_jks | quote }} +{{- end }} +{{- end }} diff --git a/chart/helm/hivemetastore/values.yaml b/chart/helm/hivemetastore/values.yaml new file mode 100644 index 0000000..5a0662a --- /dev/null +++ b/chart/helm/hivemetastore/values.yaml @@ -0,0 +1,436 @@ +# Copyright 2024-2025 NetCracker Technology Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for hive-metastore. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: ghcr.io/netcracker/qubership-hive-metastore + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: main + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "hive-metastore" + +replicaCount: 1 + +priorityClassName: ~ + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} +# fsGroup: 2000 + +securityContext: + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 10002 + +monitoring: + enabled: true + interval: 60s + +prometheusRules: + alert: + enabled: true + cpuThreshold: 90 + memoryThreshold: 90 + +service: + type: ClusterIP + port: 9083 + +tls: + enabled: false + serverSideTls: false + generateCerts: + enabled: false + secretName: hive-metastore-cm-cert + clusterIssuerName: ~ + keystores: + jks: + create: false + subjectAlternativeName: + additionalDnsNames: [ ] + additionalIpAddresses: [ ] + certificates: + jks_key: "" + keystore_jks: "" + truststore_jks: "" + +extraSecrets: {} +# mysslcert: +# stringData: | +# mysslcert.crt: | +# -----BEGIN CERTIFICATE----- +# Secret +# Content +# Goes +# Here +# -----END CERTIFICATE----- +# s3cert: +# stringData: | +# s3cert.crt: | +# -----BEGIN CERTIFICATE----- +# Secret +# Content +# Goes +# Here +# -----END CERTIFICATE----- + +extraVolumes: [] +# - name: tls-custom-cert +# secret: +# secretName: mysslcert +# - name: tls-s3-cert +# secret: +# secretName: s3cert + +extraVolumeMounts: [] +# - name: tls-custom-cert +# mountPath: /home/metastore/.postgresql/root.crt +# subPath: mysslcert.crt +# readOnly: true +# - name: tls-custom-cert +# mountPath: /opt/hive-metastore/trustcerts/ca.crt +# subPath: mysslcert.crt +# readOnly: true +# - name: tls-s3-cert +# mountPath: /opt/hive-metastore/trustcerts/s3cert.crt +# subPath: s3cert.crt +# readOnly: true + +env: [] +#env: +# - name: JAVA_TOOL_OPTIONS +# value: '-Dcom.amazonaws.sdk.disableCertChecking' +# - name: CURL_CA_BUNDLE +# value: '/etc/ssl/certs/s3.pem' + +hiveMetastore: + resources: + limits: + cpu: 700m + memory: 1Gi + requests: + cpu: 500m + memory: 256Mi + +livenessProbe: {} + # initialDelaySeconds: 20 + # periodSeconds: 10 + # timeoutSeconds: 5 + # failureThreshold: 6 +# successThreshold: 1 +readinessProbe: {} + # initialDelaySeconds: 20 + # periodSeconds: 10 + # timeoutSeconds: 5 + # failureThreshold: 6 +# successThreshold: 1 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +secretMounts: [] +# - name: metastore-cm-cert +# secretName: hive-metastore-cm-cert +# path: /opt/hive-metastore/trustcerts/ca.crt +# subPath: ca.crt +# - name: sert +# secretName: hive-metastore-cm-cert +# path: /opt/hive-metastore/certs/ +# - name: sample-secret +# secretName: sample-secret +# path: /secrets/sample.json + +hiveInitJob: + enabled: true + upgradeSchema: false + cleanupDB: false + priorityClassName: ~ + initAnnotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 300m + memory: 300Mi + +s3InitJob: + enabled: false + awsSigV4: "aws:minio:s3:s3" + disableTLSValidation: false + priorityClassName: ~ + initAnnotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation + resources: + limits: + cpu: 50m + memory: 64Mi + requests: + cpu: 50m + memory: 64Mi + +s3: + endpoint: "" + accessKey: "" + secretKey: "" + warehouseDir: s3a://warehouse/hive + +hive: + # hive PG user + user: "" + # hive PG user password + password: "" + db: metastore_db + +postgres: + host: pg-patroni.postgres.svc + port: 5432 + driver: org.postgresql.Driver + # PG admin credentials. Can be used to automate PG database creation. + adminUser: "" + adminPassword: "" + psqlParams: "" + jdbcParams: "" +# psqlParams: "sslmode=verify-ca" +# jdbcParams: "ssl=true,sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory" + +initJobCredentialsSecret: + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed + "helm.sh/hook-weight": "-15" + +metastoreConfig: + metastoreSiteProperties: | + + + metastore.warehouse.dir + {{ .Values.s3.warehouseDir }} + + + metastore.thrift.port + {{ .Values.service.port }} + + + metastore.initial.metadata.count.enabled + true + Enable a metadata count at metastore startup for metrics. + + + metastore.metrics.enabled + true + Enable metrics on the metastore. + + + metastore.metrics.reporters + JMX + A comma separated list of metrics reporters to start + + + metastore.async.log.enabled + true + + + +metastoreConfigsecret: + metastoreSitePropertiesSecret: | + + {{- if .Values.tls.serverSideTls }} + + metastore.use.SSL + true + + + metastore.keystore.path + /opt/hive-metastore/certs/keystore.jks + + + metastore.keystore.password + {{ .Values.tls.certificates.jks_key }} + + {{- end }} + + metastore.task.threads.always + org.apache.hadoop.hive.metastore.events.EventCleanerTask + + + metastore.expression.proxy + org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy + + + javax.jdo.option.ConnectionDriverName + {{ .Values.postgres.driver }} + + + javax.jdo.option.ConnectionURL + jdbc:postgresql://{{ include "postgres.host" . }}:{{ include "postgres.port" . }}/{{ .Values.hive.db }}{{ include "jdbcParams" (dict "Values" .Values "separator" "&")}} + + + javax.jdo.option.ConnectionUserName + {{ include "postgres.hive.user" . }} + + + javax.jdo.option.ConnectionPassword + {{ include "postgres.hive.password" . }} + + + + coreSitePropertiesSecret: | + + + fs.s3a.access.key + {{ include "s3.accessKey" . }} + + + fs.s3a.secret.key + {{ include "s3.secretKey" . }} + + + fs.s3a.connection.ssl.enabled + false + + + fs.s3a.endpoint + {{ include "s3.endpoint" . }} + + + fs.s3a.path.style.access + true + + + fs.s3a.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + fs.s3a.fast.upload + true + + + +jmxConfig: + jmxExporterConfig: | + + startDelaySeconds: 0 + ssl: false + lowercaseOutputName: true + lowercaseOutputLabelNames: true + attrNameSnakeCase: true + includeObjectNames: + - "metrics:*" + - "org.apache.hadoop.hive.metastore:*" + - "jvm:*" + - "process:*" + - "java.lang:*" + rules: + - pattern: "metrics<>Value" + name: "count_db" + help: "Total DBs count." + - pattern: "metrics<>Count" + name: "delete_total_count_db" + help: "metrics_delete_total_count_dbs_count Attribute exposed for management" + - pattern: "metrics<>Value" + name: "metrics_count_tables" + help: "metrics_total_count_tables_value Attribute exposed for management" + - pattern: "metrics<>Count" + name: "delete_total_count_tables" + help: "Count table delete" + - pattern: "metrics<>Count" + name: "open_connections_count" + help: "Open connections attribute exposed for management" + - pattern: "metrics<>Value" + name: "metrics_heap_used" + help: "metrics_heap_used_value metrics_memory_heap_used_value Attribute exposed for management" + - pattern: "java.lang<>CollectionTime" + name: "marksweepcompact_collectionTime" + help: "java_lang_marksweepcompact_collectiontime CollectionTime" + - pattern: "metrics<>Value" + name: "metrics_total_count_partitions_hv" + help: "metrics_total_count_partitions_value Attribute exposed for management" + - pattern: "metrics<>Count" + name: "metrics_create_total_count_partitions_hv" + help: "metrics_create_total_count_partitions_count Attribute exposed for management" + - pattern: "metrics<>Count" + name: "metrics_delete_total_count_partitions_hv" + help: "metrics_delete_total_count_partitions_count Attribute exposed for management" + +log4j2Properties: | + status = INFO + name = MetastoreLog4j2 + packages = org.apache.hadoop.hive.metastore + + # list of properties + property.metastore.log.level = INFO + property.metastore.root.logger = console + property.metastore.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} + property.metastore.log.file = metastore.log + property.hive.perflogger.log.level = INFO + + # list of all appenders + appenders = console + + # console appender + appender.console.type = Console + appender.console.name = console + appender.console.target = SYSTEM_ERR + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n + + # list of all loggers + loggers = DataNucleus, Datastore, JPOX, PerfLogger + + logger.DataNucleus.name = DataNucleus + logger.DataNucleus.level = ERROR + + logger.Datastore.name = Datastore + logger.Datastore.level = ERROR + + logger.JPOX.name = JPOX + logger.JPOX.level = ERROR + + logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger + logger.PerfLogger.level = ${sys:hive.perflogger.log.level} + + # root logger + rootLogger.level = ${sys:metastore.log.level} + rootLogger.appenderRefs = root + rootLogger.appenderRef.root.ref = ${sys:metastore.root.logger} \ No newline at end of file diff --git a/docker-transfer/Dockerfile b/docker-transfer/Dockerfile new file mode 100644 index 0000000..40a5888 --- /dev/null +++ b/docker-transfer/Dockerfile @@ -0,0 +1,5 @@ +FROM scratch + +# Transfer +COPY chart /chart +COPY docs /docs \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..300e7d3 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,69 @@ +# Copyright 2024-2025 NetCracker Technology Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM eclipse-temurin:17-jre-noble + +USER root + +COPY --chmod=0755 docker/entrypoint.sh /opt/entrypoint.sh + +RUN set -ex && \ +apt-get update && apt-get install -y procps postgresql-client --no-install-recommends + +ENV HADOOP_HOME=/opt/hadoop +ENV HIVE_HOME=/opt/hive-metastore +ENV HIVE_USER_NAME=metastore +ENV TRUST_CERTS_DIR=/opt/hive-metastore/trustcerts +ENV HADOOP_VERSION=3.3.6 +ENV HIVE_VERSION=4.0.1 + +RUN groupadd ${HIVE_USER_NAME} --gid 10002 && \ + useradd ${HIVE_USER_NAME} --uid 10002 --gid 10002 --create-home && \ + mkdir -p /usr/lib/${HIVE_USER_NAME} /data/${HIVE_USER_NAME} && \ + chown -R "${HIVE_USER_NAME}:${HIVE_USER_NAME}" /usr/lib/${HIVE_USER_NAME} /data/${HIVE_USER_NAME} + +RUN curl -k https://dlcdn.apache.org/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz | tar xvz -C opt && \ + mv opt/hadoop-$HADOOP_VERSION opt/hadoop && \ + rm -r opt/hadoop/share/doc && \ + curl -k https://repo1.maven.org/maven2/org/apache/hive/hive-standalone-metastore-server/$HIVE_VERSION/hive-standalone-metastore-server-$HIVE_VERSION-bin.tar.gz | tar xvz -C opt && \ + mv opt/apache-hive-metastore-4.0.1-bin opt/hive-metastore && \ + curl -k -L https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/1.0.1/jmx_prometheus_javaagent-1.0.1.jar --output opt/hive-metastore/lib/jmx_prometheus_javaagent-1.0.1.jar && \ + rm opt/hive-metastore/lib/postgresql-42.5.1.jar && \ + mkdir opt/postgresql-42.7.4 && \ + curl -k -L https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.4/postgresql-42.7.4.jar --output opt/postgresql-42.7.4/postgresql-42.7.4.jar && \ + rm opt/hadoop/share/hadoop/tools/lib/jdom2-2.0.6.jar && \ + curl -k -L https://repo1.maven.org/maven2/org/jdom/jdom2/2.0.6.1/jdom2-2.0.6.1.jar --output opt/hadoop/share/hadoop/tools/lib/jdom2-2.0.6.1.jar && \ + rm opt/hadoop/share/hadoop/common/lib/jackson-mapper-asl-1.9.13.jar && \ + rm opt/hadoop/share/hadoop/hdfs/lib/jackson-mapper-asl-1.9.13.jar && \ + mkdir opt/jackson-mapper-asl && \ + curl -k -L https://repo1.maven.org/maven2/org/codehaus/jackson/jackson-mapper-asl/1.9.13-cloudera.3/jackson-mapper-asl-1.9.13-cloudera.3.jar --output opt/jackson-mapper-asl/jackson-mapper-asl-1.9.13-cloudera.3.jar && \ + chmod -R 755 opt + +RUN ln -s /opt/hadoop/share/hadoop/tools/lib/hadoop-aws* /opt/hadoop/share/hadoop/common/lib/ && \ + ln -s /opt/hadoop/share/hadoop/tools/lib/aws-java-sdk* /opt/hadoop/share/hadoop/common/lib/ && \ + ln -s /opt/postgresql-42.7.4/postgresql-42.7.4.jar /opt/hadoop/share/hadoop/common/lib/ && \ + ln -s /opt/postgresql-42.7.4/postgresql-42.7.4.jar /opt/hive-metastore/lib/ && \ + ln -s /opt/jackson-mapper-asl/jackson-mapper-asl-1.9.13-cloudera.3.jar /opt/hadoop/share/hadoop/common/lib/ && \ + ln -s /opt/jackson-mapper-asl/jackson-mapper-asl-1.9.13-cloudera.3.jar /opt/hadoop/share/hadoop/hdfs/lib/ + +RUN mkdir -p /opt/hive-metastore-metrics/metrics \ + && sed -i '/export HADOOP_OPTS="\$METASTORE_HADOOP_OPTS \$HADOOP_OPTS"/a\ export HADOOP_OPTS="\$HADOOP_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9025 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -javaagent:\/opt\/hive-metastore\/lib\/jmx_prometheus_javaagent-1.0.1.jar=9028:\/opt\/hive-metastore\/conf\/jmx-exporter-config.yaml -Dcom.sun.management.jmxremote.ssl=false"' /opt/hive-metastore/bin/ext/metastore.sh + +RUN mkdir -p ${TRUST_CERTS_DIR} && \ + chmod 777 ${JAVA_HOME}/lib/security/cacerts && \ + chmod -R 777 ${TRUST_CERTS_DIR} + +USER ${HIVE_USER_NAME} +WORKDIR $HIVE_HOME +ENTRYPOINT [ "/opt/entrypoint.sh" ] \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000..c662be4 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Copyright 2024-2025 NetCracker Technology Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set +e +if [[ "$(ls ${TRUST_CERTS_DIR})" ]]; then + for filename in ${TRUST_CERTS_DIR}/*; do + echo "Import $filename certificate to Java cacerts" + ${JAVA_HOME}/bin/keytool -import -trustcacerts -keystore ${JAVA_HOME}/lib/security/cacerts -storepass changeit -noprompt -alias ${filename} -file ${filename} + done; +fi +set -e + +/opt/hive-metastore/bin/start-metastore -p "${HIVE_SERVICE_PORT:-9083}" \ No newline at end of file diff --git a/docs/internal/images/hive-metastore-ha-scheme-1-pod-off.png b/docs/internal/images/hive-metastore-ha-scheme-1-pod-off.png new file mode 100644 index 0000000..38ed45c Binary files /dev/null and b/docs/internal/images/hive-metastore-ha-scheme-1-pod-off.png differ diff --git a/docs/internal/images/hive-metastore-ha-scheme.png b/docs/internal/images/hive-metastore-ha-scheme.png new file mode 100644 index 0000000..d7cecfe Binary files /dev/null and b/docs/internal/images/hive-metastore-ha-scheme.png differ diff --git a/docs/internal/images/hive-metastore-non-ha-scheme.png b/docs/internal/images/hive-metastore-non-ha-scheme.png new file mode 100644 index 0000000..434a6f2 Binary files /dev/null and b/docs/internal/images/hive-metastore-non-ha-scheme.png differ diff --git a/docs/internal/images/hive-metastore_network.png b/docs/internal/images/hive-metastore_network.png new file mode 100644 index 0000000..abead4e Binary files /dev/null and b/docs/internal/images/hive-metastore_network.png differ diff --git a/docs/internal/images/hive_metastore_JVM.png b/docs/internal/images/hive_metastore_JVM.png new file mode 100644 index 0000000..d8f8e3a Binary files /dev/null and b/docs/internal/images/hive_metastore_JVM.png differ diff --git a/docs/internal/images/hive_metastore_cpu.png b/docs/internal/images/hive_metastore_cpu.png new file mode 100644 index 0000000..463a724 Binary files /dev/null and b/docs/internal/images/hive_metastore_cpu.png differ diff --git a/docs/internal/images/hive_metastore_dashboard_variables.png b/docs/internal/images/hive_metastore_dashboard_variables.png new file mode 100644 index 0000000..60da152 Binary files /dev/null and b/docs/internal/images/hive_metastore_dashboard_variables.png differ diff --git a/docs/internal/images/hive_metastore_data.png b/docs/internal/images/hive_metastore_data.png new file mode 100644 index 0000000..44e496c Binary files /dev/null and b/docs/internal/images/hive_metastore_data.png differ diff --git a/docs/internal/images/hive_metastore_memory.png b/docs/internal/images/hive_metastore_memory.png new file mode 100644 index 0000000..b75fdcb Binary files /dev/null and b/docs/internal/images/hive_metastore_memory.png differ diff --git a/docs/internal/images/hive_metastore_overview.png b/docs/internal/images/hive_metastore_overview.png new file mode 100644 index 0000000..fe98bfe Binary files /dev/null and b/docs/internal/images/hive_metastore_overview.png differ diff --git a/docs/internal/images/hive_metastore_space.png b/docs/internal/images/hive_metastore_space.png new file mode 100644 index 0000000..8174cd7 Binary files /dev/null and b/docs/internal/images/hive_metastore_space.png differ diff --git a/docs/public/architecture.md b/docs/public/architecture.md new file mode 100644 index 0000000..94e5232 --- /dev/null +++ b/docs/public/architecture.md @@ -0,0 +1,39 @@ +This document describes the architectural features of the qubership hive-metastore service. The following topics are covered in the document: + +* [Overview](#overview) + * [Hive-metastore](#hive-metastore) +* [Supported Deployment Scheme](#supported-deployment-scheme) + * [HA Deployment Scheme](#ha-deployment-scheme) + * [Non-HA Deployment Scheme](#non-ha-deployment-scheme) + + +# Overview + +## Hive-metastore + +The Hive-metastore is simply a relational database. +It stores metadata related to tables/schemas you create to easily query big data stored. +When you create a new Hive table, the information related to the schema (column names, data types) is stored in the Hive metastore relational database. +Other information like input/output formats, partitions, and so on, are all stored in the metastore. + +# Supported Deployment Scheme + +## HA Deployment Scheme + +Hive-metastore supports operation in the high availability mode. +In this mode, 2 hive-metastore hearths are created, which provide backup operation for the hive-metastore service. In this mode, each pod of the service operates in parallel mode, processing requests from Trino. + +Scheme of work: +![alt text](/docs/internal/images/hive-metastore-ha-scheme.png "Hive-metastore HA scheme") + +**Situations**: There are situations when one of the hive-metastore pods becomes unavailable. In such a case, the service works as shown in the following image. + +Scheme of work: +![alt text](/docs/internal/images/hive-metastore-ha-scheme-1-pod-off.png "Hive-metastore HA scheme( 1 pod disable)") + +## Non-HA Deployment Scheme + +Hive-metastore in the non-HA mode has only one replica of each component. + +Scheme of work: +![alt text](/docs/internal/images/hive-metastore-non-ha-scheme.png "Hive-metastore non-HA scheme") diff --git a/docs/public/installation.md b/docs/public/installation.md new file mode 100644 index 0000000..a689405 --- /dev/null +++ b/docs/public/installation.md @@ -0,0 +1,615 @@ +The following topics are covered in this chapter: + +* [Prerequisites](#prerequisites) + * [Common](#common) + * [Kubernetes](#common) + * [OpenShift](#common) +* [Best Practices and Recommendations](#best-practices-and-recommendations) + * [HWE](#hwe) + * [Small](#small) + * [Meduim](#medium) + * [Large](#large) +* [Parameters](#parameters) + * [Hive-Metastore](#hive-metastore) + * [Enable HTTPS/TLS](#enable-httpstls) + * [Secure Connections from Hive Metastore](#secure-connections-from-hive-metastore) + * [Adding CA Certificates to Hive Metastore's Default Java Truststore](#adding-ca-certificates-to-hive-metastores-default-java-truststore) + * [Configure Connections to Use SSL/TLS](#configure-connections-to-use-ssltls) + * [PostgreSQL](#postgresql) + * [S3](#s3) + * [Monitoring Configuration](#monitoring-configuration) + * [S3 Initialization Job](#s3-initialization-job) + * [AWS V4 Signature Configuration](#aws-v4-signature-configuration) + * [TLS](#tls) +* [Installation](#installation) + * [Manual Deployment](#manual-deployment) +* [On-Prem](#on-prem) + * [HA Scheme](#ha-scheme) + * [Non-HA Scheme](#non-ha-scheme) +* [Upgrade](#upgrade) +* [Rollback](#rollback) + +# Prerequisites + +The prerequisites for the installation are as follows: + +## Common + +The common prerequisites are specified below. + +* Helm >= 3 +* PostgreSQL service is required. +* S3 storage. It must have bucket pre-created (for example, bucket `hive`, directory `warehouse`) or [S3 Initialization Job](#s3-initialization-job) must be used. + +## Kubernetes + +The prerequisites for Kubernetes are specified below. + +* Kubernetes >= 1.13 +* A namespace in Kubernetes should be created. + +## OpenShift + +The prerequisites for OpenShift are specified below. + +* If you are using the OpenShift cloud with restricted SCC, the Hive Metastore namespace must have specific annotations: + +```bash +oc annotate --overwrite namespace openshift.io/sa.scc.uid-range="10002/10002" +oc annotate --overwrite namespace openshift.io/sa.scc.supplemental-groups="10002/10002" +``` + +Alternatively, it is possible to avoid setting annotations if you set the `runAsUser` parameter of security context to `~`: + +```yaml +securityContext: + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: ~ +``` + +# Best Practices and Recommendations + +## HWE + +The hardware requirements are specified below. + +### Small + +`Small` profile specifies the resources that are enough to start hive-metastore. + +The profile resources are specified below: + +| Container | CPU Limit | Memory Limit | Number of Containers | +|:--------------:|:---------:|:------------:|:--------------------:| +| Hive-Metastore | 700m | 1G | 1 | +| Hive Init Job | 500m | 512Mi | 1 | +| Hive S3 Job | 50m | 64Mi | 1 | + +**Note**: The above resources are required for starting, not for working under load. For production, the resources should be increased. + +### Medium + +`Medium` profile specifies the approximate resources that are enough to run hive-metastore for dev purposes. +The profile resources are specified below: + +| Container | CPU Limit | Memory Limit | Number of Containers | +|:--------------:|:---------:|:------------:|:--------------------:| +| Hive-Metastore | 1 | 2G | 1 | +| Hive Init Job | 500m | 512Mi | 1 | +| Hive S3 Job | 50m | 64Mi | 1 | + +**Note**: The above resources are enough for development purposes, not for working under production load. For production, the resources should be increased. + +### Large + +`Large` profile specifies the approximate resources that are enough to run hive-metastore for prod purposes. +The profile resources are specified below: + +| Container | CPU Limit | Memory Limit | Number of Containers | +|:-------------:|:---------:|:------------:|:--------------------:| +|Hive-Metastore | 2 | 4G | 2 | +| Hive Init Job | 500m | 512Mi | 1 | +| Hive S3 Job | 50m | 64Mi | 1 | + +# Parameters + +The parameters are specified below. + +## Hive-Metastore + +The following table lists the configurable parameters of the Hive-Metastore chart and their default values. + +**Note**: It is required to fill the user/password parameters. + +| Parameter | Type | Mandatory | Default value | Description | +|------------------------------------------|-------------------|-----------|---------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `image.repository` | string | false | `"ghcr.io/netcracker/qubership-hive-metastore"` | The image repository. | +| `image.pullPolicy` | string | false | `image.pullPolicy` | The image pull policy. | +| `image.tag` | string | false | `main` | This overrides the image tag, which is the hive-metastore version by default. | +| `securityContext.runAsUser` | string | false | `10002` | All processes' containers that run with the specified user. | +| `securityContext.runAsGroup` | string | false | `` | All processes' containers that run with the specified user group. | +| `service.type` | string | false | `"ClusterIP"` | The type of Kubernetes service. | +| `service.port` | string | false | `9083` | It specifies the Hive-metastore HTTP server port. | +| `nodeSelector` | map | false | `{}` | This parameter allows you to specify on which nodes of the cluster the Hive-metastore substations should be deployed. | +| `priorityClassName` | string | false | `~` | Priority class name for hive metastore pod. | +| `tolerations` | array | false | `[]` | This parameter allows you to specify the tolerance to special conditions on cluster nodes. | +| `affinity` | map | false | `{}` | This parameter allows you to specify the preferences for placing hearths on cluster nodes. | +| `serviceAccount.create` | string | false | `true` | It specifies whether a service account should be created for Hive-metastore. | +| `serviceAccount.name` | string | false | `""` | The Hive-metastore service account name to use. If not set and create is true, a name is generated using the fullname template. | +| `serviceAccount.annotations` | map | false | `{}` | The annotations to be added to the service account. | +| `s3.endpoint` | string | true | `""` | The MinIO S3 storage endpoint. Hive-Metastore uses S3 storage to store data. | +| `s3.accessKey` | string | true | `""` | The MinIO access key. | +| `s3.secretKey` | string | true | `""` | The MinIO secret key. | +| `s3.warehouseDir` | string | true | `s3a://warehouse/hive` | The directory in S3 storage, where Hive-Metastore should store its data. | +| `hive.user` | string | false | `""` | The Hive Metastore to PG connection user that is set as owner of the Hive database. This user is created by `hiveInitJob`. | +| `hive.password` | string | false | `""` | The Hive Metastore to PG connection user password. The password of the user is created by `hiveInitJob`. | +| `hive.db` | string | false | `metastore_db` | The Hive Metastore database created in PG by `hiveInitJob`. | +| `postgres.adminUser` | string | true | `""` | The PG user with permission to create a role and a database. This user is used to prepare a database in PG. | +| `postgres.adminPassword` | string | true | `""` | The password of the user set in `postgres.user`. | +| `postgres.host` | string | true | `""` | The PG host where the Hive Metastore DB is located. | +| `postgres.port` | string | true | `""` | The PG port where the Hive Metastore DB is located. | +| `postgres.driver` | string | true | `"org.postgresql.Driver"` | The PG connection driver. | +| `postgres.psqlParams` | string | false | `""` | PostgreSQL connection parameters used by Hive Metastore database init job. | +| `postgres.jdbcParams` | string | false | `""` | PostgreSQL connection parameters used by Hive Metastore. | +| `hiveInitJob.enabled` | bool | false | `"true"` | The parameter that enables the Pre-install/upgrade Helm job. The job creates a database in PG, initiates the Hive schema, and upgrades if necessary. The database related parameters are set in the `postgres` and `hive` sections. | +| `hiveInitJob.upgradeSchema` | bool | false | `"false"` | Enables hive metastore database upgrade. Must be set to to true when upgrading to 4.* versions from 3.* versions without database cleanup. | +| `hiveInitJob.cleanupDB` | bool | false | `"false"` | Enables the Hive metastore database cleanup. The database, the user along with it's grants will be dropped. | +| `hiveInitJob.initAnnotations` | string | false | `"helm.sh/hook": pre-install, pre-upgrade "helm.sh/hook-weight": "-10" helm.sh/hook-delete-policy": before-hook-creation` | The database init job annotations. | +| `hiveInitJob.priorityClassName` | string | false | `~` | Priority class name for init job. | +| `s3InitJob.enabled` | bool | false | `"false"` | The parameter that enables the Pre-install/upgrade Helm job. The job creates a database in PG, initiates the Hive schema, and upgrades if necessary. The database related parameters are set in the `postgres` and `hive` sections. | +| `s3InitJob.disableTLSValidation` | bool | false | `"false"` | Disables TLS certificate validation for S3 storage in the job. | +| `s3InitJob.awsSigV4` | string | false | `"aws:minio:s3:s3"` | AWS V4 signature authentication configuration of the following format ``. Used to authenticate requests to S3 storage. Configured to work with S3 MinIO by default. Details at https://curl.se/docs/manpage.html. | +| `s3InitJob.initAnnotations` | string | false | `"helm.sh/hook": pre-install, pre-upgrade "helm.sh/hook-weight": "-10" helm.sh/hook-delete-policy": before-hook-creation` | The database init job annotations. | +| `s3InitJob.priorityClassName` | string | false | `~` | Priority class name for init job. | +| `log4j2Properties` | multi-line string | false | `The default properties' examples are below the table.` | The Log4j2 configuration. The metastore logging level is set by the `property.metastore.log.level` property. | +| `monitoring.enabled` | bool | true | `true` | The string to submit a Grafana dashboard. The custom parameter is not present in the **values.yaml** file. | +| `monitoring.interval` | string | false | `60s` | This parameter parameter responsible for the amount of time after which the pod metrics are polled. | +| `prometheusRules.alert.enable` | bool | true | `true` | This parameter submits Prometheus rules for the Hive-Metastore. | +| `prometheusRules.alert.cpuThreshold` | int | false | `90` | This parameter specifies the percentage of CPU resources' limit that triggers a CPU threshold alert. Note that the resource limit should be set. | +| `prometheusRules.alert.memoryThreshold` | int | false | `90` | This parameter specifies the percentage of memory resources' limit that triggers a memory threshold alert. Note that the resource limit should be set. | +| `tls.enabled` | bool | false | `false` | Enables secure connections from Hive Metastore to PG and S3. | +| `tls.serverSideTls` | bool | false | `false` | Enables TLS for Hive Metastore server if `tls.enabled` is `true`. | +| `tls.generateCerts.enabled` | string | false | `false` | The parameter to integrate cert-manager. | +| `tls.generateCerts.secretName` | string | false | `hive-metastore-certificate` | The name of the certificate for a TLS operation. | +| `tls.generateCerts.clusterIssuerName` | string | false | `common-cluster-issuer` | The name of the issuer to create a certificate for a TLS operation. | +| `tls.generateCerts.keystores.jks.create` | bool | false | `false` | Enables keystore and truststore automatic creation if `tls.enabled`, `tls.serverSideTls` and `tls.generateCerts.enabled` are enabled. | +| `extraSecrets` | map | false | `{}` | Allows to create custom secrets to pass them to pods during the deployments. The format for secret data is "key/value" where key (can be templated) is the name of the secret that will be created, value - an object with the standard 'data' or 'stringData' key (or both). The value associated with those keys must be a string (can be templated). | +| `extraVolumes` | array | false | `[]` | One or more additional volume mounts to add to Hive Metastore and DB init job pods. | +| `extraVolumeMounts` | array | false | `[]` | One or more additional volume mounts to add to Hive Metastore and DB init job pods. | +| `secretMounts` | array | false | `[]` | One or more existing volume mounts to add to Hive Metastore and DB init job pods. | +| `env` | array | false | `[]` | Additional env paraeters for Hive Metastore Service. | + +```yaml +# Default log4j2 properties +log4j2Properties: | + status = INFO + name = MetastoreLog4j2 + packages = org.apache.hadoop.hive.metastore + + # list of properties + property.metastore.log.level = INFO + property.metastore.root.logger = console + property.metastore.log.dir = \${sys:java.io.tmpdir}/\${sys:user.name} + property.metastore.log.file = metastore.log + property.hive.perflogger.log.level = INFO + + # list of all appenders + appenders = console + + # console appender + appender.console.type = Console + appender.console.name = console + appender.console.target = SYSTEM_ERR + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n + + # list of all loggers + loggers = DataNucleus, Datastore, JPOX, PerfLogger + + logger.DataNucleus.name = DataNucleus + logger.DataNucleus.level = ERROR + + logger.Datastore.name = Datastore + logger.Datastore.level = ERROR + + logger.JPOX.name = JPOX + logger.JPOX.level = ERROR + + logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger + logger.PerfLogger.level = \${sys:hive.perflogger.log.level} + + # root logger + rootLogger.level = \${sys:metastore.log.level} + rootLogger.appenderRefs = root + rootLogger.appenderRef.root.ref = \${sys:metastore.root.logger} +``` + +## Enable HTTPS/TLS + +You can use two options to enable HTTPS/TLS. + +1. Using manual certificate for Trino. + +1.1 Generate self-signed certificates for the Hive-Metastore service. + +1.2 Pass the following parameters to the chart: + +```yaml +secretMounts: + - name: sert + secretName: hive-metastore-certificate + path: /opt/hive-metastore/certs/ +tls: + enabled: true + serverSideTls: true + certificates: + jks_key: "password_jks" + keystore.jks: "keystore_certificate" + truststore.jks: "truststore_certificate" +``` + +2. Using Cert-manager to get the certificate. + +**Note**: Cert-manager must be installed in the cluster for this to work. + +2.1 Use cluster-issuer to create a certificate. + +Pass the following parameters to the chart: + +```yaml +secretMounts: + - name: sert + secretName: hive-metastore-certificate + path: /opt/hive-metastore/certs/ +tls: + enabled: true + serverSideTls: true + generateCerts: + enabled: true + secretName: hive-metastore-certificate + clusterIssuerName: common-cluster-issuer + keystores: + jks: + create: true + certificates: + jks_key: "password_jks" +``` + +## Secure Connections from Hive Metastore + +In order to secure connections from Hive Metastore using TLS/SSL: + +- An appropriate CA certificate needs to be imported to Java default truststore depending on the certificate used by a service that Hive Metastore will be connecting. +- In case of PostgreSQL, the certificate also needs to be mounted to `/home/metastore/.postgresql/root.crt` path using `extraVolumes` and `extraVolumeMounts` as described below in options 2 and 3. +- Connection settings should be configured to use TLS/SSL. + +### Adding CA Certificates to Hive Metastore's Default Java Truststore + +There are three options for adding certificates to Hive Metastore. + +**Note**: The certificate should be mounted to `opt/hive-metastore/trustcerts`. All certificates from that directory are added to Java default truststore - cacerts. + +1. Enable certificate generation to use cert-manager certificates. + +```yaml +tls: + enabled: true + generateCerts: + enabled: true + secretName: hive-metastore-cm-cert + clusterIssuerName: common-cluster-issuer <--------- issuer should be the same as for the connecting service's certificate + keystores: + jks: + create: true + subjectAlternativeName: + additionalDnsNames: [ ] + additionalIpAddresses: [ ] +``` + +2. Add certificates using `extraSecrets` deployment parameter. In this case, the added secret should be mounted into the coordinator and the workers pod. + Mounting details are set using the `extraVolumes` and `extraVolumeMounts` parameters. The Certificates should be mounted to `opt/hive-metastore/trustcerts`. +```yaml +extraSecrets: + mysslcert: # secret name + stringData: | + mysslcert.crt: | + -----BEGIN CERTIFICATE----- + Secret + Content + Goes + Here + -----END CERTIFICATE----- + +extraVolumes: + - name: tls-custom-cert + secret: + secretName: mysslcert + +extraVolumeMounts: + - name: tls-custom-cert + mountPath: /opt/hive-metastore/trustcerts/mysslcert.crt + subPath: mysslcert.crt + readOnly: true +``` + +3. Add an existing secret to truststore. In this case, only `extraVolumes` and `extraVolumeMounts` parameters need to be configured to mount the secret. + For example, to use `defaultsslcertificate` (`ca-bundle.crt` part), it can be configured as described in option 2. + +```yaml +extraVolumes: + - name: defaultcert + secret: + secretName: defaultsslcertificate + +extraVolumeMounts: + - name: defaultcert + mountPath: /opt/hive-metastore/trustcerts/ca-bundle.crt + subPath: ca-bundle.crt + readOnly: true +``` + +### Configure Connections to Use SSL/TLS + +This section provides details about configuring connections to use SSL or TLS. + +#### PostgreSQL + +Hive Metastore service deployment connects to PostgreSQL in several ways - using Java code, PSQL tool and Hive Metastore's schema tool. + +1. In order to secure connections from Hive Metastore to PostgreSQL using TLS/SSL an appropriate CA certificate needs to be imported to Java default truststore and also mounted to `/home/metastore/.postgresql/root.crt` path. +You can add the certificates as described above in [Adding CA Certificates to Hive Metastore's Default Java Truststore](#adding-ca-certificates-to-hive-metastores-default-java-truststore). +Configuration example: + +```yaml +extraSecrets: + pgcert: + stringData: | + pgcert.crt: | + -----BEGIN CERTIFICATE----- + Secret + Content + Goes + Here + -----END CERTIFICATE----- + +extraVolumes: + - name: tls-pg-cert + secret: + secretName: pgcert + +extraVolumeMounts: + - name: tls-pg-cert + mountPath: /home/metastore/.postgresql/root.crt + subPath: pgcert.crt + readOnly: true +``` + +2. To enable secure connection to PostgreSQL, the following parameters should be added to `postgres` section of deployemnt parameters. +```yaml +postgres: + psqlParams: "sslmode=verify-ca" + jdbcParams: "ssl=true,sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory" +``` + +To ignore certificate validation, `sslfactory` should be set to `org.postgresql.ssl.NonValidatingFactory`, `sslmode` to `allow`: +```yaml +postgres: + psqlParams: sslmode=allow + jdbcParams: ssl=true,sslfactory=org.postgresql.ssl.NonValidatingFactory +``` + +PosgreSQL in metastore site properties should be configured as follows: + +```yaml +metastoreConfigsecret: + metastoreSitePropertiesSecret: | + + ... + + javax.jdo.option.ConnectionURL + jdbc:postgresql://{{ include "postgres.host" . }}:{{ include "postgres.port" . }}/{{ .Values.hive.db }}{{ include "jdbcParams" (dict "Values" .Values "separator" "&")}} + + ... + +``` + +#### S3 + +To enable secure connection to S3 storage, add the certificate as described above in [Adding CA Certificates to Hive Metastore's Default Java Truststore](#adding-ca-certificates-to-hive-metastores-default-java-truststore). + +Enable S3 in coreSiteProperties as follows: + +```yaml +metastoreConfigsecret: + coreSitePropertiesSecret: | + + ... + + fs.s3a.connection.ssl.enabled + true + + ... + +``` + +For S3 buckets auto creation mount S3 CA certificate using `extraSecrets` and set `CURL_CA_BUNDLE`: +```yaml +extraSecrets: + s3cert: + stringData: | + s3.pem: | + -----BEGIN CERTIFICATE----- + Secret + Content + Goes + Here + -----END CERTIFICATE----- + +extraVolumes: + - name: s3-tls-cert + secret: + secretName: s3cert + +extraVolumeMounts: + - name: s3-tls-cert + mountPath: /opt/apache-hive-metastore-4.0.1-bin/certs/s3.pem + subPath: s3.pem + readOnly: true + +env: + - name: CURL_CA_BUNDLE + value: '/opt/apache-hive-metastore-4.0.1-bin/certs/' +``` + +To ignore certificate validation for S3, the following properties should be added: + +```yaml +env: + - name: JAVA_TOOL_OPTIONS + value: '-Dcom.amazonaws.sdk.disableCertChecking' + +s3InitJob: + enabled: true + disableTLSValidation: true +``` + +## Monitoring Configuration + +Monitoring is represented by Grafana Dashboard. To get the Hive-Metastore dashboards configured, perform the following steps: + +1. Enable the Grafana dashboard. + + ```yaml + metrics: + enable: true + ``` + +2. Configure and enable the Prometheus pod monitor. + + ```yaml + prometheusRules: + alert: + enabled: true + cpuThreshold: 90 + memoryThreshold: 90 + ``` + +## S3 Initialization Job + +Hive Metastore requires a warehouse directory in S3 storage. It is configured by `s3.warehouseDir` deployment parameter. +To automatically create the bucket and path, `s3InitJob` should be enabled(`s3InitJob.enabled` parameter must be set to `true`). After the bucket and path are created, a mock file is uploaded to the path to keep the path from being removed. +`Curl` communicates with S3 storage by AWS S3 REST API uses `curl`. `Curl` supports AWS V4 signature authentication for requests. + +```yaml +s3InitJob: + enabled: true + awsSigV4: "aws:minio:s3:s3" + disableTLSValidation: true + priorityClassName: ~ + initAnnotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation + resources: + limits: + cpu: 50m + memory: 64Mi + requests: + cpu: 50m + memory: 64Mi +``` + +Signature configuration string is set in `s3InitJob.awsSigV4` and is different for different S3 storages. +By default, `s3InitJob.awsSigV4: "aws:minio:s3:s3"` to work with S3 MinIO. If a different S3 storage is used, the signature configuration should be updated according the below instruction. + +### AWS V4 Signature Configuration + +Configuration string format: + +- The provider argument is a string that is used by the algorithm when creating outgoing authentication headers. +- The region argument is a string that points to a geographic area of a resources collection (region-code) when the region name is omitted from the endpoint. +- The service argument is a string that points to a function provided by a cloud (service-code) when the service name is omitted from the endpoint. + +### TLS + +TLS configuration is described in [Configure Connections to Use SSL/TLS](#s3) + +# Installation + +## Manual Deployment + +Refer to the releases page to find the release tag. + +1. Navigate to the desired release tag and download the `/chart/helm/hivemetastore` directory. + +2. Edit the parameters in the **values.yaml** file. The configuration parameter details are described above. For example, modify the following parameters: + +```yaml + For example, + + ```yaml + s3: + endpoint: your.s3.endpoint.address.com + accessKey: minioaccess + secretKey: miniosecret + warehouseDir: s3a://warehouse/hive + + hive: + user: hive + password: hive_password + db: metastore_db + + postgres: + user: postgres_user + password: password + host: pg-patroni.postgres-service.svc + port: 5432 + driver: org.postgresql.Driver +``` + +4. Install the chart to the K8s namespace created in the [Prerequisites](#prerequisites) section. + + ``` + helm install --values --namespace --debug + #Example + helm install hive-metastore . --debug + ``` + +# On-Prem + +The installation process for On-Prem is specified below. + +## HA Scheme + +Hive-metastore supports the high availability mode. + +To do this, you need to specify several replicas for its operation when deploying. + +To do this, pass the following parameters to the chart: + +```yaml +replicaCount: 2 # or more replicas +``` + +## Non-HA Scheme + +To do this, pass the following parameters to the chart: + +```yaml +replicaCount: 1 +``` + +# Upgrade + +The upgrade process can be performed using [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/). +It is required to check the new release notes for incompatibility points of parameters with the new version of hive-metastore. + +**Note**: `hiveInitJob.upgradeSchema` parameter must be set to true when updating from hive 3.* versions to hive 4.* versions and when hive metastore database is not cleaned. + +**Note**: If you need to change S3 connection properties, Postgres Hive-metastore database must be manually recreated and Hive-metastore must be reinstalled after removing the previous release from the namespace. diff --git a/docs/public/monitoring.md b/docs/public/monitoring.md new file mode 100644 index 0000000..4bc7493 --- /dev/null +++ b/docs/public/monitoring.md @@ -0,0 +1,157 @@ +This chapter describes the Qubership Hive-Metastore monitoring. Qubership Hive-Metastore monitoring requires Qubership Monitoring installed in the cluster, but you can try to make it work using [Grafana Operator](https://github.com/grafana/grafana-operator) and [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) + +# Table of Contents + +* [Grafana Dashboard](#grafana-dashboard) +* [Prometheus Rules](#prometheus-rules) + +# Grafana Dashboard + +This section describes the Qubership Hive-Metastore monitoring dashboard. + +![alt text](/docs/internal/images/hive_metastore_dashboard_variables.png "Dashboard Variables") + +To access the dashboard: + +1. Navigate to the Grafana server and log in using the provided credentials. +2. Select the Hive-Metastore dashboard. +3. Select the `namespace`. +4. Select the operator's `pod`. +5. Select the time range. + +## Metrics + +* [Hive-Metastore Overview](#hive-metastore-overview) +* [CPU Usage](#cpu-usage) +* [Memory Usage](#memory-usage) +* [JVM](#jvm) +* [Disk Space Usage](#disk-space-usage) +* [Network](#network) +* [Data](#data) + +### Hive-Metastore Overview + +This section describes the Qubership Hive-Metastore overall state. + +![alt text](/docs/internal/images/hive_metastore_overview.png "Hive-Metastore Overview") + +#### Hive-Metastore Status + +This displays the Qubership Hive-Metastore health status. +In case of several replicas, the status is "UP" if one of the pods is in the "Running" phase. +This approach is based on the fact that only one of the replicas is always active. + +#### Active Replicas Count + +This displays the number of Qubership Hive-Metastore running pods. +Only one pod, which is the leader, is active. + +#### Pods Count + +This displays the number of pods in the namespace. + +#### Pod Status + +This displays the status of the pod. + +The possible values are: + +* Failed +* Pending +* Running +* Succeeded +* Unknown + +### CPU Usage + +This displays the CPU consumption in Qubership Hive-Metastore pods based on the metrics collected from the docker. + +![alt text](/docs/internal/images/hive_metastore_cpu.png "CPU") + +### Memory Usage + +This displays the memory consumption in Qubership Hive-Metastore pods based on the metrics collected from the docker. + +![alt text](/docs/internal/images/hive_metastore_memory.png "Memory") + +### JVM + +This section describes the Qubership Hive-Metastore JVM state. + +![alt text](/docs/internal/images/hive_metastore_JVM.png "Hive-Metastore JVM") + +#### JVM Heap Usage + +This displays the JVM heap usage by Qubership Hive-Metastore. + +#### Threads + +This displays the threads rate per second by Qubership Hive-Metastore. + +#### GC Time + +This displays the garbage collection time rate per second by Qubership Hive-Metastore. + +### Disk Space Usage + +This displays the disk usage for Qubership Hive-Metastore pods. + +![alt text](/docs/internal/images/hive_metastore_space.png "Disk") + +### Network + +![alt text](/docs/internal/images/hive-metastore_network.png "Network") + +### Open connection + +This displays open connections to Qubership Hive-Metastore. + +#### Receive/Transmit Bandwidth + +This displays the network traffic in bytes per second for the pod. + +#### Rate of Received/Transmitted Packets + +This displays the network packets for the pod. + +#### Rate of Received/Transmitted Packets Dropped + +This displays the dropped packets for the pod. + +### Data + +This section describes the Qubership Hive-Metastore data state. + +![alt text](/docs/internal/images/hive_metastore_data.png "Hive-Metastore data") + +#### DB + +This displays the count DB of the Qubership Hive-Metastore. +Count DB - The total number of databases created. +Delete DB - The number of deleted databases over a period of time. + +# Prometheus Rules + +This section describes the Prometheus rules configured for the Qubership Hive-Metastore. + +## Alerts + +The following alerts are configured for the Qubership Hive-Metastore. + +### Hive-Metastore CPU Usage + +When some of Qubership Hive-Metastore pods' CPU load is higher than `prometheusRules.alert.cpuThreshold`, Prometheus fires an alert. +The threshold is the percentage of the CPU limit specified for the pod. + +### Hive-Metastore Memory Usage + +When some of Qubership Hive-Metastore pods' memory usage is higher than `prometheusRules.alert.memoryThreshold`, Prometheus fires an alert. +The threshold is the percentage of the memory limit specified for the pod. + +### Hive-Metastore is Degraded + +If some of the Qubership Hive-Metastore pods go down, Prometheus fires an alert. + +### Hive-Metastore is Down + +If none of the Qubership Hive-Metastore pods are in the 'Running' phase, Prometheus fires an alert notifying that the operator is down. diff --git a/docs/public/security.md b/docs/public/security.md new file mode 100644 index 0000000..ea7f487 --- /dev/null +++ b/docs/public/security.md @@ -0,0 +1,27 @@ +This guide provides information about the main security parameters and its configuration in the Hive-Metastore service. + +## Exposed Ports + +List of ports used by Hive-Metastore are as follows: + +| Port | Service | Description | +|------|-------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 9028 | Hive-Metastore | Port for monitoring. | +| 9083 | Hive-Metastore | Service port. | + + +## Secure Protocols + +It is possible to enable TLS in Patform Hive-Metastore. This process is described in the respective [Installation](/docs/public/installation.md#enable-httpstls) guide. + +## Changing Credentials + +Qubership Hive-Metastore does not contain a user management mechanism and there is no ability to change the credentials in runtime. + +Credentials for the underlying services are managed in the underlying services. You can configure them in Qubership Hive-Metastore parameters. For more details refer to the [Hive Metastore Installation Procedure](/docs/public/installation.md#deployment-using-dppubhelm_deployer). + +There is no mechanism in Qubership Hive-Metastore to manage the credentials for Qubership Hive-Metastore. If the external system is used to manage Qubership Hive-Metastore users, the credentials can be managed there. + +## Session Management + +Qubership Hive-Metastore does not support session management. For these purposes, it is recommended to integrate Qubership Hive-Metastore with external user management systems. diff --git a/docs/sources/Hive-Metastore.drawio b/docs/sources/Hive-Metastore.drawio new file mode 100644 index 0000000..b2613f7 --- /dev/null +++ b/docs/sources/Hive-Metastore.drawio @@ -0,0 +1,468 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +