From aa64df9d06ddd46fe71d89930856c3b6d26adc11 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 14 Nov 2025 12:42:22 -0500 Subject: [PATCH 001/155] feat(deployment): Add k8s Helm chart for CLP package deployment. --- tools/deployment/package-helm/.helmignore | 23 ++ tools/deployment/package-helm/Chart.yaml | 17 ++ .../package-helm/templates/00-local-pvs.yaml | 25 ++ .../package-helm/templates/NOTES.txt | 1 + .../package-helm/templates/_helpers.tpl | 215 ++++++++++++++++++ .../compression-scheduler-deployment.yaml | 128 +++++++++++ .../compression-worker-deployment.yaml | 143 ++++++++++++ .../package-helm/templates/configmap.yaml | 150 ++++++++++++ .../templates/database-statefulset.yaml | 133 +++++++++++ .../templates/db-table-creator-job.yaml | 58 +++++ .../templates/query-scheduler-deployment.yaml | 121 ++++++++++ .../templates/query-worker-deployment.yaml | 118 ++++++++++ .../templates/queue-statefulset.yaml | 83 +++++++ .../templates/redis-statefulset.yaml | 110 +++++++++ .../results-cache-indices-creator-job.yaml | 39 ++++ .../templates/results-cache-statefulset.yaml | 121 ++++++++++ .../package-helm/templates/secret.yaml | 13 ++ .../templates/webui-deployment.yaml | 132 +++++++++++ tools/deployment/package-helm/test.sh | 37 +++ tools/deployment/package-helm/values.yaml | 193 ++++++++++++++++ 20 files changed, 1860 insertions(+) create mode 100644 tools/deployment/package-helm/.helmignore create mode 100644 tools/deployment/package-helm/Chart.yaml create mode 100644 tools/deployment/package-helm/templates/00-local-pvs.yaml create mode 100644 tools/deployment/package-helm/templates/NOTES.txt create mode 100644 tools/deployment/package-helm/templates/_helpers.tpl create mode 100644 tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/compression-worker-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/configmap.yaml create mode 100644 tools/deployment/package-helm/templates/database-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/db-table-creator-job.yaml create mode 100644 tools/deployment/package-helm/templates/query-scheduler-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/query-worker-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/queue-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/redis-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/secret.yaml create mode 100644 tools/deployment/package-helm/templates/webui-deployment.yaml create mode 100755 tools/deployment/package-helm/test.sh create mode 100644 tools/deployment/package-helm/values.yaml diff --git a/tools/deployment/package-helm/.helmignore b/tools/deployment/package-helm/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/tools/deployment/package-helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml new file mode 100644 index 0000000000..6621aa2842 --- /dev/null +++ b/tools/deployment/package-helm/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: "v2" +name: "clp-package" +description: "A Helm chart for CLP (Compressed Log Processor) package deployment" +type: "application" +version: "0.1.0" +appVersion: "0.6.1-dev" +home: "https://github.com/y-scope/clp" +icon: "https://github.com/y-scope/clp/raw/main/docs/src/clp-logo.png" +sources: ["https://github.com/y-scope/clp"] +maintainers: [{name: "YScope Inc.", email: "dev@yscope.com"}] +keywords: + - "analytics" + - "compression" + - "log-management" + - "log-parser" + - "logging" + - "search" diff --git a/tools/deployment/package-helm/templates/00-local-pvs.yaml b/tools/deployment/package-helm/templates/00-local-pvs.yaml new file mode 100644 index 0000000000..94da93965a --- /dev/null +++ b/tools/deployment/package-helm/templates/00-local-pvs.yaml @@ -0,0 +1,25 @@ +{{- include "clp-package.createLocalDataPv" (list . "database") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "database") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "queue") }} +--- +{{- include "clp-package.createLocalDataPv" (list . "redis") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "redis") }} +--- +{{- include "clp-package.createLocalDataPv" (list . "results-cache") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "results-cache") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "compression-scheduler") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "compression-worker") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "query-scheduler") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "query-worker") }} +--- +{{- include "clp-package.createStreamsPv" . }} +--- +{{- include "clp-package.createTmpPv" . }} diff --git a/tools/deployment/package-helm/templates/NOTES.txt b/tools/deployment/package-helm/templates/NOTES.txt new file mode 100644 index 0000000000..97a261da8a --- /dev/null +++ b/tools/deployment/package-helm/templates/NOTES.txt @@ -0,0 +1 @@ +TODO: This should be filled with usage instructions. \ No newline at end of file diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl new file mode 100644 index 0000000000..a02d98809e --- /dev/null +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -0,0 +1,215 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "clp-package.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "clp-package.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "clp-package.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "clp-package.labels" -}} +helm.sh/chart: {{ include "clp-package.chart" . }} +{{ include "clp-package.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "clp-package.selectorLabels" -}} +app.kubernetes.io/name: {{ include "clp-package.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Image reference for CLP Package +*/}} +{{- define "clp-package.image.ref" -}} +{{- $tag := .Values.image.clpPackage.tag | default .Chart.AppVersion }} +{{- printf "%s:%s" .Values.image.clpPackage.repository $tag }} +{{- end }} + +{{/* +Require a value with a standard error message +*/}} +{{- define "clp-package.requireInput" -}} +{{- . | required "Please set a value." }} +{{- end }} + +{{/* +Creates timings for readiness probes (faster checks for quicker startup). +*/}} +{{- define "clp-package.readinessProbeTimings" -}} +initialDelaySeconds: 1 +periodSeconds: 1 +timeoutSeconds: 2 +failureThreshold: 3 +{{- end }} + +{{/* +Creates timings for liveness probes. +*/}} +{{- define "clp-package.livenessProbeTimings" -}} +initialDelaySeconds: 30 +periodSeconds: 10 +timeoutSeconds: 5 +failureThreshold: 3 +{{- end }} + +{{/* +CLP logs directory path on host +*/}} +{{- define "clp-package.logsDirHost" -}} +{{ .Values.storage.localPathBase }}/{{ .Values.clpConfig.logs_directory }} +{{- end }} + +{{/* +CLP data directory path on host +*/}} +{{- define "clp-package.dataDirHost" -}} +{{ .Values.storage.localPathBase }}/{{ .Values.clpConfig.data_directory }} +{{- end }} + +{{/* +Creates a local PersistentVolume for data storage. +*/}} +{{- define "clp-package.createLocalDataPv" -}} +{{- $root := index . 0 -}} +{{- $component := index . 1 -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-data-pv-{{ $component }} + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: {{ $component | quote }} +spec: + capacity: + storage: "20Gi" + accessModes: ["ReadWriteOnce"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ include "clp-package.dataDirHost" $root }}/{{ $component }} + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + +{{/* +Creates a local PersistentVolume for logs storage. +*/}} +{{- define "clp-package.createLocalLogsPv" -}} +{{- $root := index . 0 -}} +{{- $component := index . 1 -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-logs-pv-{{ $component }} + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: {{ $component | quote }} +spec: + capacity: + storage: "5Gi" + accessModes: ["ReadWriteOnce"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ include "clp-package.logsDirHost" $root }}/{{ $component }} + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + +{{/* +Creates a local PersistentVolume for streams storage (ReadWriteMany). +*/}} +{{- define "clp-package.createStreamsPv" -}} +{{- $root := . -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-streams-pv + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: "streams" +spec: + capacity: + storage: "20Gi" + accessModes: ["ReadWriteMany"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ include "clp-package.dataDirHost" $root }}/streams + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + +{{/* +Creates a local PersistentVolume for tmp storage (ReadWriteMany). +*/}} +{{- define "clp-package.createTmpPv" -}} +{{- $root := . -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-tmp-pv + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: "tmp" +spec: + capacity: + storage: "10Gi" + accessModes: ["ReadWriteMany"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ .Values.storage.localPathBase }}/var/tmp + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml new file mode 100644 index 0000000000..458cc0dc96 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -0,0 +1,128 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-compression-scheduler + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" + ports: + - port: 7000 + targetPort: 7000 +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-compression-scheduler + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-scheduler" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "compression-scheduler" + spec: + terminationGracePeriodSeconds: 300 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + - name: "wait-for-queue" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + ] + - name: "wait-for-redis" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + ] + containers: + - name: "compression-scheduler" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "BROKER_URL" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/compression_scheduler" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + - name: "RESULT_BACKEND" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "compression-scheduler-logs" + mountPath: "/var/log/compression_scheduler" + - name: "logs-input" + mountPath: "/mnt/logs" + readOnly: true + command: [ + "python3", "-u", + "-m", "job_orchestration.scheduler.compress.compression_scheduler", + "--config", "/etc/clp-config.yml" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "compression-scheduler-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-compression-scheduler-logs + - name: "logs-input" + hostPath: + path: "{{ .Values.clpConfig.logs_input.directory }}" + type: "Directory" +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-compression-scheduler-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-scheduler" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml new file mode 100644 index 0000000000..3964667b6f --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -0,0 +1,143 @@ +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-compression-worker + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-worker" +spec: + replicas: {{ .Values.workerConcurrency }} + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-worker" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "compression-worker" + spec: + terminationGracePeriodSeconds: 60 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-queue" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + ] + - name: "wait-for-redis" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + ] + containers: + - name: "compression-worker" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "BROKER_URL" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + - name: "CLP_CONFIG_PATH" + value: "/etc/clp-config.yml" + - name: "CLP_HOME" + value: "/opt/clp" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.compression_worker.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/compression_worker" + - name: "CLP_WORKER_LOG_PATH" + value: "/var/log/compression_worker/worker.log" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + - name: "RESULT_BACKEND" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "compression-worker-logs" + mountPath: "/var/log/compression_worker" + - name: "archives" + mountPath: "/var/data/archives" + - name: "logs-input" + mountPath: "/mnt/logs" + readOnly: true + - name: "staged-archives" + mountPath: "/var/data/staged-archives" + - name: "tmp" + mountPath: "/var/tmp" + command: [ + "python3", "-u", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.compress", + "worker", + "--concurrency", "{{ .Values.workerConcurrency }}", + "--loglevel", "WARNING", + "-f", "/var/log/compression_worker/worker.log", + "-Q", "compression", + "-n", "compression-worker" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "compression-worker-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-compression-worker-logs + - name: "archives" + hostPath: + path: {{ include "clp-package.dataDirHost" . }}/archives + type: "DirectoryOrCreate" + - name: "logs-input" + hostPath: + path: "{{ .Values.clpConfig.logs_input.directory }}" + type: "Directory" + - name: "staged-archives" + hostPath: + path: {{ include "clp-package.dataDirHost" . }}/staged-archives + type: "DirectoryOrCreate" + - name: "tmp" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-tmp +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-compression-worker-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-worker" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-worker" + resources: + requests: + storage: "5Gi" +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-tmp + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "tmp" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "tmp" + resources: + requests: + storage: "10Gi" diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml new file mode 100644 index 0000000000..47aa6a0d5a --- /dev/null +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -0,0 +1,150 @@ +apiVersion: "v1" +kind: "ConfigMap" +metadata: + name: {{ include "clp-package.fullname" . }}-config + labels: + {{- include "clp-package.labels" . | nindent 4 }} +data: + clp-config.yml: | + api_server: + default_max_num_query_results: {{ .Values.clpConfig.api_server.default_max_num_query_results }} + host: {{ .Values.clpConfig.api_server.host | quote }} + port: {{ .Values.clpConfig.api_server.port }} + query_job_polling: + initial_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.initial_backoff_ms }} + max_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.max_backoff_ms }} + archive_output: + compression_level: {{ .Values.clpConfig.archive_output.compression_level }} + retention_period: {{ .Values.clpConfig.archive_output.retention_period }} + storage: + directory: "/var/data/archives" + type: "fs" + target_archive_size: {{ .Values.clpConfig.archive_output.target_archive_size }} + target_dictionaries_size: {{ .Values.clpConfig.archive_output.target_dictionaries_size }} + target_encoded_file_size: {{ .Values.clpConfig.archive_output.target_encoded_file_size }} + target_segment_size: {{ .Values.clpConfig.archive_output.target_segment_size }} + aws_config_directory: {{ .Values.clpConfig.aws_config_directory }} + compression_scheduler: + jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} + logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} + compression_worker: + logging_level: {{ .Values.clpConfig.compression_worker.logging_level | quote }} + data_directory: "/var/data" + database: + auto_commit: false + compress: true + host: {{ include "clp-package.fullname" . }}-database + name: {{ .Values.clpConfig.database.name | quote }} + port: {{ .Values.clpConfig.database.port }} + ssl_cert: null + type: {{ .Values.clpConfig.database.type | quote }} + garbage_collector: + logging_level: {{ .Values.clpConfig.garbage_collector.logging_level | quote }} + sweep_interval: + archive: {{ .Values.clpConfig.garbage_collector.sweep_interval.archive }} + search_result: {{ .Values.clpConfig.garbage_collector.sweep_interval.search_result }} + logs_directory: "/var/log" + logs_input: + directory: "/mnt/logs" + type: "fs" + mcp_server: {{ .Values.clpConfig.mcp_server }} + package: + query_engine: {{ .Values.clpConfig.package.query_engine | quote }} + storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} + presto: {{ .Values.clpConfig.presto }} + query_scheduler: + host: {{ include "clp-package.fullname" . }}-query-scheduler + jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }} + logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} + num_archives_to_search_per_sub_job: {{ .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job }} + port: {{ .Values.clpConfig.query_scheduler.port }} + query_worker: + logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }} + queue: + host: {{ include "clp-package.fullname" . }}-queue + port: {{ .Values.clpConfig.queue.port }} + redis: + compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} + host: {{ include "clp-package.fullname" . }}-redis + port: {{ .Values.clpConfig.redis.port }} + query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} + reducer: + base_port: {{ .Values.clpConfig.reducer.base_port }} + host: {{ include "clp-package.fullname" . }}-reducer + logging_level: {{ .Values.clpConfig.reducer.logging_level | quote }} + upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval }} + results_cache: + db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} + host: {{ include "clp-package.fullname" . }}-results-cache + port: {{ .Values.clpConfig.results_cache.port }} + retention_period: {{ .Values.clpConfig.results_cache.retention_period }} + stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} + stream_output: + storage: + directory: "/var/data/streams" + type: "fs" + target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size }} + tmp_directory: "/var/tmp" + webui: + host: {{ .Values.clpConfig.webui.host | quote }} + port: {{ .Values.clpConfig.webui.port }} + rate_limit: {{ .Values.clpConfig.webui.rate_limit }} + results_metadata_collection_name: {{ .Values.clpConfig.webui.results_metadata_collection_name | quote }} + mysql-logging.cnf: | + [mysqld] + # https://dev.mysql.com/doc/refman/8.0/en/log-destinations.html + # Enable the general log + general_log=1 + # Set the destination file + general_log_file=/var/log/mysql/mysql.log + + # https://dev.mysql.com/doc/refman/8.0/en/error-log-destination-configuration.html + # Send error logs to file + log_error=/var/log/mysql/mysql-error.log + redis.conf: | + # For full reference of configuration options see + # https://raw.githubusercontent.com/redis/redis/7.2/redis.conf + + databases 2 + logfile "/var/log/redis/redis.log" + mongod.conf: | + replication: + replSetName: "rs0" + systemLog: + destination: file + path: /var/log/mongodb/mongod.log + logAppend: true + timeStampFormat: iso8601-local + webui-client-settings.json: | + { + "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, + "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, + "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, + "SqlDbClpArchivesTableName": "", + "SqlDbClpDatasetsTableName": "clp_datasets", + "SqlDbClpFilesTableName": "", + "SqlDbClpTablePrefix": "clp_", + "SqlDbCompressionJobsTableName": "compression_jobs" + } + webui-server-settings.json: | + { + "SqlDbHost": "{{ include "clp-package.fullname" . }}-database", + "SqlDbPort": {{ .Values.clpConfig.database.port }}, + "SqlDbName": {{ .Values.clpConfig.database.name | quote }}, + "SqlDbQueryJobsTableName": "query_jobs", + "MongoDbHost": "{{ include "clp-package.fullname" . }}-results-cache", + "MongoDbPort": 27017, + "MongoDbName": {{ .Values.clpConfig.results_cache.db_name | quote }}, + "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, + "MongoDbStreamFilesCollectionName": "stream-files", + "ClientDir": "/opt/clp/var/www/webui/client", + "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer", + "StreamFilesDir": "/var/data/streams", + "StreamTargetUncompressedSize": 134217728, + "StreamFilesS3Region": null, + "StreamFilesS3PathPrefix": null, + "StreamFilesS3Profile": null, + "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, + "PrestoHost": null, + "PrestoPort": null + } diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml new file mode 100644 index 0000000000..3721691dce --- /dev/null +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -0,0 +1,133 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-database + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "database" + ports: + - port: 3306 + targetPort: 3306 +--- +{{- if .Values.allowSbinScripts }} +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-database-public + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +spec: + type: "NodePort" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "database" + ports: + - port: 3306 + nodePort: 30306 +--- +{{- end }} +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp-package.fullname" . }}-database + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +spec: + serviceName: {{ include "clp-package.fullname" . }}-database + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "database" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "database" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "database" + image: "mariadb:10-jammy" + imagePullPolicy: "Always" + env: + - name: "MYSQL_DATABASE" + value: {{ .Values.clpConfig.database.name | quote }} + - name: "MYSQL_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "MYSQL_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "MYSQL_ROOT_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + ports: + - name: "database" + containerPort: 3306 + volumeMounts: + - name: "database-data" + mountPath: "/var/lib/mysql" + - name: "database-logs" + mountPath: "/var/log/mysql" + - name: "config" + mountPath: "/etc/mysql/conf.d/logging.cnf" + subPath: "mysql-logging.cnf" + readOnly: true + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + exec: &database-health-check + command: [ + "mysqladmin", "ping", + "--silent", + "-h", "127.0.0.1" + ] + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + exec: + <<: *database-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + volumeClaimTemplates: + - metadata: + name: "database-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "database" + resources: + requests: + storage: "20Gi" + - metadata: + name: "database-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "database" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml new file mode 100644 index 0000000000..97dd731c67 --- /dev/null +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -0,0 +1,58 @@ +apiVersion: "batch/v1" +kind: "Job" +metadata: + name: {{ include "clp-package.fullname" . }}-db-table-creator + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "db-table-creator" +spec: + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "db-table-creator" + spec: + restartPolicy: "OnFailure" + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + containers: + - name: "db-table-creator" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + command: [ + "python3", "-u", + "-m", "clp_py_utils.create-db-tables", + "--config", "/etc/clp-config.yml", + "--storage-engine", {{ .Values.clpConfig.package.storage_engine | quote }} + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml new file mode 100644 index 0000000000..7104650c1c --- /dev/null +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -0,0 +1,121 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-query-scheduler + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" + ports: + - port: 7000 + targetPort: 7000 +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-query-scheduler + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "query-scheduler" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "query-scheduler" + spec: + terminationGracePeriodSeconds: 10 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + - name: "wait-for-queue" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + ] + - name: "wait-for-redis" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + ] + containers: + - name: "query-scheduler" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "BROKER_URL" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/query_scheduler" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + - name: "RESULT_BACKEND" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "query-scheduler-logs" + mountPath: "/var/log/query_scheduler" + command: [ + "python3", "-u", + "-m", "job_orchestration.scheduler.query.query_scheduler", + "--config", "/etc/clp-config.yml" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "query-scheduler-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-query-scheduler-logs +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-query-scheduler-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "query-scheduler" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml new file mode 100644 index 0000000000..c21a90380d --- /dev/null +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -0,0 +1,118 @@ +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-query-worker + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-worker" +spec: + replicas: {{ .Values.workerConcurrency }} + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "query-worker" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "query-worker" + spec: + terminationGracePeriodSeconds: 60 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-queue" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + ] + - name: "wait-for-redis" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + ] + containers: + - name: "query-worker" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "BROKER_URL" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + - name: "CLP_CONFIG_PATH" + value: "/etc/clp-config.yml" + - name: "CLP_HOME" + value: "/opt/clp" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.query_worker.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/query_worker" + - name: "CLP_WORKER_LOG_PATH" + value: "/var/log/query_worker/worker.log" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + - name: "RESULT_BACKEND" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "query-worker-logs" + mountPath: "/var/log/query_worker" + - name: "archives" + mountPath: "/var/data/archives" + - name: "staged-streams" + mountPath: "/var/data/staged-streams" + - name: "streams" + mountPath: "/var/data/streams" + command: [ + "python3", "-u", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.query", + "worker", + "--concurrency", "{{ .Values.workerConcurrency }}", + "--loglevel", "WARNING", + "-f", "/var/log/query_worker/worker.log", + "-Q", "query", + "-n", "query-worker" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "query-worker-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-query-worker-logs + - name: "archives" + hostPath: + path: {{ include "clp-package.dataDirHost" . }}/archives + type: "DirectoryOrCreate" + - name: "staged-streams" + hostPath: + path: {{ include "clp-package.dataDirHost" . }}/staged-streams + type: "DirectoryOrCreate" + - name: "streams" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-streams +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-query-worker-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-worker" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "query-worker" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml new file mode 100644 index 0000000000..f0f5cf1d60 --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -0,0 +1,83 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-queue + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "queue" + ports: + - port: 5672 + targetPort: 5672 +--- +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp-package.fullname" . }}-queue + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +spec: + serviceName: {{ include "clp-package.fullname" . }}-queue + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "queue" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "queue" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "queue" + image: "rabbitmq:3.9.8" + imagePullPolicy: "Always" + env: + - name: "RABBITMQ_DEFAULT_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "queue-username" + - name: "RABBITMQ_DEFAULT_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "queue-password" + - name: "RABBITMQ_LOGS" + value: "/var/log/rabbitmq/rabbitmq.log" + ports: + - containerPort: 5672 + volumeMounts: + - name: "queue-logs" + mountPath: "/var/log/rabbitmq" + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + exec: &queue-health-check + command: ["rabbitmq-diagnostics", "check_running"] + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + exec: + <<: *queue-health-check + volumeClaimTemplates: + - metadata: + name: "queue-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "queue" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml new file mode 100644 index 0000000000..56866947f0 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -0,0 +1,110 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-redis + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "redis" + ports: + - port: 6379 + targetPort: 6379 +--- +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp-package.fullname" . }}-redis + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +spec: + serviceName: {{ include "clp-package.fullname" . }}-redis + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "redis" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "redis" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "redis" + image: "redis:7.2.4" + imagePullPolicy: "Always" + command: [ + "redis-server", "/etc/redis/redis.conf", + "--requirepass", "$(REDIS_PASSWORD)" + ] + env: + - name: "REDIS_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "redis-password" + ports: + - containerPort: 6379 + volumeMounts: + - name: "redis-data" + mountPath: "/data" + - name: "redis-logs" + mountPath: "/var/log/redis" + - name: "config" + mountPath: "/etc/redis/redis.conf" + subPath: "redis.conf" + readOnly: true + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + exec: &redis-health-check + command: + - "redis-cli" + - "-h" + - "127.0.0.1" + - "-p" + - "6379" + - "-a" + - "$(REDIS_PASSWORD)" + - "PING" + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + exec: + <<: *redis-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + volumeClaimTemplates: + - metadata: + name: "redis-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "redis" + resources: + requests: + storage: "10Gi" + - metadata: + name: "redis-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "redis" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml new file mode 100644 index 0000000000..bdc479a54e --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -0,0 +1,39 @@ +apiVersion: "batch/v1" +kind: "Job" +metadata: + name: {{ include "clp-package.fullname" . }}-results-cache-indices-creator + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache-indices-creator" +spec: + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "results-cache-indices-creator" + spec: + restartPolicy: "OnFailure" + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-results-cache" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + ] + containers: + - name: "results-cache-indices-creator" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + command: [ + "python3", "-u", + "-m", "clp_py_utils.initialize-results-cache", + "--uri", "mongodb://{{ include "clp-package.fullname" . }}-results-cache:27017/{{ .Values.clpConfig.results_cache.db_name }}", + "--stream-collection", {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} + ] diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml new file mode 100644 index 0000000000..2673cb4ecd --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -0,0 +1,121 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-results-cache + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" + ports: + - port: 27017 + targetPort: 27017 +--- +{{- if .Values.allowSbinScripts }} +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-results-cache-public + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + type: "NodePort" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" + ports: + - port: 27017 + targetPort: 27017 + nodePort: 30017 +--- +{{- end }} +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp-package.fullname" . }}-results-cache + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + serviceName: {{ include "clp-package.fullname" . }}-results-cache + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "results-cache" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "results-cache" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "results-cache" + image: "mongo:7.0.1" + imagePullPolicy: "Always" + command: [ + "mongod", + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0" + ] + ports: + - containerPort: 27017 + volumeMounts: + - name: "results-cache-data" + mountPath: "/data/db" + - name: "results-cache-logs" + mountPath: "/var/log/mongodb" + - name: "config" + mountPath: "/etc/mongo/mongod.conf" + subPath: "mongod.conf" + readOnly: true + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + exec: &results-cache-health-check + command: + - "mongosh" + - "127.0.0.1:27017/test" + - "--quiet" + - "--eval" + - "db.runCommand('ping').ok" + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + exec: + <<: *results-cache-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + volumeClaimTemplates: + - metadata: + name: "results-cache-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "results-cache" + resources: + requests: + storage: "20Gi" + - metadata: + name: "results-cache-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "results-cache" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/secret.yaml b/tools/deployment/package-helm/templates/secret.yaml new file mode 100644 index 0000000000..b50867fcbc --- /dev/null +++ b/tools/deployment/package-helm/templates/secret.yaml @@ -0,0 +1,13 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp-package.fullname" . }}-secret +type: "Opaque" +stringData: + db-username: {{ .Values.credentials.database.user | include "clp-package.requireInput" | quote }} + db-password: {{ .Values.credentials.database.password | include "clp-package.requireInput" | quote }} + + queue-username: {{ .Values.credentials.queue.user | include "clp-package.requireInput" | quote }} + queue-password: {{ .Values.credentials.queue.password | include "clp-package.requireInput" | quote }} + + redis-password: {{ .Values.credentials.redis.password | include "clp-package.requireInput" | quote }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml new file mode 100644 index 0000000000..61d6f53bc6 --- /dev/null +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -0,0 +1,132 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-webui + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "webui" +spec: + type: "NodePort" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "webui" + ports: + - port: 4000 + targetPort: 4000 + nodePort: 30400 +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-webui + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "webui" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "webui" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "webui" + spec: + terminationGracePeriodSeconds: 60 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + - name: "wait-for-results-cache" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + ] + containers: + - name: "webui" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "HOST" + value: "0.0.0.0" + - name: "NODE_ENV" + value: "production" + - name: "NODE_PATH" + value: "/opt/clp/var/www/webui/server/node_modules" + - name: "PORT" + value: "4000" + - name: "RATE_LIMIT" + value: "1000" + ports: + - containerPort: 4000 + volumeMounts: + - name: "client-settings" + mountPath: "/opt/clp/var/www/webui/client/settings.json" + subPath: "webui-client-settings.json" + readOnly: true + - name: "server-settings" + mountPath: "/opt/clp/var/www/webui/server/dist/settings.json" + subPath: "webui-server-settings.json" + readOnly: true + - name: "streams" + mountPath: "/var/data/streams" + command: [ + "/opt/clp/bin/node-22", + "/opt/clp/var/www/webui/server/dist/src/main.js" + ] + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + tcpSocket: + port: 4000 + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + tcpSocket: + port: 4000 + volumes: + - name: "client-settings" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "server-settings" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "streams" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-streams +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-streams + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "streams" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "streams" + resources: + requests: + storage: "20Gi" diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh new file mode 100755 index 0000000000..e7985c8de8 --- /dev/null +++ b/tools/deployment/package-helm/test.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# TODO: to be deleted / migrated to integration tests + +kind delete cluster --name clp-test +sudo rm -rf /tmp/clp +mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker} +mkdir -p /tmp/clp/var/data/{archives,staged-streams,streams} +mkdir -p /tmp/clp/var/tmp +mkdir -p /tmp/clp/samples + +cat < Date: Sat, 15 Nov 2025 20:29:03 -0500 Subject: [PATCH 002/155] Add reducer service and deployment to Helm chart. --- .../package-helm/templates/00-local-pvs.yaml | 2 + .../templates/reducer-deployment.yaml | 110 ++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 tools/deployment/package-helm/templates/reducer-deployment.yaml diff --git a/tools/deployment/package-helm/templates/00-local-pvs.yaml b/tools/deployment/package-helm/templates/00-local-pvs.yaml index 94da93965a..7f57dd0ef7 100644 --- a/tools/deployment/package-helm/templates/00-local-pvs.yaml +++ b/tools/deployment/package-helm/templates/00-local-pvs.yaml @@ -20,6 +20,8 @@ --- {{- include "clp-package.createLocalLogsPv" (list . "query-worker") }} --- +{{- include "clp-package.createLocalLogsPv" (list . "reducer") }} +--- {{- include "clp-package.createStreamsPv" . }} --- {{- include "clp-package.createTmpPv" . }} diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml new file mode 100644 index 0000000000..869be59d5b --- /dev/null +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -0,0 +1,110 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-reducer + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" + ports: + {{- range $i := until (int .Values.workerConcurrency) }} + - name: "reducer-{{ $i }}" + port: {{ add $.Values.clpConfig.reducer.base_port $i }} + targetPort: {{ add $.Values.clpConfig.reducer.base_port $i }} + {{- end }} +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-reducer + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "reducer" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "reducer" + spec: + hostname: {{ include "clp-package.fullname" . }}-reducer + subdomain: {{ include "clp-package.fullname" . }}-reducer + terminationGracePeriodSeconds: 10 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-query-scheduler" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-query-scheduler 7000; do echo waiting for query-scheduler; sleep 2; done" + ] + - name: "wait-for-results-cache" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + ] + containers: + - name: "reducer" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_HOME" + value: "/opt/clp" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.reducer.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/reducer" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "reducer-logs" + mountPath: "/var/log/reducer" + command: [ + "python3", "-u", + "-m", "job_orchestration.reducer.reducer", + "--config", "/etc/clp-config.yml", + "--concurrency", "{{ .Values.workerConcurrency }}", + "--upsert-interval", "{{ .Values.clpConfig.reducer.upsert_interval }}" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "reducer-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-reducer-logs +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-reducer-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "reducer" + resources: + requests: + storage: "5Gi" From c0c9d02dbdf8d2ea7d4a46ff5138b181e3c13a08 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 20:39:17 -0500 Subject: [PATCH 003/155] Replace hostPath volumes with PersistentVolumeClaims in Helm chart; fix query scheduler hostname binding issue. --- .../package-helm/templates/00-local-pvs.yaml | 6 ++ .../package-helm/templates/_helpers.tpl | 84 +++++++++++++++++++ .../compression-worker-deployment.yaml | 46 ++++++++-- .../templates/query-scheduler-deployment.yaml | 2 + .../templates/query-worker-deployment.yaml | 28 +++++-- 5 files changed, 154 insertions(+), 12 deletions(-) diff --git a/tools/deployment/package-helm/templates/00-local-pvs.yaml b/tools/deployment/package-helm/templates/00-local-pvs.yaml index 7f57dd0ef7..96bff3f9a0 100644 --- a/tools/deployment/package-helm/templates/00-local-pvs.yaml +++ b/tools/deployment/package-helm/templates/00-local-pvs.yaml @@ -25,3 +25,9 @@ {{- include "clp-package.createStreamsPv" . }} --- {{- include "clp-package.createTmpPv" . }} +--- +{{- include "clp-package.createArchivesPv" . }} +--- +{{- include "clp-package.createStagedArchivesPv" . }} +--- +{{- include "clp-package.createStagedStreamsPv" . }} diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index a02d98809e..1763654a02 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -213,3 +213,87 @@ spec: operator: "Exists" {{- end }} +{{/* +Creates a local PersistentVolume for archives storage (ReadWriteMany). +*/}} +{{- define "clp-package.createArchivesPv" -}} +{{- $root := . -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-archives-pv + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: "archives" +spec: + capacity: + storage: "50Gi" + accessModes: ["ReadWriteMany"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ include "clp-package.dataDirHost" $root }}/archives + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + +{{/* +Creates a local PersistentVolume for staged-archives storage (ReadWriteMany). +*/}} +{{- define "clp-package.createStagedArchivesPv" -}} +{{- $root := . -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-staged-archives-pv + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: "staged-archives" +spec: + capacity: + storage: "20Gi" + accessModes: ["ReadWriteMany"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ include "clp-package.dataDirHost" $root }}/staged-archives + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + +{{/* +Creates a local PersistentVolume for staged-streams storage (ReadWriteMany). +*/}} +{{- define "clp-package.createStagedStreamsPv" -}} +{{- $root := . -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-staged-streams-pv + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: "staged-streams" +spec: + capacity: + storage: "20Gi" + accessModes: ["ReadWriteMany"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ include "clp-package.dataDirHost" $root }}/staged-streams + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 3964667b6f..2f9449fbe4 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -91,17 +91,15 @@ spec: persistentVolumeClaim: claimName: {{ include "clp-package.fullname" . }}-compression-worker-logs - name: "archives" - hostPath: - path: {{ include "clp-package.dataDirHost" . }}/archives - type: "DirectoryOrCreate" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-archives - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" type: "Directory" - name: "staged-archives" - hostPath: - path: {{ include "clp-package.dataDirHost" . }}/staged-archives - type: "DirectoryOrCreate" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-staged-archives - name: "tmp" persistentVolumeClaim: claimName: {{ include "clp-package.fullname" . }}-tmp @@ -126,6 +124,42 @@ spec: --- apiVersion: "v1" kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-archives + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "archives" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "archives" + resources: + requests: + storage: "50Gi" +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-staged-archives + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "staged-archives" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "staged-archives" + resources: + requests: + storage: "20Gi" +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" metadata: name: {{ include "clp-package.fullname" . }}-tmp labels: diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 7104650c1c..a71e33ef5d 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -33,6 +33,8 @@ spec: {{- include "clp-package.labels" . | nindent 8 }} app.kubernetes.io/component: "query-scheduler" spec: + hostname: {{ include "clp-package.fullname" . }}-query-scheduler + subdomain: {{ include "clp-package.fullname" . }}-query-scheduler terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index c21a90380d..83e9198010 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -88,13 +88,11 @@ spec: persistentVolumeClaim: claimName: {{ include "clp-package.fullname" . }}-query-worker-logs - name: "archives" - hostPath: - path: {{ include "clp-package.dataDirHost" . }}/archives - type: "DirectoryOrCreate" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-archives - name: "staged-streams" - hostPath: - path: {{ include "clp-package.dataDirHost" . }}/staged-streams - type: "DirectoryOrCreate" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-staged-streams - name: "streams" persistentVolumeClaim: claimName: {{ include "clp-package.fullname" . }}-streams @@ -116,3 +114,21 @@ spec: resources: requests: storage: "5Gi" +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-staged-streams + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "staged-streams" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "staged-streams" + resources: + requests: + storage: "20Gi" From 8bfd5e1469c31a38cdb96fa9849a582c70ff4ba4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 20:41:04 -0500 Subject: [PATCH 004/155] update test.sh --- tools/deployment/package-helm/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index e7985c8de8..85485148fd 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -4,7 +4,7 @@ kind delete cluster --name clp-test sudo rm -rf /tmp/clp -mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker} +mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker,reducer} mkdir -p /tmp/clp/var/data/{archives,staged-streams,streams} mkdir -p /tmp/clp/var/tmp mkdir -p /tmp/clp/samples From a3a1356cfa5498585de52bcf4af7e7579bbcb997 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 20:43:27 -0500 Subject: [PATCH 005/155] update test.sh --- tools/deployment/package-helm/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 85485148fd..9adbe38baa 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -5,7 +5,7 @@ kind delete cluster --name clp-test sudo rm -rf /tmp/clp mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker,reducer} -mkdir -p /tmp/clp/var/data/{archives,staged-streams,streams} +mkdir -p /tmp/clp/var/data/{archives,staged-archives,staged-streams,streams} mkdir -p /tmp/clp/var/tmp mkdir -p /tmp/clp/samples From 37000f699300c0a5734d3e6fdb55e88d9f99968c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 20:53:25 -0500 Subject: [PATCH 006/155] Update readiness probe timeoutSeconds to 1. --- tools/deployment/package-helm/templates/_helpers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 1763654a02..10479cd318 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -71,7 +71,7 @@ Creates timings for readiness probes (faster checks for quicker startup). {{- define "clp-package.readinessProbeTimings" -}} initialDelaySeconds: 1 periodSeconds: 1 -timeoutSeconds: 2 +timeoutSeconds: 1 failureThreshold: 3 {{- end }} From 34b3dca852aea281a2eb44cf518470d1a3e88c75 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 21:52:42 -0500 Subject: [PATCH 007/155] feat(deployment): Add deployments for MCP server, API server, and garbage collector. --- .../package-helm/templates/00-local-pvs.yaml | 8 ++ .../templates/api-server-deployment.yaml | 111 +++++++++++++++ .../garbage-collector-deployment.yaml | 107 +++++++++++++++ .../templates/mcp-server-deployment.yaml | 126 ++++++++++++++++++ tools/deployment/package-helm/test.sh | 8 +- 5 files changed, 359 insertions(+), 1 deletion(-) create mode 100644 tools/deployment/package-helm/templates/api-server-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/garbage-collector-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/mcp-server-deployment.yaml diff --git a/tools/deployment/package-helm/templates/00-local-pvs.yaml b/tools/deployment/package-helm/templates/00-local-pvs.yaml index 96bff3f9a0..0bd1f663ed 100644 --- a/tools/deployment/package-helm/templates/00-local-pvs.yaml +++ b/tools/deployment/package-helm/templates/00-local-pvs.yaml @@ -22,6 +22,14 @@ --- {{- include "clp-package.createLocalLogsPv" (list . "reducer") }} --- +{{- include "clp-package.createLocalLogsPv" (list . "garbage-collector") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "api-server") }} +--- +{{- if .Values.clpConfig.mcp_server }} +{{- include "clp-package.createLocalLogsPv" (list . "mcp-server") }} +--- +{{- end }} {{- include "clp-package.createStreamsPv" . }} --- {{- include "clp-package.createTmpPv" . }} diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml new file mode 100644 index 0000000000..be76af0b35 --- /dev/null +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -0,0 +1,111 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-api-server + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" +spec: + type: "NodePort" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" + ports: + - port: {{ .Values.clpConfig.api_server.port }} + targetPort: {{ .Values.clpConfig.api_server.port }} + nodePort: 30301 +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-api-server + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "api-server" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "api-server" + spec: + terminationGracePeriodSeconds: 10 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + - name: "wait-for-results-cache" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + ] + containers: + - name: "api-server" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "CLP_LOGS_DIR" + value: "/var/log/api_server" + - name: "RUST_LOG" + value: "DEBUG" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "api-server-logs" + mountPath: "/var/log/api_server" + command: [ + "/opt/clp/bin/api_server", + "--host", "0.0.0.0", + "--port", "{{ .Values.clpConfig.api_server.port }}", + "--config", "/etc/clp-config.yml" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "api-server-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-api-server-logs +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-api-server-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "api-server" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml new file mode 100644 index 0000000000..47b989b372 --- /dev/null +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -0,0 +1,107 @@ +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-garbage-collector + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "garbage-collector" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "garbage-collector" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "garbage-collector" + spec: + terminationGracePeriodSeconds: 10 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + - name: "wait-for-results-cache" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + ] + containers: + - name: "garbage-collector" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "CLP_HOME" + value: "/opt/clp" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.garbage_collector.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/garbage_collector" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "garbage-collector-logs" + mountPath: "/var/log/garbage_collector" + - name: "archives" + mountPath: "/var/data/archives" + - name: "streams" + mountPath: "/var/data/streams" + command: [ + "python3", "-u", + "-m", "job_orchestration.garbage_collector.garbage_collector", + "--config", "/etc/clp-config.yml" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "garbage-collector-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-garbage-collector-logs + - name: "archives" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-archives + - name: "streams" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-streams +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-garbage-collector-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "garbage-collector" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "garbage-collector" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml new file mode 100644 index 0000000000..c86fb16efd --- /dev/null +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -0,0 +1,126 @@ +{{- if .Values.clpConfig.mcp_server }} +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-mcp-server + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" +spec: + type: "NodePort" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" + ports: + - port: 8000 + targetPort: 8000 + nodePort: 30800 +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-mcp-server + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "mcp-server" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "mcp-server" + spec: + terminationGracePeriodSeconds: 10 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + - name: "wait-for-results-cache" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + ] + containers: + - name: "mcp-server" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.mcp_server.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/mcp_server" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "mcp-server-logs" + mountPath: "/var/log/mcp_server" + command: [ + "python3", "-u", + "-m", "clp_mcp_server.clp_mcp_server", + "--host", "0.0.0.0", + "--port", "8000", + "--config-path", "/etc/clp-config.yml" + ] + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + httpGet: + path: "/health" + port: 8000 + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + httpGet: + path: "/health" + port: 8000 + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "mcp-server-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-mcp-server-logs +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-mcp-server-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "mcp-server" + resources: + requests: + storage: "5Gi" +{{- end }} diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 9adbe38baa..33fbc61651 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -4,7 +4,7 @@ kind delete cluster --name clp-test sudo rm -rf /tmp/clp -mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker,reducer} +mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker,reducer,garbage-collector,api-server,mcp-server} mkdir -p /tmp/clp/var/data/{archives,staged-archives,staged-streams,streams} mkdir -p /tmp/clp/var/tmp mkdir -p /tmp/clp/samples @@ -27,6 +27,12 @@ cat < Date: Sat, 15 Nov 2025 21:55:03 -0500 Subject: [PATCH 008/155] feat(deployment): Add liveness and readiness probes to API server deployment. --- .../package-helm/templates/api-server-deployment.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index be76af0b35..c9e04643b3 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -84,6 +84,16 @@ spec: "--port", "{{ .Values.clpConfig.api_server.port }}", "--config", "/etc/clp-config.yml" ] + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + httpGet: + path: "/health" + port: {{ .Values.clpConfig.api_server.port }} + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + httpGet: + path: "/health" + port: {{ .Values.clpConfig.api_server.port }} volumes: - name: "config" configMap: From a1afc7f5d32bf3335709733185dd4c9ef523e8a2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 22:12:23 -0500 Subject: [PATCH 009/155] Simplify service templates with conditional NodePort configuration. --- .../templates/database-statefulset.yaml | 23 +++++------------- .../templates/results-cache-statefulset.yaml | 24 +++++-------------- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 3721691dce..456c0128d4 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -6,32 +6,21 @@ metadata: {{- include "clp-package.labels" . | nindent 4 }} app.kubernetes.io/component: "database" spec: + {{- if .Values.allowSbinScripts }} + type: "NodePort" + {{- else }} clusterIP: "None" + {{- end }} selector: {{- include "clp-package.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "database" ports: - port: 3306 targetPort: 3306 ---- -{{- if .Values.allowSbinScripts }} -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp-package.fullname" . }}-database-public - labels: - {{- include "clp-package.labels" . | nindent 4 }} - app.kubernetes.io/component: "database" -spec: - type: "NodePort" - selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "database" - ports: - - port: 3306 + {{- if .Values.allowSbinScripts }} nodePort: 30306 + {{- end }} --- -{{- end }} apiVersion: "apps/v1" kind: "StatefulSet" metadata: diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 2673cb4ecd..1e25c5493d 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -6,33 +6,21 @@ metadata: {{- include "clp-package.labels" . | nindent 4 }} app.kubernetes.io/component: "results-cache" spec: - clusterIP: "None" - selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "results-cache" - ports: - - port: 27017 - targetPort: 27017 ---- -{{- if .Values.allowSbinScripts }} -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp-package.fullname" . }}-results-cache-public - labels: - {{- include "clp-package.labels" . | nindent 4 }} - app.kubernetes.io/component: "results-cache" -spec: + {{- if .Values.allowSbinScripts }} type: "NodePort" + {{- else }} + clusterIP: "None" + {{- end }} selector: {{- include "clp-package.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "results-cache" ports: - port: 27017 targetPort: 27017 + {{- if .Values.allowSbinScripts }} nodePort: 30017 + {{- end }} --- -{{- end }} apiVersion: "apps/v1" kind: "StatefulSet" metadata: From e3d78a42bc49a8a55e031b5241f6b58444d2016e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 22:18:41 -0500 Subject: [PATCH 010/155] Add named ports in Helm templates for standardization. --- .../package-helm/templates/api-server-deployment.yaml | 5 ++++- .../package-helm/templates/database-statefulset.yaml | 2 +- .../package-helm/templates/mcp-server-deployment.yaml | 5 ++++- .../package-helm/templates/query-scheduler-deployment.yaml | 5 ++++- .../deployment/package-helm/templates/queue-statefulset.yaml | 5 +++-- .../deployment/package-helm/templates/redis-statefulset.yaml | 5 +++-- .../package-helm/templates/results-cache-statefulset.yaml | 5 +++-- .../deployment/package-helm/templates/webui-deployment.yaml | 5 +++-- 8 files changed, 25 insertions(+), 12 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index c9e04643b3..9294df9014 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -12,7 +12,7 @@ spec: app.kubernetes.io/component: "api-server" ports: - port: {{ .Values.clpConfig.api_server.port }} - targetPort: {{ .Values.clpConfig.api_server.port }} + targetPort: "http" nodePort: 30301 --- apiVersion: "apps/v1" @@ -71,6 +71,9 @@ spec: value: "/var/log/api_server" - name: "RUST_LOG" value: "DEBUG" + ports: + - name: "http" + containerPort: {{ .Values.clpConfig.api_server.port }} volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 456c0128d4..ab7848fa23 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -16,7 +16,7 @@ spec: app.kubernetes.io/component: "database" ports: - port: 3306 - targetPort: 3306 + targetPort: "database" {{- if .Values.allowSbinScripts }} nodePort: 30306 {{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index c86fb16efd..d00c37aad2 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -13,7 +13,7 @@ spec: app.kubernetes.io/component: "mcp-server" ports: - port: 8000 - targetPort: 8000 + targetPort: "http" nodePort: 30800 --- apiVersion: "apps/v1" @@ -74,6 +74,9 @@ spec: value: "/var/log/mcp_server" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" + ports: + - name: "http" + containerPort: 8000 volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index a71e33ef5d..ea193637dc 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -12,7 +12,7 @@ spec: app.kubernetes.io/component: "query-scheduler" ports: - port: 7000 - targetPort: 7000 + targetPort: "query-scheduler" --- apiVersion: "apps/v1" kind: "Deployment" @@ -84,6 +84,9 @@ spec: value: "/opt/clp/lib/python3/site-packages" - name: "RESULT_BACKEND" value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" + ports: + - name: "query-scheduler" + containerPort: 7000 volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index f0f5cf1d60..20ebf719a1 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -12,7 +12,7 @@ spec: app.kubernetes.io/component: "queue" ports: - port: 5672 - targetPort: 5672 + targetPort: "amqp" --- apiVersion: "apps/v1" kind: "StatefulSet" @@ -56,7 +56,8 @@ spec: - name: "RABBITMQ_LOGS" value: "/var/log/rabbitmq/rabbitmq.log" ports: - - containerPort: 5672 + - name: "amqp" + containerPort: 5672 volumeMounts: - name: "queue-logs" mountPath: "/var/log/rabbitmq" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 56866947f0..882c8b33db 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -12,7 +12,7 @@ spec: app.kubernetes.io/component: "redis" ports: - port: 6379 - targetPort: 6379 + targetPort: "redis" --- apiVersion: "apps/v1" kind: "StatefulSet" @@ -53,7 +53,8 @@ spec: name: {{ include "clp-package.fullname" . }}-secret key: "redis-password" ports: - - containerPort: 6379 + - name: "redis" + containerPort: 6379 volumeMounts: - name: "redis-data" mountPath: "/data" diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 1e25c5493d..4d45371858 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -16,7 +16,7 @@ spec: app.kubernetes.io/component: "results-cache" ports: - port: 27017 - targetPort: 27017 + targetPort: "mongodb" {{- if .Values.allowSbinScripts }} nodePort: 30017 {{- end }} @@ -55,7 +55,8 @@ spec: "--bind_ip", "0.0.0.0" ] ports: - - containerPort: 27017 + - name: "mongodb" + containerPort: 27017 volumeMounts: - name: "results-cache-data" mountPath: "/data/db" diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 61d6f53bc6..9f2f783683 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -12,7 +12,7 @@ spec: app.kubernetes.io/component: "webui" ports: - port: 4000 - targetPort: 4000 + targetPort: "http" nodePort: 30400 --- apiVersion: "apps/v1" @@ -78,7 +78,8 @@ spec: - name: "RATE_LIMIT" value: "1000" ports: - - containerPort: 4000 + - name: "http" + containerPort: 4000 volumeMounts: - name: "client-settings" mountPath: "/opt/clp/var/www/webui/client/settings.json" From f679f3803a085a70e34570fa95c5a1ff322ab2d0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 22:36:41 -0500 Subject: [PATCH 011/155] fix nodePort 's value --- .../templates/api-server-deployment.yaml | 12 ++++++------ .../package-helm/templates/configmap.yaml | 10 +++++----- .../templates/database-statefulset.yaml | 2 +- .../templates/mcp-server-deployment.yaml | 2 +- .../templates/results-cache-statefulset.yaml | 2 +- .../package-helm/templates/webui-deployment.yaml | 2 +- tools/deployment/package-helm/values.yaml | 13 +++++++++---- 7 files changed, 24 insertions(+), 19 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 9294df9014..e764af866d 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -11,9 +11,9 @@ spec: {{- include "clp-package.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "api-server" ports: - - port: {{ .Values.clpConfig.api_server.port }} + - port: 3001 targetPort: "http" - nodePort: 30301 + nodePort: {{ .Values.clpConfig.api_server.port }} --- apiVersion: "apps/v1" kind: "Deployment" @@ -73,7 +73,7 @@ spec: value: "DEBUG" ports: - name: "http" - containerPort: {{ .Values.clpConfig.api_server.port }} + containerPort: 3001 volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" @@ -84,19 +84,19 @@ spec: command: [ "/opt/clp/bin/api_server", "--host", "0.0.0.0", - "--port", "{{ .Values.clpConfig.api_server.port }}", + "--port", "3001", "--config", "/etc/clp-config.yml" ] livenessProbe: {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} httpGet: path: "/health" - port: {{ .Values.clpConfig.api_server.port }} + port: 3001 readinessProbe: {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} httpGet: path: "/health" - port: {{ .Values.clpConfig.api_server.port }} + port: 3001 volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 47aa6a0d5a..a5de1ff6b5 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -9,7 +9,7 @@ data: api_server: default_max_num_query_results: {{ .Values.clpConfig.api_server.default_max_num_query_results }} host: {{ .Values.clpConfig.api_server.host | quote }} - port: {{ .Values.clpConfig.api_server.port }} + port: 3001 query_job_polling: initial_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.initial_backoff_ms }} max_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.max_backoff_ms }} @@ -35,7 +35,7 @@ data: compress: true host: {{ include "clp-package.fullname" . }}-database name: {{ .Values.clpConfig.database.name | quote }} - port: {{ .Values.clpConfig.database.port }} + port: 3306 ssl_cert: null type: {{ .Values.clpConfig.database.type | quote }} garbage_collector: @@ -76,7 +76,7 @@ data: results_cache: db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} host: {{ include "clp-package.fullname" . }}-results-cache - port: {{ .Values.clpConfig.results_cache.port }} + port: 27017 retention_period: {{ .Values.clpConfig.results_cache.retention_period }} stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} stream_output: @@ -87,7 +87,7 @@ data: tmp_directory: "/var/tmp" webui: host: {{ .Values.clpConfig.webui.host | quote }} - port: {{ .Values.clpConfig.webui.port }} + port: 4000 rate_limit: {{ .Values.clpConfig.webui.rate_limit }} results_metadata_collection_name: {{ .Values.clpConfig.webui.results_metadata_collection_name | quote }} mysql-logging.cnf: | @@ -129,7 +129,7 @@ data: webui-server-settings.json: | { "SqlDbHost": "{{ include "clp-package.fullname" . }}-database", - "SqlDbPort": {{ .Values.clpConfig.database.port }}, + "SqlDbPort": 3306, "SqlDbName": {{ .Values.clpConfig.database.name | quote }}, "SqlDbQueryJobsTableName": "query_jobs", "MongoDbHost": "{{ include "clp-package.fullname" . }}-results-cache", diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index ab7848fa23..95fdc13f06 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -18,7 +18,7 @@ spec: - port: 3306 targetPort: "database" {{- if .Values.allowSbinScripts }} - nodePort: 30306 + nodePort: {{ .Values.clpConfig.database.port }} {{- end }} --- apiVersion: "apps/v1" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index d00c37aad2..143a57b3ca 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -14,7 +14,7 @@ spec: ports: - port: 8000 targetPort: "http" - nodePort: 30800 + nodePort: {{ .Values.clpConfig.mcp_server.port }} --- apiVersion: "apps/v1" kind: "Deployment" diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 4d45371858..010fde0c98 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -18,7 +18,7 @@ spec: - port: 27017 targetPort: "mongodb" {{- if .Values.allowSbinScripts }} - nodePort: 30017 + nodePort: {{ .Values.clpConfig.results_cache.port }} {{- end }} --- apiVersion: "apps/v1" diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 9f2f783683..965a54420f 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -13,7 +13,7 @@ spec: ports: - port: 4000 targetPort: "http" - nodePort: 30400 + nodePort: {{ .Values.clpConfig.webui.port }} --- apiVersion: "apps/v1" kind: "Deployment" diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 65a54d0e7a..d542d108db 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -44,7 +44,7 @@ clpConfig: database: type: "mariadb" # "mariadb" or "mysql" host: "localhost" - port: 3306 + port: 30306 name: "clp-db" compression_scheduler: @@ -76,7 +76,7 @@ clpConfig: results_cache: host: "localhost" - port: 27017 + port: 30017 db_name: "clp-query-results" stream_collection_name: "stream-files" @@ -91,11 +91,14 @@ clpConfig: webui: host: "localhost" - port: 4000 + port: 30400 results_metadata_collection_name: "results-metadata" rate_limit: 1000 mcp_server: null + # host: "localhost" + # port: 30800 + # logging_level: "INFO" # Where archives should be output to archive_output: @@ -148,7 +151,7 @@ clpConfig: # API server config api_server: host: "localhost" - port: 3001 + port: 30301 default_max_num_query_results: 1000 query_job_polling: initial_backoff_ms: 100 @@ -156,6 +159,8 @@ clpConfig: # Presto client config presto: null + # host: "localhost" + # port: 30889 # Location where other data (besides archives) are stored. It will be created if # it doesn't exist. From 3108ecad4b410b86938367438f10cb99795b9550 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 22:40:05 -0500 Subject: [PATCH 012/155] fix(helm): Update default port values in configmap for query scheduler, queue, and redis. --- tools/deployment/package-helm/templates/configmap.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index a5de1ff6b5..fa570eacf0 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -57,16 +57,16 @@ data: jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} num_archives_to_search_per_sub_job: {{ .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job }} - port: {{ .Values.clpConfig.query_scheduler.port }} + port: 7000 query_worker: logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }} queue: host: {{ include "clp-package.fullname" . }}-queue - port: {{ .Values.clpConfig.queue.port }} + port: 5672 redis: compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} host: {{ include "clp-package.fullname" . }}-redis - port: {{ .Values.clpConfig.redis.port }} + port: 6379 query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} reducer: base_port: {{ .Values.clpConfig.reducer.base_port }} From 597de93c7c9ce8396577c427d1e6943e9f8c2526 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 23:13:01 -0500 Subject: [PATCH 013/155] remove comment --- tools/deployment/package-helm/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index d542d108db..023ebb93cd 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -8,7 +8,6 @@ storage: # Base directory on host when using local PVs. localPathBase: "/tmp/clp" -# UID and GID for services securityContext: # UID/GID for first-party CLP services firstParty: From 9c68929e1eff024565331186af723f9fb00333bf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 23:23:55 -0500 Subject: [PATCH 014/155] Rename `user` to `username` across all components --- components/clp-package-utils/clp_package_utils/general.py | 4 ++-- components/clp-py-utils/clp_py_utils/clp_config.py | 4 ++-- .../package-template/src/etc/credentials.template.yml | 4 ++-- tools/deployment/package-helm/templates/secret.yaml | 4 ++-- tools/deployment/package-helm/values.yaml | 4 ++-- tools/deployment/presto-clp/scripts/init.py | 6 +++--- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a1c5fb5e15..d0ad00034c 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -453,8 +453,8 @@ def load_config_file( def generate_credentials_file(credentials_file_path: pathlib.Path): credentials = { - DB_COMPONENT_NAME: {"user": "clp-user", "password": secrets.token_urlsafe(8)}, - QUEUE_COMPONENT_NAME: {"user": "clp-user", "password": secrets.token_urlsafe(8)}, + DB_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)}, + QUEUE_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)}, REDIS_COMPONENT_NAME: {"password": secrets.token_urlsafe(16)}, } diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index c35a30dc0f..9e791754b0 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -232,7 +232,7 @@ def load_credentials_from_file(self, credentials_file_path: pathlib.Path): if config is None: raise ValueError(f"Credentials file '{credentials_file_path}' is empty.") try: - self.username = get_config_value(config, f"{DB_COMPONENT_NAME}.user") + self.username = get_config_value(config, f"{DB_COMPONENT_NAME}.username") self.password = get_config_value(config, f"{DB_COMPONENT_NAME}.password") except KeyError as ex: raise ValueError( @@ -360,7 +360,7 @@ def load_credentials_from_file(self, credentials_file_path: pathlib.Path): if config is None: raise ValueError(f"Credentials file '{credentials_file_path}' is empty.") try: - self.username = get_config_value(config, f"{QUEUE_COMPONENT_NAME}.user") + self.username = get_config_value(config, f"{QUEUE_COMPONENT_NAME}.username") self.password = get_config_value(config, f"{QUEUE_COMPONENT_NAME}.password") except KeyError as ex: raise ValueError( diff --git a/components/package-template/src/etc/credentials.template.yml b/components/package-template/src/etc/credentials.template.yml index 9a0dd73fa2..eb2f9b73f7 100644 --- a/components/package-template/src/etc/credentials.template.yml +++ b/components/package-template/src/etc/credentials.template.yml @@ -1,11 +1,11 @@ ## Database credentials #database: -# user: "user" +# username: "clp-user" # password: "pass" # ## Queue credentials #queue: -# user: "user" +# username: "clp-user" # password: "pass" # ## Redis credentials diff --git a/tools/deployment/package-helm/templates/secret.yaml b/tools/deployment/package-helm/templates/secret.yaml index b50867fcbc..145d85f568 100644 --- a/tools/deployment/package-helm/templates/secret.yaml +++ b/tools/deployment/package-helm/templates/secret.yaml @@ -4,10 +4,10 @@ metadata: name: {{ include "clp-package.fullname" . }}-secret type: "Opaque" stringData: - db-username: {{ .Values.credentials.database.user | include "clp-package.requireInput" | quote }} + db-username: {{ .Values.credentials.database.username | include "clp-package.requireInput" | quote }} db-password: {{ .Values.credentials.database.password | include "clp-package.requireInput" | quote }} - queue-username: {{ .Values.credentials.queue.user | include "clp-package.requireInput" | quote }} + queue-username: {{ .Values.credentials.queue.username | include "clp-package.requireInput" | quote }} queue-password: {{ .Values.credentials.queue.password | include "clp-package.requireInput" | quote }} redis-password: {{ .Values.credentials.redis.password | include "clp-package.requireInput" | quote }} diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 023ebb93cd..7a69d9aeb3 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -184,12 +184,12 @@ clpConfig: credentials: # Database credentials database: - user: "clp-user" + username: "clp-user" password: "pass" # Queue credentials queue: - user: "clp-user" + username: "clp-user" password: "pass" # Redis credentials diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py index 608f938c29..43a748f491 100644 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -164,15 +164,15 @@ def _add_clp_env_vars( credentials = yaml.load(credentials_file, Loader=yaml.CSafeLoader) try: - database_user = _get_required_config_value( - credentials, "database.user", credentials_file_path + database_username = _get_required_config_value( + credentials, "database.username", credentials_file_path ) database_password = _get_required_config_value( credentials, "database.password", credentials_file_path ) except KeyError: return False - env_vars["PRESTO_COORDINATOR_CLPPROPERTIES_METADATA_DATABASE_USER"] = database_user + env_vars["PRESTO_COORDINATOR_CLPPROPERTIES_METADATA_DATABASE_USER"] = database_username env_vars["PRESTO_COORDINATOR_CLPPROPERTIES_METADATA_DATABASE_PASSWORD"] = database_password instance_id = _get_clp_package_instance_id(clp_config, clp_package_dir) From 1f6a5f09c12e767803712061f9d8daf04933479d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 23:30:36 -0500 Subject: [PATCH 015/155] refactor(helm): Split secrets into component-specific files; remove requireInput --- .../deployment/package-helm/templates/_helpers.tpl | 7 ------- .../templates/api-server-deployment.yaml | 8 ++++---- .../templates/compression-scheduler-deployment.yaml | 8 ++++---- .../package-helm/templates/database-secret.yaml | 11 +++++++++++ .../templates/database-statefulset.yaml | 12 ++++++------ .../templates/db-table-creator-job.yaml | 8 ++++---- .../templates/garbage-collector-deployment.yaml | 8 ++++---- .../templates/mcp-server-deployment.yaml | 8 ++++---- .../templates/query-scheduler-deployment.yaml | 8 ++++---- .../package-helm/templates/queue-secret.yaml | 11 +++++++++++ .../package-helm/templates/queue-statefulset.yaml | 8 ++++---- .../package-helm/templates/redis-secret.yaml | 10 ++++++++++ .../package-helm/templates/redis-statefulset.yaml | 4 ++-- tools/deployment/package-helm/templates/secret.yaml | 13 ------------- .../package-helm/templates/webui-deployment.yaml | 8 ++++---- 15 files changed, 72 insertions(+), 60 deletions(-) create mode 100644 tools/deployment/package-helm/templates/database-secret.yaml create mode 100644 tools/deployment/package-helm/templates/queue-secret.yaml create mode 100644 tools/deployment/package-helm/templates/redis-secret.yaml delete mode 100644 tools/deployment/package-helm/templates/secret.yaml diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 10479cd318..db7c62bc1c 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -58,13 +58,6 @@ Image reference for CLP Package {{- printf "%s:%s" .Values.image.clpPackage.repository $tag }} {{- end }} -{{/* -Require a value with a standard error message -*/}} -{{- define "clp-package.requireInput" -}} -{{- . | required "Please set a value." }} -{{- end }} - {{/* Creates timings for readiness probes (faster checks for quicker startup). */}} diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index e764af866d..4085d5ccb7 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -60,13 +60,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "CLP_LOGS_DIR" value: "/var/log/api_server" - name: "RUST_LOG" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 458cc0dc96..df89829013 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -67,13 +67,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} - name: "CLP_LOGS_DIR" diff --git a/tools/deployment/package-helm/templates/database-secret.yaml b/tools/deployment/package-helm/templates/database-secret.yaml new file mode 100644 index 0000000000..8ed317c108 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp-package.fullname" . }}-database-secret + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +type: "Opaque" +stringData: + username: {{ .Values.credentials.database.username | quote }} + password: {{ .Values.credentials.database.password | quote }} diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 95fdc13f06..7a14774276 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -55,18 +55,18 @@ spec: - name: "MYSQL_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "MYSQL_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "MYSQL_ROOT_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" ports: - name: "database" containerPort: 3306 diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index 97dd731c67..367e8934ff 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -32,13 +32,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 47b989b372..d1a3bfdf7f 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -43,13 +43,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "CLP_HOME" value: "/opt/clp" - name: "CLP_LOGGING_LEVEL" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 143a57b3ca..0cb9f3da9f 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -61,13 +61,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.mcp_server.logging_level | quote }} - name: "CLP_LOGS_DIR" diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index ea193637dc..6fc9565df3 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -69,13 +69,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} - name: "CLP_LOGS_DIR" diff --git a/tools/deployment/package-helm/templates/queue-secret.yaml b/tools/deployment/package-helm/templates/queue-secret.yaml new file mode 100644 index 0000000000..8896941986 --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp-package.fullname" . }}-queue-secret + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +type: "Opaque" +stringData: + username: {{ .Values.credentials.queue.username | quote }} + password: {{ .Values.credentials.queue.password | quote }} diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index 20ebf719a1..631cc2e1c0 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -46,13 +46,13 @@ spec: - name: "RABBITMQ_DEFAULT_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "queue-username" + name: {{ include "clp-package.fullname" . }}-queue-secret + key: "username" - name: "RABBITMQ_DEFAULT_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "queue-password" + name: {{ include "clp-package.fullname" . }}-queue-secret + key: "password" - name: "RABBITMQ_LOGS" value: "/var/log/rabbitmq/rabbitmq.log" ports: diff --git a/tools/deployment/package-helm/templates/redis-secret.yaml b/tools/deployment/package-helm/templates/redis-secret.yaml new file mode 100644 index 0000000000..4b1dae946a --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp-package.fullname" . }}-redis-secret + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +type: "Opaque" +stringData: + password: {{ .Values.credentials.redis.password | quote }} diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 882c8b33db..204c9b99c8 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -50,8 +50,8 @@ spec: - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "redis-password" + name: {{ include "clp-package.fullname" . }}-redis-secret + key: "password" ports: - name: "redis" containerPort: 6379 diff --git a/tools/deployment/package-helm/templates/secret.yaml b/tools/deployment/package-helm/templates/secret.yaml deleted file mode 100644 index 145d85f568..0000000000 --- a/tools/deployment/package-helm/templates/secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: "v1" -kind: "Secret" -metadata: - name: {{ include "clp-package.fullname" . }}-secret -type: "Opaque" -stringData: - db-username: {{ .Values.credentials.database.username | include "clp-package.requireInput" | quote }} - db-password: {{ .Values.credentials.database.password | include "clp-package.requireInput" | quote }} - - queue-username: {{ .Values.credentials.queue.username | include "clp-package.requireInput" | quote }} - queue-password: {{ .Values.credentials.queue.password | include "clp-package.requireInput" | quote }} - - redis-password: {{ .Values.credentials.redis.password | include "clp-package.requireInput" | quote }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 965a54420f..f08022b2fb 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -60,13 +60,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "HOST" value: "0.0.0.0" - name: "NODE_ENV" From 89af6958d63e56a1faf854cf9660c9538a9934f4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 23:33:38 -0500 Subject: [PATCH 016/155] Replace `clp-package` references with `clp` across templates. --- tools/deployment/package-helm/Chart.yaml | 2 +- .../package-helm/templates/00-local-pvs.yaml | 40 +++++----- .../package-helm/templates/_helpers.tpl | 80 +++++++++---------- .../templates/api-server-deployment.yaml | 38 ++++----- .../compression-scheduler-deployment.yaml | 40 +++++----- .../compression-worker-deployment.yaml | 52 ++++++------ .../package-helm/templates/configmap.yaml | 20 ++--- .../templates/database-secret.yaml | 4 +- .../templates/database-statefulset.yaml | 32 ++++---- .../templates/db-table-creator-job.yaml | 16 ++-- .../garbage-collector-deployment.yaml | 32 ++++---- .../templates/mcp-server-deployment.yaml | 38 ++++----- .../templates/query-scheduler-deployment.yaml | 44 +++++----- .../templates/query-worker-deployment.yaml | 40 +++++----- .../package-helm/templates/queue-secret.yaml | 4 +- .../templates/queue-statefulset.yaml | 26 +++--- .../package-helm/templates/redis-secret.yaml | 4 +- .../templates/redis-statefulset.yaml | 28 +++---- .../templates/reducer-deployment.yaml | 34 ++++---- .../results-cache-indices-creator-job.yaml | 12 +-- .../templates/results-cache-statefulset.yaml | 26 +++--- .../templates/webui-deployment.yaml | 40 +++++----- 22 files changed, 326 insertions(+), 326 deletions(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 6621aa2842..035b371050 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: "v2" -name: "clp-package" +name: "clp" description: "A Helm chart for CLP (Compressed Log Processor) package deployment" type: "application" version: "0.1.0" diff --git a/tools/deployment/package-helm/templates/00-local-pvs.yaml b/tools/deployment/package-helm/templates/00-local-pvs.yaml index 0bd1f663ed..94ebc58cd7 100644 --- a/tools/deployment/package-helm/templates/00-local-pvs.yaml +++ b/tools/deployment/package-helm/templates/00-local-pvs.yaml @@ -1,41 +1,41 @@ -{{- include "clp-package.createLocalDataPv" (list . "database") }} +{{- include "clp.createLocalDataPv" (list . "database") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "database") }} +{{- include "clp.createLocalLogsPv" (list . "database") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "queue") }} +{{- include "clp.createLocalLogsPv" (list . "queue") }} --- -{{- include "clp-package.createLocalDataPv" (list . "redis") }} +{{- include "clp.createLocalDataPv" (list . "redis") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "redis") }} +{{- include "clp.createLocalLogsPv" (list . "redis") }} --- -{{- include "clp-package.createLocalDataPv" (list . "results-cache") }} +{{- include "clp.createLocalDataPv" (list . "results-cache") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "results-cache") }} +{{- include "clp.createLocalLogsPv" (list . "results-cache") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "compression-scheduler") }} +{{- include "clp.createLocalLogsPv" (list . "compression-scheduler") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "compression-worker") }} +{{- include "clp.createLocalLogsPv" (list . "compression-worker") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "query-scheduler") }} +{{- include "clp.createLocalLogsPv" (list . "query-scheduler") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "query-worker") }} +{{- include "clp.createLocalLogsPv" (list . "query-worker") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "reducer") }} +{{- include "clp.createLocalLogsPv" (list . "reducer") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "garbage-collector") }} +{{- include "clp.createLocalLogsPv" (list . "garbage-collector") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "api-server") }} +{{- include "clp.createLocalLogsPv" (list . "api-server") }} --- {{- if .Values.clpConfig.mcp_server }} -{{- include "clp-package.createLocalLogsPv" (list . "mcp-server") }} +{{- include "clp.createLocalLogsPv" (list . "mcp-server") }} --- {{- end }} -{{- include "clp-package.createStreamsPv" . }} +{{- include "clp.createStreamsPv" . }} --- -{{- include "clp-package.createTmpPv" . }} +{{- include "clp.createTmpPv" . }} --- -{{- include "clp-package.createArchivesPv" . }} +{{- include "clp.createArchivesPv" . }} --- -{{- include "clp-package.createStagedArchivesPv" . }} +{{- include "clp.createStagedArchivesPv" . }} --- -{{- include "clp-package.createStagedStreamsPv" . }} +{{- include "clp.createStagedStreamsPv" . }} diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index db7c62bc1c..3c6815ab24 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "clp-package.name" -}} +{{- define "clp.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "clp-package.fullname" -}} +{{- define "clp.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "clp-package.chart" -}} +{{- define "clp.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "clp-package.labels" -}} -helm.sh/chart: {{ include "clp-package.chart" . }} -{{ include "clp-package.selectorLabels" . }} +{{- define "clp.labels" -}} +helm.sh/chart: {{ include "clp.chart" . }} +{{ include "clp.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,15 +45,15 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "clp-package.selectorLabels" -}} -app.kubernetes.io/name: {{ include "clp-package.name" . }} +{{- define "clp.selectorLabels" -}} +app.kubernetes.io/name: {{ include "clp.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* Image reference for CLP Package */}} -{{- define "clp-package.image.ref" -}} +{{- define "clp.image.ref" -}} {{- $tag := .Values.image.clpPackage.tag | default .Chart.AppVersion }} {{- printf "%s:%s" .Values.image.clpPackage.repository $tag }} {{- end }} @@ -61,7 +61,7 @@ Image reference for CLP Package {{/* Creates timings for readiness probes (faster checks for quicker startup). */}} -{{- define "clp-package.readinessProbeTimings" -}} +{{- define "clp.readinessProbeTimings" -}} initialDelaySeconds: 1 periodSeconds: 1 timeoutSeconds: 1 @@ -71,7 +71,7 @@ failureThreshold: 3 {{/* Creates timings for liveness probes. */}} -{{- define "clp-package.livenessProbeTimings" -}} +{{- define "clp.livenessProbeTimings" -}} initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 @@ -81,29 +81,29 @@ failureThreshold: 3 {{/* CLP logs directory path on host */}} -{{- define "clp-package.logsDirHost" -}} +{{- define "clp.logsDirHost" -}} {{ .Values.storage.localPathBase }}/{{ .Values.clpConfig.logs_directory }} {{- end }} {{/* CLP data directory path on host */}} -{{- define "clp-package.dataDirHost" -}} +{{- define "clp.dataDirHost" -}} {{ .Values.storage.localPathBase }}/{{ .Values.clpConfig.data_directory }} {{- end }} {{/* Creates a local PersistentVolume for data storage. */}} -{{- define "clp-package.createLocalDataPv" -}} +{{- define "clp.createLocalDataPv" -}} {{- $root := index . 0 -}} {{- $component := index . 1 -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-data-pv-{{ $component }} + name: {{ include "clp.fullname" $root }}-data-pv-{{ $component }} labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: {{ $component | quote }} spec: capacity: @@ -112,7 +112,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp-package.dataDirHost" $root }}/{{ $component }} + path: {{ include "clp.dataDirHost" $root }}/{{ $component }} nodeAffinity: required: nodeSelectorTerms: @@ -124,15 +124,15 @@ spec: {{/* Creates a local PersistentVolume for logs storage. */}} -{{- define "clp-package.createLocalLogsPv" -}} +{{- define "clp.createLocalLogsPv" -}} {{- $root := index . 0 -}} {{- $component := index . 1 -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-logs-pv-{{ $component }} + name: {{ include "clp.fullname" $root }}-logs-pv-{{ $component }} labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: {{ $component | quote }} spec: capacity: @@ -141,7 +141,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp-package.logsDirHost" $root }}/{{ $component }} + path: {{ include "clp.logsDirHost" $root }}/{{ $component }} nodeAffinity: required: nodeSelectorTerms: @@ -153,14 +153,14 @@ spec: {{/* Creates a local PersistentVolume for streams storage (ReadWriteMany). */}} -{{- define "clp-package.createStreamsPv" -}} +{{- define "clp.createStreamsPv" -}} {{- $root := . -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-streams-pv + name: {{ include "clp.fullname" $root }}-streams-pv labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: "streams" spec: capacity: @@ -169,7 +169,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp-package.dataDirHost" $root }}/streams + path: {{ include "clp.dataDirHost" $root }}/streams nodeAffinity: required: nodeSelectorTerms: @@ -181,14 +181,14 @@ spec: {{/* Creates a local PersistentVolume for tmp storage (ReadWriteMany). */}} -{{- define "clp-package.createTmpPv" -}} +{{- define "clp.createTmpPv" -}} {{- $root := . -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-tmp-pv + name: {{ include "clp.fullname" $root }}-tmp-pv labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: "tmp" spec: capacity: @@ -209,14 +209,14 @@ spec: {{/* Creates a local PersistentVolume for archives storage (ReadWriteMany). */}} -{{- define "clp-package.createArchivesPv" -}} +{{- define "clp.createArchivesPv" -}} {{- $root := . -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-archives-pv + name: {{ include "clp.fullname" $root }}-archives-pv labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: "archives" spec: capacity: @@ -225,7 +225,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp-package.dataDirHost" $root }}/archives + path: {{ include "clp.dataDirHost" $root }}/archives nodeAffinity: required: nodeSelectorTerms: @@ -237,14 +237,14 @@ spec: {{/* Creates a local PersistentVolume for staged-archives storage (ReadWriteMany). */}} -{{- define "clp-package.createStagedArchivesPv" -}} +{{- define "clp.createStagedArchivesPv" -}} {{- $root := . -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-staged-archives-pv + name: {{ include "clp.fullname" $root }}-staged-archives-pv labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: "staged-archives" spec: capacity: @@ -253,7 +253,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp-package.dataDirHost" $root }}/staged-archives + path: {{ include "clp.dataDirHost" $root }}/staged-archives nodeAffinity: required: nodeSelectorTerms: @@ -265,14 +265,14 @@ spec: {{/* Creates a local PersistentVolume for staged-streams storage (ReadWriteMany). */}} -{{- define "clp-package.createStagedStreamsPv" -}} +{{- define "clp.createStagedStreamsPv" -}} {{- $root := . -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-staged-streams-pv + name: {{ include "clp.fullname" $root }}-staged-streams-pv labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: "staged-streams" spec: capacity: @@ -281,7 +281,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp-package.dataDirHost" $root }}/staged-streams + path: {{ include "clp.dataDirHost" $root }}/staged-streams nodeAffinity: required: nodeSelectorTerms: diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 4085d5ccb7..1f6752909e 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-api-server + name: {{ include "clp.fullname" . }}-api-server labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "api-server" spec: type: "NodePort" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "api-server" ports: - port: 3001 @@ -18,20 +18,20 @@ spec: apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-api-server + name: {{ include "clp.fullname" . }}-api-server labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "api-server" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "api-server" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "api-server" spec: terminationGracePeriodSeconds: 10 @@ -44,28 +44,28 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] - name: "wait-for-results-cache" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" ] containers: - name: "api-server" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "CLP_LOGS_DIR" value: "/var/log/api_server" @@ -88,36 +88,36 @@ spec: "--config", "/etc/clp-config.yml" ] livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} httpGet: path: "/health" port: 3001 readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} httpGet: path: "/health" port: 3001 volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "api-server-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-api-server-logs + claimName: {{ include "clp.fullname" . }}-api-server-logs --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-api-server-logs + name: {{ include "clp.fullname" . }}-api-server-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "api-server" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "api-server" resources: requests: diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index df89829013..fe43c9e896 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-compression-scheduler + name: {{ include "clp.fullname" . }}-compression-scheduler labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-scheduler" spec: clusterIP: "None" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "compression-scheduler" ports: - port: 7000 @@ -17,20 +17,20 @@ spec: apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-compression-scheduler + name: {{ include "clp.fullname" . }}-compression-scheduler labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-scheduler" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "compression-scheduler" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "compression-scheduler" spec: terminationGracePeriodSeconds: 300 @@ -43,36 +43,36 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] - name: "wait-for-queue" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" ] - name: "wait-for-redis" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" ] containers: - name: "compression-scheduler" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} @@ -81,7 +81,7 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" @@ -100,10 +100,10 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "compression-scheduler-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-compression-scheduler-logs + claimName: {{ include "clp.fullname" . }}-compression-scheduler-logs - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" @@ -112,16 +112,16 @@ spec: apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-compression-scheduler-logs + name: {{ include "clp.fullname" . }}-compression-scheduler-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-scheduler" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "compression-scheduler" resources: requests: diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 2f9449fbe4..6ce71cb371 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -1,20 +1,20 @@ apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-compression-worker + name: {{ include "clp.fullname" . }}-compression-worker labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-worker" spec: replicas: {{ .Values.workerConcurrency }} selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "compression-worker" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "compression-worker" spec: terminationGracePeriodSeconds: 60 @@ -27,21 +27,21 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" ] - name: "wait-for-redis" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" ] containers: - name: "compression-worker" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yml" - name: "CLP_HOME" @@ -55,7 +55,7 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" @@ -86,37 +86,37 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "compression-worker-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-compression-worker-logs + claimName: {{ include "clp.fullname" . }}-compression-worker-logs - name: "archives" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-archives + claimName: {{ include "clp.fullname" . }}-archives - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" type: "Directory" - name: "staged-archives" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-staged-archives + claimName: {{ include "clp.fullname" . }}-staged-archives - name: "tmp" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-tmp + claimName: {{ include "clp.fullname" . }}-tmp --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-compression-worker-logs + name: {{ include "clp.fullname" . }}-compression-worker-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-worker" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "compression-worker" resources: requests: @@ -125,16 +125,16 @@ spec: apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-archives + name: {{ include "clp.fullname" . }}-archives labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "archives" spec: accessModes: ["ReadWriteMany"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "archives" resources: requests: @@ -143,16 +143,16 @@ spec: apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-staged-archives + name: {{ include "clp.fullname" . }}-staged-archives labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "staged-archives" spec: accessModes: ["ReadWriteMany"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "staged-archives" resources: requests: @@ -161,16 +161,16 @@ spec: apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-tmp + name: {{ include "clp.fullname" . }}-tmp labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "tmp" spec: accessModes: ["ReadWriteMany"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "tmp" resources: requests: diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index fa570eacf0..cd94d9d489 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -1,9 +1,9 @@ apiVersion: "v1" kind: "ConfigMap" metadata: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} data: clp-config.yml: | api_server: @@ -33,7 +33,7 @@ data: database: auto_commit: false compress: true - host: {{ include "clp-package.fullname" . }}-database + host: {{ include "clp.fullname" . }}-database name: {{ .Values.clpConfig.database.name | quote }} port: 3306 ssl_cert: null @@ -53,7 +53,7 @@ data: storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} presto: {{ .Values.clpConfig.presto }} query_scheduler: - host: {{ include "clp-package.fullname" . }}-query-scheduler + host: {{ include "clp.fullname" . }}-query-scheduler jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} num_archives_to_search_per_sub_job: {{ .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job }} @@ -61,21 +61,21 @@ data: query_worker: logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }} queue: - host: {{ include "clp-package.fullname" . }}-queue + host: {{ include "clp.fullname" . }}-queue port: 5672 redis: compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} - host: {{ include "clp-package.fullname" . }}-redis + host: {{ include "clp.fullname" . }}-redis port: 6379 query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} reducer: base_port: {{ .Values.clpConfig.reducer.base_port }} - host: {{ include "clp-package.fullname" . }}-reducer + host: {{ include "clp.fullname" . }}-reducer logging_level: {{ .Values.clpConfig.reducer.logging_level | quote }} upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval }} results_cache: db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} - host: {{ include "clp-package.fullname" . }}-results-cache + host: {{ include "clp.fullname" . }}-results-cache port: 27017 retention_period: {{ .Values.clpConfig.results_cache.retention_period }} stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} @@ -128,11 +128,11 @@ data: } webui-server-settings.json: | { - "SqlDbHost": "{{ include "clp-package.fullname" . }}-database", + "SqlDbHost": "{{ include "clp.fullname" . }}-database", "SqlDbPort": 3306, "SqlDbName": {{ .Values.clpConfig.database.name | quote }}, "SqlDbQueryJobsTableName": "query_jobs", - "MongoDbHost": "{{ include "clp-package.fullname" . }}-results-cache", + "MongoDbHost": "{{ include "clp.fullname" . }}-results-cache", "MongoDbPort": 27017, "MongoDbName": {{ .Values.clpConfig.results_cache.db_name | quote }}, "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, diff --git a/tools/deployment/package-helm/templates/database-secret.yaml b/tools/deployment/package-helm/templates/database-secret.yaml index 8ed317c108..f2814d6877 100644 --- a/tools/deployment/package-helm/templates/database-secret.yaml +++ b/tools/deployment/package-helm/templates/database-secret.yaml @@ -1,9 +1,9 @@ apiVersion: "v1" kind: "Secret" metadata: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "database" type: "Opaque" stringData: diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 7a14774276..ca2796fc04 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -1,9 +1,9 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-database + name: {{ include "clp.fullname" . }}-database labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "database" spec: {{- if .Values.allowSbinScripts }} @@ -12,7 +12,7 @@ spec: clusterIP: "None" {{- end }} selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "database" ports: - port: 3306 @@ -24,21 +24,21 @@ spec: apiVersion: "apps/v1" kind: "StatefulSet" metadata: - name: {{ include "clp-package.fullname" . }}-database + name: {{ include "clp.fullname" . }}-database labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "database" spec: - serviceName: {{ include "clp-package.fullname" . }}-database + serviceName: {{ include "clp.fullname" . }}-database replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "database" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "database" spec: securityContext: @@ -55,17 +55,17 @@ spec: - name: "MYSQL_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "MYSQL_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "MYSQL_ROOT_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" ports: - name: "database" @@ -80,7 +80,7 @@ spec: subPath: "mysql-logging.cnf" readOnly: true livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: &database-health-check command: [ "mysqladmin", "ping", @@ -88,13 +88,13 @@ spec: "-h", "127.0.0.1" ] readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: <<: *database-health-check volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config volumeClaimTemplates: - metadata: name: "database-data" @@ -103,7 +103,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "database" resources: requests: @@ -115,7 +115,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "database" resources: requests: diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index 367e8934ff..f8f89a260a 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -1,15 +1,15 @@ apiVersion: "batch/v1" kind: "Job" metadata: - name: {{ include "clp-package.fullname" . }}-db-table-creator + name: {{ include "clp.fullname" . }}-db-table-creator labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "db-table-creator" spec: template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "db-table-creator" spec: restartPolicy: "OnFailure" @@ -22,22 +22,22 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] containers: - name: "db-table-creator" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" @@ -55,4 +55,4 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index d1a3bfdf7f..483946dd36 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -1,20 +1,20 @@ apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-garbage-collector + name: {{ include "clp.fullname" . }}-garbage-collector labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "garbage-collector" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "garbage-collector" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "garbage-collector" spec: terminationGracePeriodSeconds: 10 @@ -27,28 +27,28 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] - name: "wait-for-results-cache" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" ] containers: - name: "garbage-collector" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "CLP_HOME" value: "/opt/clp" @@ -77,30 +77,30 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "garbage-collector-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-garbage-collector-logs + claimName: {{ include "clp.fullname" . }}-garbage-collector-logs - name: "archives" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-archives + claimName: {{ include "clp.fullname" . }}-archives - name: "streams" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-streams + claimName: {{ include "clp.fullname" . }}-streams --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-garbage-collector-logs + name: {{ include "clp.fullname" . }}-garbage-collector-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "garbage-collector" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "garbage-collector" resources: requests: diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 0cb9f3da9f..e3de1b3550 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -2,14 +2,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-mcp-server + name: {{ include "clp.fullname" . }}-mcp-server labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "mcp-server" spec: type: "NodePort" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "mcp-server" ports: - port: 8000 @@ -19,20 +19,20 @@ spec: apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-mcp-server + name: {{ include "clp.fullname" . }}-mcp-server labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "mcp-server" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "mcp-server" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "mcp-server" spec: terminationGracePeriodSeconds: 10 @@ -45,28 +45,28 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] - name: "wait-for-results-cache" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" ] containers: - name: "mcp-server" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.mcp_server.logging_level | quote }} @@ -92,36 +92,36 @@ spec: "--config-path", "/etc/clp-config.yml" ] livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} httpGet: path: "/health" port: 8000 readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} httpGet: path: "/health" port: 8000 volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "mcp-server-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-mcp-server-logs + claimName: {{ include "clp.fullname" . }}-mcp-server-logs --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-mcp-server-logs + name: {{ include "clp.fullname" . }}-mcp-server-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "mcp-server" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "mcp-server" resources: requests: diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 6fc9565df3..97f1d133f9 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-query-scheduler + name: {{ include "clp.fullname" . }}-query-scheduler labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-scheduler" spec: clusterIP: "None" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "query-scheduler" ports: - port: 7000 @@ -17,24 +17,24 @@ spec: apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-query-scheduler + name: {{ include "clp.fullname" . }}-query-scheduler labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-scheduler" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "query-scheduler" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "query-scheduler" spec: - hostname: {{ include "clp-package.fullname" . }}-query-scheduler - subdomain: {{ include "clp-package.fullname" . }}-query-scheduler + hostname: {{ include "clp.fullname" . }}-query-scheduler + subdomain: {{ include "clp.fullname" . }}-query-scheduler terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} @@ -45,36 +45,36 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] - name: "wait-for-queue" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" ] - name: "wait-for-redis" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" ] containers: - name: "query-scheduler" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} @@ -83,7 +83,7 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" ports: - name: "query-scheduler" containerPort: 7000 @@ -102,24 +102,24 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "query-scheduler-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-query-scheduler-logs + claimName: {{ include "clp.fullname" . }}-query-scheduler-logs --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-query-scheduler-logs + name: {{ include "clp.fullname" . }}-query-scheduler-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-scheduler" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "query-scheduler" resources: requests: diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 83e9198010..1de4df841d 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -1,20 +1,20 @@ apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-query-worker + name: {{ include "clp.fullname" . }}-query-worker labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-worker" spec: replicas: {{ .Values.workerConcurrency }} selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "query-worker" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "query-worker" spec: terminationGracePeriodSeconds: 60 @@ -27,21 +27,21 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" ] - name: "wait-for-redis" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" ] containers: - name: "query-worker" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yml" - name: "CLP_HOME" @@ -55,7 +55,7 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" @@ -83,33 +83,33 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "query-worker-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-query-worker-logs + claimName: {{ include "clp.fullname" . }}-query-worker-logs - name: "archives" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-archives + claimName: {{ include "clp.fullname" . }}-archives - name: "staged-streams" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-staged-streams + claimName: {{ include "clp.fullname" . }}-staged-streams - name: "streams" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-streams + claimName: {{ include "clp.fullname" . }}-streams --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-query-worker-logs + name: {{ include "clp.fullname" . }}-query-worker-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-worker" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "query-worker" resources: requests: @@ -118,16 +118,16 @@ spec: apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-staged-streams + name: {{ include "clp.fullname" . }}-staged-streams labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "staged-streams" spec: accessModes: ["ReadWriteMany"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "staged-streams" resources: requests: diff --git a/tools/deployment/package-helm/templates/queue-secret.yaml b/tools/deployment/package-helm/templates/queue-secret.yaml index 8896941986..8b909ce22a 100644 --- a/tools/deployment/package-helm/templates/queue-secret.yaml +++ b/tools/deployment/package-helm/templates/queue-secret.yaml @@ -1,9 +1,9 @@ apiVersion: "v1" kind: "Secret" metadata: - name: {{ include "clp-package.fullname" . }}-queue-secret + name: {{ include "clp.fullname" . }}-queue-secret labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "queue" type: "Opaque" stringData: diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index 631cc2e1c0..93c0733b27 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-queue + name: {{ include "clp.fullname" . }}-queue labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "queue" spec: clusterIP: "None" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "queue" ports: - port: 5672 @@ -17,21 +17,21 @@ spec: apiVersion: "apps/v1" kind: "StatefulSet" metadata: - name: {{ include "clp-package.fullname" . }}-queue + name: {{ include "clp.fullname" . }}-queue labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "queue" spec: - serviceName: {{ include "clp-package.fullname" . }}-queue + serviceName: {{ include "clp.fullname" . }}-queue replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "queue" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "queue" spec: securityContext: @@ -46,12 +46,12 @@ spec: - name: "RABBITMQ_DEFAULT_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-queue-secret + name: {{ include "clp.fullname" . }}-queue-secret key: "username" - name: "RABBITMQ_DEFAULT_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-queue-secret + name: {{ include "clp.fullname" . }}-queue-secret key: "password" - name: "RABBITMQ_LOGS" value: "/var/log/rabbitmq/rabbitmq.log" @@ -62,11 +62,11 @@ spec: - name: "queue-logs" mountPath: "/var/log/rabbitmq" livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: &queue-health-check command: ["rabbitmq-diagnostics", "check_running"] readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: <<: *queue-health-check volumeClaimTemplates: @@ -77,7 +77,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "queue" resources: requests: diff --git a/tools/deployment/package-helm/templates/redis-secret.yaml b/tools/deployment/package-helm/templates/redis-secret.yaml index 4b1dae946a..4923e0ce5d 100644 --- a/tools/deployment/package-helm/templates/redis-secret.yaml +++ b/tools/deployment/package-helm/templates/redis-secret.yaml @@ -1,9 +1,9 @@ apiVersion: "v1" kind: "Secret" metadata: - name: {{ include "clp-package.fullname" . }}-redis-secret + name: {{ include "clp.fullname" . }}-redis-secret labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "redis" type: "Opaque" stringData: diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 204c9b99c8..5db313885d 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-redis + name: {{ include "clp.fullname" . }}-redis labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "redis" spec: clusterIP: "None" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "redis" ports: - port: 6379 @@ -17,21 +17,21 @@ spec: apiVersion: "apps/v1" kind: "StatefulSet" metadata: - name: {{ include "clp-package.fullname" . }}-redis + name: {{ include "clp.fullname" . }}-redis labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "redis" spec: - serviceName: {{ include "clp-package.fullname" . }}-redis + serviceName: {{ include "clp.fullname" . }}-redis replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "redis" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "redis" spec: securityContext: @@ -50,7 +50,7 @@ spec: - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-redis-secret + name: {{ include "clp.fullname" . }}-redis-secret key: "password" ports: - name: "redis" @@ -65,7 +65,7 @@ spec: subPath: "redis.conf" readOnly: true livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: &redis-health-check command: - "redis-cli" @@ -77,13 +77,13 @@ spec: - "$(REDIS_PASSWORD)" - "PING" readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: <<: *redis-health-check volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config volumeClaimTemplates: - metadata: name: "redis-data" @@ -92,7 +92,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "redis" resources: requests: @@ -104,7 +104,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "redis" resources: requests: diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index 869be59d5b..db4a7af1c3 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-reducer + name: {{ include "clp.fullname" . }}-reducer labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "reducer" spec: clusterIP: "None" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "reducer" ports: {{- range $i := until (int .Values.workerConcurrency) }} @@ -20,24 +20,24 @@ spec: apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-reducer + name: {{ include "clp.fullname" . }}-reducer labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "reducer" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "reducer" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "reducer" spec: - hostname: {{ include "clp-package.fullname" . }}-reducer - subdomain: {{ include "clp-package.fullname" . }}-reducer + hostname: {{ include "clp.fullname" . }}-reducer + subdomain: {{ include "clp.fullname" . }}-reducer terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} @@ -48,17 +48,17 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-query-scheduler 7000; do echo waiting for query-scheduler; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-query-scheduler 7000; do echo waiting for query-scheduler; sleep 2; done" ] - name: "wait-for-results-cache" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" ] containers: - name: "reducer" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "CLP_HOME" @@ -86,24 +86,24 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "reducer-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-reducer-logs + claimName: {{ include "clp.fullname" . }}-reducer-logs --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-reducer-logs + name: {{ include "clp.fullname" . }}-reducer-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "reducer" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "reducer" resources: requests: diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index bdc479a54e..4b45368f3f 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -1,15 +1,15 @@ apiVersion: "batch/v1" kind: "Job" metadata: - name: {{ include "clp-package.fullname" . }}-results-cache-indices-creator + name: {{ include "clp.fullname" . }}-results-cache-indices-creator labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "results-cache-indices-creator" spec: template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "results-cache-indices-creator" spec: restartPolicy: "OnFailure" @@ -22,11 +22,11 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" ] containers: - name: "results-cache-indices-creator" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "PYTHONPATH" @@ -34,6 +34,6 @@ spec: command: [ "python3", "-u", "-m", "clp_py_utils.initialize-results-cache", - "--uri", "mongodb://{{ include "clp-package.fullname" . }}-results-cache:27017/{{ .Values.clpConfig.results_cache.db_name }}", + "--uri", "mongodb://{{ include "clp.fullname" . }}-results-cache:27017/{{ .Values.clpConfig.results_cache.db_name }}", "--stream-collection", {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} ] diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 010fde0c98..c4453cd9d0 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -1,9 +1,9 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-results-cache + name: {{ include "clp.fullname" . }}-results-cache labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "results-cache" spec: {{- if .Values.allowSbinScripts }} @@ -12,7 +12,7 @@ spec: clusterIP: "None" {{- end }} selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "results-cache" ports: - port: 27017 @@ -24,21 +24,21 @@ spec: apiVersion: "apps/v1" kind: "StatefulSet" metadata: - name: {{ include "clp-package.fullname" . }}-results-cache + name: {{ include "clp.fullname" . }}-results-cache labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "results-cache" spec: - serviceName: {{ include "clp-package.fullname" . }}-results-cache + serviceName: {{ include "clp.fullname" . }}-results-cache replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "results-cache" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "results-cache" spec: securityContext: @@ -67,7 +67,7 @@ spec: subPath: "mongod.conf" readOnly: true livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: &results-cache-health-check command: - "mongosh" @@ -76,13 +76,13 @@ spec: - "--eval" - "db.runCommand('ping').ok" readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: <<: *results-cache-health-check volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config volumeClaimTemplates: - metadata: name: "results-cache-data" @@ -91,7 +91,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "results-cache" resources: requests: @@ -103,7 +103,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "results-cache" resources: requests: diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index f08022b2fb..873e56c8f7 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-webui + name: {{ include "clp.fullname" . }}-webui labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "webui" spec: type: "NodePort" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "webui" ports: - port: 4000 @@ -18,20 +18,20 @@ spec: apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-webui + name: {{ include "clp.fullname" . }}-webui labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "webui" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "webui" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "webui" spec: terminationGracePeriodSeconds: 60 @@ -44,28 +44,28 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] - name: "wait-for-results-cache" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" ] containers: - name: "webui" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "HOST" value: "0.0.0.0" @@ -96,37 +96,37 @@ spec: "/opt/clp/var/www/webui/server/dist/src/main.js" ] livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: port: 4000 readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} tcpSocket: port: 4000 volumes: - name: "client-settings" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "server-settings" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "streams" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-streams + claimName: {{ include "clp.fullname" . }}-streams --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-streams + name: {{ include "clp.fullname" . }}-streams labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "streams" spec: accessModes: ["ReadWriteMany"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "streams" resources: requests: From ca7d8200b854c2070c1bbce5b14b842fcd680e58 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 00:00:33 -0500 Subject: [PATCH 017/155] split yamls --- .../package-helm/templates/00-local-pvs.yaml | 41 ----------- .../templates/api-server-deployment.yaml | 35 --------- .../templates/api-server-logs-pv.yaml | 1 + .../templates/api-server-pvc.yaml | 17 +++++ .../templates/api-server-service.yaml | 16 +++++ .../package-helm/templates/archives-pv.yaml | 1 + .../compression-scheduler-deployment.yaml | 34 --------- .../compression-scheduler-logs-pv.yaml | 1 + .../templates/compression-scheduler-pvc.yaml | 17 +++++ .../compression-scheduler-service.yaml | 15 ++++ .../compression-worker-archives-pvc.yaml | 17 +++++ .../compression-worker-deployment.yaml | 72 ------------------- .../templates/compression-worker-logs-pv.yaml | 1 + .../compression-worker-logs-pvc.yaml | 17 +++++ ...ompression-worker-staged-archives-pvc.yaml | 17 +++++ .../templates/database-logs-pv.yaml | 1 + .../package-helm/templates/database-pv.yaml | 1 + .../templates/database-service.yaml | 22 ++++++ .../templates/database-statefulset.yaml | 23 ------ .../garbage-collector-deployment.yaml | 18 ----- .../templates/garbage-collector-logs-pv.yaml | 1 + .../templates/garbage-collector-pvc.yaml | 17 +++++ .../templates/mcp-server-deployment.yaml | 37 ---------- .../templates/mcp-server-logs-pv.yaml | 3 + .../templates/mcp-server-pvc.yaml | 18 +++++ .../templates/mcp-server-service.yaml | 17 +++++ .../templates/query-scheduler-deployment.yaml | 34 --------- .../templates/query-scheduler-logs-pv.yaml | 1 + .../templates/query-scheduler-pvc.yaml | 17 +++++ .../templates/query-scheduler-service.yaml | 15 ++++ .../templates/query-worker-archives-pvc.yaml | 17 +++++ .../templates/query-worker-deployment.yaml | 36 ---------- .../templates/query-worker-logs-pv.yaml | 1 + .../templates/query-worker-logs-pvc.yaml | 17 +++++ .../query-worker-staged-streams-pvc.yaml | 17 +++++ .../package-helm/templates/queue-logs-pv.yaml | 1 + .../package-helm/templates/queue-service.yaml | 15 ++++ .../templates/queue-statefulset.yaml | 16 ----- .../package-helm/templates/redis-logs-pv.yaml | 1 + .../package-helm/templates/redis-pv.yaml | 1 + .../package-helm/templates/redis-service.yaml | 15 ++++ .../templates/redis-statefulset.yaml | 16 ----- .../templates/reducer-deployment.yaml | 37 ---------- .../templates/reducer-logs-pv.yaml | 1 + .../package-helm/templates/reducer-pvc.yaml | 17 +++++ .../templates/reducer-service.yaml | 18 +++++ .../templates/results-cache-logs-pv.yaml | 1 + .../templates/results-cache-pv.yaml | 1 + .../templates/results-cache-service.yaml | 22 ++++++ .../templates/results-cache-statefulset.yaml | 23 ------ .../templates/staged-archives-pv.yaml | 1 + .../templates/staged-streams-pv.yaml | 1 + .../package-helm/templates/streams-pv.yaml | 1 + .../package-helm/templates/tmp-pv.yaml | 1 + .../templates/webui-deployment.yaml | 35 --------- .../package-helm/templates/webui-pvc.yaml | 17 +++++ .../package-helm/templates/webui-service.yaml | 16 +++++ 57 files changed, 415 insertions(+), 457 deletions(-) delete mode 100644 tools/deployment/package-helm/templates/00-local-pvs.yaml create mode 100644 tools/deployment/package-helm/templates/api-server-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/api-server-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/api-server-service.yaml create mode 100644 tools/deployment/package-helm/templates/archives-pv.yaml create mode 100644 tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/compression-scheduler-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/compression-scheduler-service.yaml create mode 100644 tools/deployment/package-helm/templates/compression-worker-archives-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/database-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/database-pv.yaml create mode 100644 tools/deployment/package-helm/templates/database-service.yaml create mode 100644 tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/garbage-collector-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/mcp-server-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/mcp-server-service.yaml create mode 100644 tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/query-scheduler-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/query-scheduler-service.yaml create mode 100644 tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/query-worker-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/queue-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/queue-service.yaml create mode 100644 tools/deployment/package-helm/templates/redis-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/redis-pv.yaml create mode 100644 tools/deployment/package-helm/templates/redis-service.yaml create mode 100644 tools/deployment/package-helm/templates/reducer-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/reducer-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/reducer-service.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-pv.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-service.yaml create mode 100644 tools/deployment/package-helm/templates/staged-archives-pv.yaml create mode 100644 tools/deployment/package-helm/templates/staged-streams-pv.yaml create mode 100644 tools/deployment/package-helm/templates/streams-pv.yaml create mode 100644 tools/deployment/package-helm/templates/tmp-pv.yaml create mode 100644 tools/deployment/package-helm/templates/webui-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/webui-service.yaml diff --git a/tools/deployment/package-helm/templates/00-local-pvs.yaml b/tools/deployment/package-helm/templates/00-local-pvs.yaml deleted file mode 100644 index 94ebc58cd7..0000000000 --- a/tools/deployment/package-helm/templates/00-local-pvs.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- include "clp.createLocalDataPv" (list . "database") }} ---- -{{- include "clp.createLocalLogsPv" (list . "database") }} ---- -{{- include "clp.createLocalLogsPv" (list . "queue") }} ---- -{{- include "clp.createLocalDataPv" (list . "redis") }} ---- -{{- include "clp.createLocalLogsPv" (list . "redis") }} ---- -{{- include "clp.createLocalDataPv" (list . "results-cache") }} ---- -{{- include "clp.createLocalLogsPv" (list . "results-cache") }} ---- -{{- include "clp.createLocalLogsPv" (list . "compression-scheduler") }} ---- -{{- include "clp.createLocalLogsPv" (list . "compression-worker") }} ---- -{{- include "clp.createLocalLogsPv" (list . "query-scheduler") }} ---- -{{- include "clp.createLocalLogsPv" (list . "query-worker") }} ---- -{{- include "clp.createLocalLogsPv" (list . "reducer") }} ---- -{{- include "clp.createLocalLogsPv" (list . "garbage-collector") }} ---- -{{- include "clp.createLocalLogsPv" (list . "api-server") }} ---- -{{- if .Values.clpConfig.mcp_server }} -{{- include "clp.createLocalLogsPv" (list . "mcp-server") }} ---- -{{- end }} -{{- include "clp.createStreamsPv" . }} ---- -{{- include "clp.createTmpPv" . }} ---- -{{- include "clp.createArchivesPv" . }} ---- -{{- include "clp.createStagedArchivesPv" . }} ---- -{{- include "clp.createStagedStreamsPv" . }} diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 1f6752909e..a55769e02c 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -1,20 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-api-server - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "api-server" -spec: - type: "NodePort" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "api-server" - ports: - - port: 3001 - targetPort: "http" - nodePort: {{ .Values.clpConfig.api_server.port }} ---- apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -104,21 +87,3 @@ spec: - name: "api-server-logs" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-api-server-logs ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-api-server-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "api-server" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "api-server" - resources: - requests: - storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml new file mode 100644 index 0000000000..c717de287a --- /dev/null +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "api-server") }} diff --git a/tools/deployment/package-helm/templates/api-server-pvc.yaml b/tools/deployment/package-helm/templates/api-server-pvc.yaml new file mode 100644 index 0000000000..53db492b5a --- /dev/null +++ b/tools/deployment/package-helm/templates/api-server-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-api-server-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "api-server" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/api-server-service.yaml b/tools/deployment/package-helm/templates/api-server-service.yaml new file mode 100644 index 0000000000..7c24c0f6e5 --- /dev/null +++ b/tools/deployment/package-helm/templates/api-server-service.yaml @@ -0,0 +1,16 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-api-server + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" +spec: + type: "NodePort" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" + ports: + - port: 3001 + targetPort: "http" + nodePort: {{ .Values.clpConfig.api_server.port }} diff --git a/tools/deployment/package-helm/templates/archives-pv.yaml b/tools/deployment/package-helm/templates/archives-pv.yaml new file mode 100644 index 0000000000..84edd505ec --- /dev/null +++ b/tools/deployment/package-helm/templates/archives-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createArchivesPv" . }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index fe43c9e896..21d210e18b 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -1,19 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-compression-scheduler - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "compression-scheduler" -spec: - clusterIP: "None" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "compression-scheduler" - ports: - - port: 7000 - targetPort: 7000 ---- apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -108,21 +92,3 @@ spec: hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" type: "Directory" ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-compression-scheduler-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "compression-scheduler" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "compression-scheduler" - resources: - requests: - storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml new file mode 100644 index 0000000000..05becd1f26 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "compression-scheduler") }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-pvc.yaml new file mode 100644 index 0000000000..9e5a5170d9 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-scheduler-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-compression-scheduler-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-scheduler" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-service.yaml b/tools/deployment/package-helm/templates/compression-scheduler-service.yaml new file mode 100644 index 0000000000..605daf9055 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-scheduler-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-compression-scheduler + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" + ports: + - port: 7000 + targetPort: 7000 diff --git a/tools/deployment/package-helm/templates/compression-worker-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-archives-pvc.yaml new file mode 100644 index 0000000000..2534d4ca78 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-worker-archives-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-archives + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "archives" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "archives" + resources: + requests: + storage: "50Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 6ce71cb371..35db257d75 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -103,75 +103,3 @@ spec: - name: "tmp" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-tmp ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-compression-worker-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "compression-worker" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "compression-worker" - resources: - requests: - storage: "5Gi" ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-archives - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "archives" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "archives" - resources: - requests: - storage: "50Gi" ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-staged-archives - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "staged-archives" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "staged-archives" - resources: - requests: - storage: "20Gi" ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-tmp - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "tmp" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "tmp" - resources: - requests: - storage: "10Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml new file mode 100644 index 0000000000..737e205c52 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "compression-worker") }} diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml new file mode 100644 index 0000000000..617290ea74 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-compression-worker-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-worker" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-worker" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml new file mode 100644 index 0000000000..541516ed3b --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-staged-archives + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "staged-archives" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "staged-archives" + resources: + requests: + storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml new file mode 100644 index 0000000000..ada6357ba0 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "database") }} diff --git a/tools/deployment/package-helm/templates/database-pv.yaml b/tools/deployment/package-helm/templates/database-pv.yaml new file mode 100644 index 0000000000..a72331b6f1 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalDataPv" (list . "database") }} diff --git a/tools/deployment/package-helm/templates/database-service.yaml b/tools/deployment/package-helm/templates/database-service.yaml new file mode 100644 index 0000000000..fe6f2f823e --- /dev/null +++ b/tools/deployment/package-helm/templates/database-service.yaml @@ -0,0 +1,22 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-database + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +spec: + {{- if .Values.allowSbinScripts }} + type: "NodePort" + {{- else }} + clusterIP: "None" + {{- end }} + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "database" + ports: + - port: 3306 + targetPort: "database" + {{- if .Values.allowSbinScripts }} + nodePort: {{ .Values.clpConfig.database.port }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index ca2796fc04..911adbd117 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -1,26 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-database - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "database" -spec: - {{- if .Values.allowSbinScripts }} - type: "NodePort" - {{- else }} - clusterIP: "None" - {{- end }} - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "database" - ports: - - port: 3306 - targetPort: "database" - {{- if .Values.allowSbinScripts }} - nodePort: {{ .Values.clpConfig.database.port }} - {{- end }} ---- apiVersion: "apps/v1" kind: "StatefulSet" metadata: diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 483946dd36..2369b9538a 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -87,21 +87,3 @@ spec: - name: "streams" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-streams ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-garbage-collector-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "garbage-collector" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "garbage-collector" - resources: - requests: - storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml new file mode 100644 index 0000000000..a55e94608f --- /dev/null +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "garbage-collector") }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-pvc.yaml new file mode 100644 index 0000000000..ec16d99ae7 --- /dev/null +++ b/tools/deployment/package-helm/templates/garbage-collector-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-garbage-collector-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "garbage-collector" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "garbage-collector" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index e3de1b3550..8465ea59ae 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -1,21 +1,3 @@ -{{- if .Values.clpConfig.mcp_server }} -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-mcp-server - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "mcp-server" -spec: - type: "NodePort" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "mcp-server" - ports: - - port: 8000 - targetPort: "http" - nodePort: {{ .Values.clpConfig.mcp_server.port }} ---- apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -108,22 +90,3 @@ spec: - name: "mcp-server-logs" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-mcp-server-logs ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-mcp-server-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "mcp-server" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "mcp-server" - resources: - requests: - storage: "5Gi" -{{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml new file mode 100644 index 0000000000..48fd58f61f --- /dev/null +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -0,0 +1,3 @@ +{{- if .Values.clpConfig.mcp_server }} +{{- include "clp.createLocalLogsPv" (list . "mcp-server") }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-pvc.yaml b/tools/deployment/package-helm/templates/mcp-server-pvc.yaml new file mode 100644 index 0000000000..98caca2667 --- /dev/null +++ b/tools/deployment/package-helm/templates/mcp-server-pvc.yaml @@ -0,0 +1,18 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-mcp-server-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "mcp-server" + resources: + requests: + storage: "5Gi" +{{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-service.yaml b/tools/deployment/package-helm/templates/mcp-server-service.yaml new file mode 100644 index 0000000000..81e8261428 --- /dev/null +++ b/tools/deployment/package-helm/templates/mcp-server-service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.clpConfig.mcp_server }} +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-mcp-server + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" +spec: + type: "NodePort" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" + ports: + - port: 8000 + targetPort: "http" + nodePort: {{ .Values.clpConfig.mcp_server.port }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 97f1d133f9..f9620d4d8c 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -1,19 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-query-scheduler - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "query-scheduler" -spec: - clusterIP: "None" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "query-scheduler" - ports: - - port: 7000 - targetPort: "query-scheduler" ---- apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -106,21 +90,3 @@ spec: - name: "query-scheduler-logs" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-query-scheduler-logs ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-query-scheduler-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "query-scheduler" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "query-scheduler" - resources: - requests: - storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml new file mode 100644 index 0000000000..ff15477c5e --- /dev/null +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "query-scheduler") }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-pvc.yaml b/tools/deployment/package-helm/templates/query-scheduler-pvc.yaml new file mode 100644 index 0000000000..7b1c09aba6 --- /dev/null +++ b/tools/deployment/package-helm/templates/query-scheduler-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-query-scheduler-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "query-scheduler" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/query-scheduler-service.yaml b/tools/deployment/package-helm/templates/query-scheduler-service.yaml new file mode 100644 index 0000000000..9f2ffa7172 --- /dev/null +++ b/tools/deployment/package-helm/templates/query-scheduler-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-query-scheduler + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" + ports: + - port: 7000 + targetPort: "query-scheduler" diff --git a/tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml new file mode 100644 index 0000000000..20b4dd1c75 --- /dev/null +++ b/tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-staged-streams + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "staged-streams" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "staged-streams" + resources: + requests: + storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 1de4df841d..ede349b8ed 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -96,39 +96,3 @@ spec: - name: "streams" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-streams ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-query-worker-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "query-worker" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "query-worker" - resources: - requests: - storage: "5Gi" ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-staged-streams - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "staged-streams" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "staged-streams" - resources: - requests: - storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml new file mode 100644 index 0000000000..9faec0d79d --- /dev/null +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "query-worker") }} diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml new file mode 100644 index 0000000000..b3948b64aa --- /dev/null +++ b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-query-worker-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-worker" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "query-worker" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml new file mode 100644 index 0000000000..20b4dd1c75 --- /dev/null +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-staged-streams + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "staged-streams" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "staged-streams" + resources: + requests: + storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml new file mode 100644 index 0000000000..57a08a0e5b --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "queue") }} diff --git a/tools/deployment/package-helm/templates/queue-service.yaml b/tools/deployment/package-helm/templates/queue-service.yaml new file mode 100644 index 0000000000..2971725fdb --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-queue + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "queue" + ports: + - port: 5672 + targetPort: "amqp" diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index 93c0733b27..7c7245a5e7 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -1,19 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-queue - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "queue" -spec: - clusterIP: "None" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "queue" - ports: - - port: 5672 - targetPort: "amqp" ---- apiVersion: "apps/v1" kind: "StatefulSet" metadata: diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml new file mode 100644 index 0000000000..5dcc2821de --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "redis") }} diff --git a/tools/deployment/package-helm/templates/redis-pv.yaml b/tools/deployment/package-helm/templates/redis-pv.yaml new file mode 100644 index 0000000000..418696c290 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalDataPv" (list . "redis") }} diff --git a/tools/deployment/package-helm/templates/redis-service.yaml b/tools/deployment/package-helm/templates/redis-service.yaml new file mode 100644 index 0000000000..5cd8e61386 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-redis + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "redis" + ports: + - port: 6379 + targetPort: "redis" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 5db313885d..c9ef5b8223 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -1,19 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-redis - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "redis" -spec: - clusterIP: "None" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "redis" - ports: - - port: 6379 - targetPort: "redis" ---- apiVersion: "apps/v1" kind: "StatefulSet" metadata: diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index db4a7af1c3..edd1693849 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -1,22 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-reducer - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "reducer" -spec: - clusterIP: "None" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "reducer" - ports: - {{- range $i := until (int .Values.workerConcurrency) }} - - name: "reducer-{{ $i }}" - port: {{ add $.Values.clpConfig.reducer.base_port $i }} - targetPort: {{ add $.Values.clpConfig.reducer.base_port $i }} - {{- end }} ---- apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -90,21 +71,3 @@ spec: - name: "reducer-logs" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-reducer-logs ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-reducer-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "reducer" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "reducer" - resources: - requests: - storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml new file mode 100644 index 0000000000..e94ed53016 --- /dev/null +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "reducer") }} diff --git a/tools/deployment/package-helm/templates/reducer-pvc.yaml b/tools/deployment/package-helm/templates/reducer-pvc.yaml new file mode 100644 index 0000000000..e4502864c7 --- /dev/null +++ b/tools/deployment/package-helm/templates/reducer-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-reducer-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "reducer" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/reducer-service.yaml b/tools/deployment/package-helm/templates/reducer-service.yaml new file mode 100644 index 0000000000..5c3fc6d7b1 --- /dev/null +++ b/tools/deployment/package-helm/templates/reducer-service.yaml @@ -0,0 +1,18 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-reducer + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" + ports: + {{- range $i := until (int .Values.workerConcurrency) }} + - name: "reducer-{{ $i }}" + port: {{ add $.Values.clpConfig.reducer.base_port $i }} + targetPort: {{ add $.Values.clpConfig.reducer.base_port $i }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml new file mode 100644 index 0000000000..5547f06955 --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "results-cache") }} diff --git a/tools/deployment/package-helm/templates/results-cache-pv.yaml b/tools/deployment/package-helm/templates/results-cache-pv.yaml new file mode 100644 index 0000000000..bbaacab351 --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalDataPv" (list . "results-cache") }} diff --git a/tools/deployment/package-helm/templates/results-cache-service.yaml b/tools/deployment/package-helm/templates/results-cache-service.yaml new file mode 100644 index 0000000000..2c8e540fd8 --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-service.yaml @@ -0,0 +1,22 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-results-cache + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + {{- if .Values.allowSbinScripts }} + type: "NodePort" + {{- else }} + clusterIP: "None" + {{- end }} + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" + ports: + - port: 27017 + targetPort: "mongodb" + {{- if .Values.allowSbinScripts }} + nodePort: {{ .Values.clpConfig.results_cache.port }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index c4453cd9d0..818093fde7 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -1,26 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-results-cache - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "results-cache" -spec: - {{- if .Values.allowSbinScripts }} - type: "NodePort" - {{- else }} - clusterIP: "None" - {{- end }} - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "results-cache" - ports: - - port: 27017 - targetPort: "mongodb" - {{- if .Values.allowSbinScripts }} - nodePort: {{ .Values.clpConfig.results_cache.port }} - {{- end }} ---- apiVersion: "apps/v1" kind: "StatefulSet" metadata: diff --git a/tools/deployment/package-helm/templates/staged-archives-pv.yaml b/tools/deployment/package-helm/templates/staged-archives-pv.yaml new file mode 100644 index 0000000000..28b5053f43 --- /dev/null +++ b/tools/deployment/package-helm/templates/staged-archives-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createStagedArchivesPv" . }} diff --git a/tools/deployment/package-helm/templates/staged-streams-pv.yaml b/tools/deployment/package-helm/templates/staged-streams-pv.yaml new file mode 100644 index 0000000000..ae7e52821b --- /dev/null +++ b/tools/deployment/package-helm/templates/staged-streams-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createStagedStreamsPv" . }} diff --git a/tools/deployment/package-helm/templates/streams-pv.yaml b/tools/deployment/package-helm/templates/streams-pv.yaml new file mode 100644 index 0000000000..7f773161a0 --- /dev/null +++ b/tools/deployment/package-helm/templates/streams-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createStreamsPv" . }} diff --git a/tools/deployment/package-helm/templates/tmp-pv.yaml b/tools/deployment/package-helm/templates/tmp-pv.yaml new file mode 100644 index 0000000000..1864277a59 --- /dev/null +++ b/tools/deployment/package-helm/templates/tmp-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createTmpPv" . }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 873e56c8f7..23c2b57320 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -1,20 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-webui - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "webui" -spec: - type: "NodePort" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "webui" - ports: - - port: 4000 - targetPort: "http" - nodePort: {{ .Values.clpConfig.webui.port }} ---- apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -113,21 +96,3 @@ spec: - name: "streams" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-streams ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-streams - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "streams" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "streams" - resources: - requests: - storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/webui-pvc.yaml b/tools/deployment/package-helm/templates/webui-pvc.yaml new file mode 100644 index 0000000000..dfde2e91e1 --- /dev/null +++ b/tools/deployment/package-helm/templates/webui-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-streams + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "streams" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "streams" + resources: + requests: + storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/webui-service.yaml b/tools/deployment/package-helm/templates/webui-service.yaml new file mode 100644 index 0000000000..ebe1ea2074 --- /dev/null +++ b/tools/deployment/package-helm/templates/webui-service.yaml @@ -0,0 +1,16 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-webui + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "webui" +spec: + type: "NodePort" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "webui" + ports: + - port: 4000 + targetPort: "http" + nodePort: {{ .Values.clpConfig.webui.port }} From 0b8ef72b8e23d1389d2b1dd60700993abf7eaa3c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 00:46:22 -0500 Subject: [PATCH 018/155] exclude helm directory from yaml lint --- taskfiles/lint.yaml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index ecfb24279f..a82ca09351 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -98,16 +98,17 @@ tasks: - |- . "{{.G_LINT_VENV_DIR}}/bin/activate" yamllint --strict \ - .github \ - .yamllint.yml \ - components/core/.clang-format \ - components/core/config \ - components/core/tools/ \ - components/package-template/src/etc \ - docs \ - taskfile.yaml \ - taskfiles \ - tools/deployment + ".github" \ + ".yamllint.yml" \ + "components/core/.clang-format" \ + "components/core/config" \ + "components/core/tools/" \ + "components/package-template/src/etc" \ + "docs" \ + "taskfile.yaml" \ + "taskfiles" \ + "tools/deployment/package" \ + "tools/deployment/presto-clp" check-cpp-format: sources: &cpp_source_files From 9b5b53b70ae34462c232ed4ab8efddb446997124 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 03:01:06 -0500 Subject: [PATCH 019/155] fix templates --- .../package-helm/templates/mcp-server-deployment.yaml | 2 ++ tools/deployment/package-helm/templates/mcp-server-pvc.yaml | 1 + tools/deployment/package-helm/templates/mcp-server-service.yaml | 1 + 3 files changed, 4 insertions(+) diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 8465ea59ae..436719558f 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -1,3 +1,4 @@ +{{- if .Values.clpConfig.mcp_server }} apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -90,3 +91,4 @@ spec: - name: "mcp-server-logs" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-mcp-server-logs +{{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-pvc.yaml b/tools/deployment/package-helm/templates/mcp-server-pvc.yaml index 98caca2667..4ab1171bc2 100644 --- a/tools/deployment/package-helm/templates/mcp-server-pvc.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-pvc.yaml @@ -1,3 +1,4 @@ +{{- if .Values.clpConfig.mcp_server }} apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: diff --git a/tools/deployment/package-helm/templates/mcp-server-service.yaml b/tools/deployment/package-helm/templates/mcp-server-service.yaml index 81e8261428..ee2ae9fbe9 100644 --- a/tools/deployment/package-helm/templates/mcp-server-service.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-service.yaml @@ -15,3 +15,4 @@ spec: - port: 8000 targetPort: "http" nodePort: {{ .Values.clpConfig.mcp_server.port }} +{{- end }} From c1dccd6c2d1c0302c1dc9c6fe794b4beac0978ef Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 04:08:19 -0500 Subject: [PATCH 020/155] fix(helm): Update maintainer name in Chart.yaml. --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 035b371050..ba2ac6ee3d 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -7,7 +7,7 @@ appVersion: "0.6.1-dev" home: "https://github.com/y-scope/clp" icon: "https://github.com/y-scope/clp/raw/main/docs/src/clp-logo.png" sources: ["https://github.com/y-scope/clp"] -maintainers: [{name: "YScope Inc.", email: "dev@yscope.com"}] +maintainers: [{name: "y-scope", email: "dev@yscope.com"}] keywords: - "analytics" - "compression" From 236c024d55aa704eefd62a354b206d7b6b989fa0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 04:08:32 -0500 Subject: [PATCH 021/155] fix(helm): Set default tag to "nightly" for clpPackage. --- tools/deployment/package-helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 7a69d9aeb3..ffd87b17ab 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -24,7 +24,7 @@ image: clpPackage: repository: "ghcr.io/y-scope/clp/clp-package" pullPolicy: "IfNotPresent" - tag: "" + tag: "nightly" clpConfig: package: From 5aa26cdded5243e45a56462de8deaea25b86fb12 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 05:11:02 -0500 Subject: [PATCH 022/155] Add Helm linting and toolchain support --- .github/workflows/clp-lint.yaml | 4 ++++ chart_schema.yaml | 38 +++++++++++++++++++++++++++++ ct.yaml | 8 +++++++ lint-requirements.txt | 8 ++++++- taskfiles/lint.yaml | 16 +++++++++++++ taskfiles/toolchains.yaml | 42 +++++++++++++++++++++++++++++++++ 6 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 chart_schema.yaml create mode 100644 ct.yaml diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml index 1d97737e74..933ac97325 100644 --- a/.github/workflows/clp-lint.yaml +++ b/.github/workflows/clp-lint.yaml @@ -78,3 +78,7 @@ jobs: - name: "Lint .yaml files" shell: "bash" run: "task lint:check-yaml" + + - name: "Lint Helm charts" + shell: "bash" + run: "task lint:check-helm" diff --git a/chart_schema.yaml b/chart_schema.yaml new file mode 100644 index 0000000000..4f1d69b73b --- /dev/null +++ b/chart_schema.yaml @@ -0,0 +1,38 @@ +# yamllint disable rule:document-start +name: "str()" +home: "str(required=False)" +version: "str()" +apiVersion: "str()" +appVersion: "any(str(), num(), required=False)" +description: "str(required=False)" +keywords: "list(str(), required=False)" +sources: "list(str(), required=False)" +maintainers: "list(include('maintainer'), required=False)" +dependencies: "list(include('dependency'), required=False)" +icon: "str(required=False)" +engine: "str(required=False)" +condition: "str(required=False)" +tags: "str(required=False)" +deprecated: "bool(required=False)" +kubeVersion: "str(required=False)" +annotations: "map(str(), str(), required=False)" +type: "str(required=False)" +--- +maintainer: + name: "str()" + email: "str(required=False)" + url: "str(required=False)" +--- +dependency: + name: "str()" + version: "str()" + repository: "str(required=False)" + condition: "str(required=False)" + tags: "list(str(), required=False)" + enabled: "bool(required=False)" + import-values: "list(any(str(), include('import-value')), required=False)" + alias: "str(required=False)" +--- +import-value: + child: "str()" + parent: "str()" diff --git a/ct.yaml b/ct.yaml new file mode 100644 index 0000000000..159fec6d15 --- /dev/null +++ b/ct.yaml @@ -0,0 +1,8 @@ +chart-dirs: + - "tools/deployment" +validate-maintainers: true +validate-chart-schema: true +check-version-increment: true +validate-yaml: true +lint-conf: ".yamllint.yml" +chart-yaml-schema: "chart_schema.yaml" diff --git a/lint-requirements.txt b/lint-requirements.txt index a5e2ef60d9..2cab04f19a 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,6 +1,12 @@ -black>=24.4.2 +# C++ clang-format>=21.1.2 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 + +# Python +black>=24.4.2 ruff>=0.4.4 + +# YAML yamllint>=1.35.1 +yamale>=6.0.0 \ No newline at end of file diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index a82ca09351..4d64625bcb 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -14,6 +14,7 @@ tasks: check: cmds: - task: "check-cpp-full" + - task: "check-helm" - task: "check-js" - task: "check-py" - task: "check-rust" @@ -32,6 +33,7 @@ tasks: fix: cmds: - task: "fix-cpp-full" + - task: "fix-helm" - task: "fix-js" - task: "fix-py" - task: "fix-rust" @@ -89,6 +91,18 @@ tasks: RUFF_CHECK_FLAGS: "--fix" RUFF_FORMAT_FLAGS: "" + helm: + aliases: + - "check-helm" + - "fix-helm" + deps: + - "toolchains:helm" + - "venv" + cmds: + - |- + . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + ct lint + yaml: aliases: - "check-yaml" @@ -104,6 +118,8 @@ tasks: "components/core/config" \ "components/core/tools/" \ "components/package-template/src/etc" \ + "chart_schema.yaml" \ + "ct.yaml" \ "docs" \ "taskfile.yaml" \ "taskfiles" \ diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 5dfae671ee..84f1694349 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -1,6 +1,48 @@ version: "3" +includes: + utils: "../tools/yscope-dev-utils/exports/taskfiles/utils/remote.yaml" + +vars: + G_HELM_TOOLCHAIN_DIR: "{{.G_BUILD_DIR}}/toolchains/helm" + G_HELM_TOOLCHAIN_ENV_FILE: "{{.G_HELM_TOOLCHAIN_DIR}}/env" + tasks: + helm: + run: "once" + vars: + CHART_TESTING_VERSION: "3.14.0" + cmds: + - "mkdir -p '{{.G_HELM_TOOLCHAIN_DIR}}'" + - task: "utils:download-and-extract-tar" + vars: + OUTPUT_DIR: "{{.G_BUILD_DIR}}/toolchains/helm" + URL: "https://github.com/helm/chart-testing/releases/download/v{{.CHART_TESTING_VERSION}}\ + /chart-testing_{{.CHART_TESTING_VERSION}}_{{OS}}_{{ARCH}}.tar.gz" + FILE_SHA256: >- + {{- if and (eq OS "linux") (eq ARCH "amd64") -}} + d16f0583616885423826241164ce1f6589c6fe5332fa74f374ebd2bd3cb3fe1f + {{- else if and (eq OS "darwin") (eq ARCH "arm64") -}} + db10dbbb42b110c7a5da5a3202908f32ad2ca6ad600d423426dc7886d09aad07 + {{- else -}} + unsupported + {{- end }} + INCLUDE_PATTERNS: ["ct"] + NUM_COMPONENTS_TO_STRIP: "0" + - |- + export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" + export HELM_INSTALL_DIR="{{.G_HELM_TOOLCHAIN_DIR}}" + curl --proto "=https" --tlsv1.2 --silent --show-error --fail \ + https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-4 | bash -s -- --no-sudo + + # Create an environment file to configure the custom Helm toolchain location. + - |- + cat < "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + #!/bin/sh + export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" + EOF + chmod +x "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + rust: run: "once" cmds: From 2ab3e74a9358c8fa94c5257de21e5fe96c66e4f9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 06:12:17 -0500 Subject: [PATCH 023/155] fix(helm): Rename `user` to `username` in query-worker-deployment template. --- .../templates/compression-scheduler-deployment.yaml | 2 +- .../package-helm/templates/compression-worker-deployment.yaml | 2 +- .../package-helm/templates/query-scheduler-deployment.yaml | 2 +- .../package-helm/templates/query-worker-deployment.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 21d210e18b..9365b60327 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -47,7 +47,7 @@ spec: imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_DB_USER" valueFrom: secretKeyRef: diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 35db257d75..6aee42b056 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -41,7 +41,7 @@ spec: imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yml" - name: "CLP_HOME" diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index f9620d4d8c..8728244139 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -49,7 +49,7 @@ spec: imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_DB_USER" valueFrom: secretKeyRef: diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index ede349b8ed..6f913e2a86 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -41,7 +41,7 @@ spec: imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yml" - name: "CLP_HOME" From 2ac391be067ed66a529e91479cf5f37cadd512e0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 12:44:56 -0500 Subject: [PATCH 024/155] refactor(helm): Rename PVC and PV files; add tmp-pvc definition. --- ...rker-archives-pvc.yaml => archives-pvc.yaml} | 0 ... compression-worker-staged-archives-pv.yaml} | 0 ...yaml => query-worker-staged-streams-pv.yaml} | 0 .../package-helm/templates/tmp-pvc.yaml | 17 +++++++++++++++++ 4 files changed, 17 insertions(+) rename tools/deployment/package-helm/templates/{compression-worker-archives-pvc.yaml => archives-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{staged-archives-pv.yaml => compression-worker-staged-archives-pv.yaml} (100%) rename tools/deployment/package-helm/templates/{staged-streams-pv.yaml => query-worker-staged-streams-pv.yaml} (100%) create mode 100644 tools/deployment/package-helm/templates/tmp-pvc.yaml diff --git a/tools/deployment/package-helm/templates/compression-worker-archives-pvc.yaml b/tools/deployment/package-helm/templates/archives-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/compression-worker-archives-pvc.yaml rename to tools/deployment/package-helm/templates/archives-pvc.yaml diff --git a/tools/deployment/package-helm/templates/staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml similarity index 100% rename from tools/deployment/package-helm/templates/staged-archives-pv.yaml rename to tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml diff --git a/tools/deployment/package-helm/templates/staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml similarity index 100% rename from tools/deployment/package-helm/templates/staged-streams-pv.yaml rename to tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml diff --git a/tools/deployment/package-helm/templates/tmp-pvc.yaml b/tools/deployment/package-helm/templates/tmp-pvc.yaml new file mode 100644 index 0000000000..c0e4cffad0 --- /dev/null +++ b/tools/deployment/package-helm/templates/tmp-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-tmp + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "tmp" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "tmp" + resources: + requests: + storage: "10Gi" From f1f3a541792b2c58cb8c38b2d8d09134fe457e61 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 13:12:33 -0500 Subject: [PATCH 025/155] refactor(helm): Remove localPathBase; redefine directory paths for storage and logs. --- .../package-helm/templates/_helpers.tpl | 28 +++++-------------- tools/deployment/package-helm/values.yaml | 14 ++++------ 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 3c6815ab24..0db2fb59b7 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -78,20 +78,6 @@ timeoutSeconds: 5 failureThreshold: 3 {{- end }} -{{/* -CLP logs directory path on host -*/}} -{{- define "clp.logsDirHost" -}} -{{ .Values.storage.localPathBase }}/{{ .Values.clpConfig.logs_directory }} -{{- end }} - -{{/* -CLP data directory path on host -*/}} -{{- define "clp.dataDirHost" -}} -{{ .Values.storage.localPathBase }}/{{ .Values.clpConfig.data_directory }} -{{- end }} - {{/* Creates a local PersistentVolume for data storage. */}} @@ -112,7 +98,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp.dataDirHost" $root }}/{{ $component }} + path: {{ $root.Values.clpConfig.data_directory }}/{{ $component }} nodeAffinity: required: nodeSelectorTerms: @@ -141,7 +127,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp.logsDirHost" $root }}/{{ $component }} + path: {{ $root.Values.clpConfig.logs_directory }}/{{ $component }} nodeAffinity: required: nodeSelectorTerms: @@ -169,7 +155,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp.dataDirHost" $root }}/streams + path: {{ $root.Values.clpConfig.stream_output.storage.directory }} nodeAffinity: required: nodeSelectorTerms: @@ -197,7 +183,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ .Values.storage.localPathBase }}/var/tmp + path: {{ .Values.clpConfig.tmp_directory }} nodeAffinity: required: nodeSelectorTerms: @@ -225,7 +211,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp.dataDirHost" $root }}/archives + path: {{ $root.Values.clpConfig.archive_output.storage.directory }} nodeAffinity: required: nodeSelectorTerms: @@ -253,7 +239,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp.dataDirHost" $root }}/staged-archives + path: "/tmp/clp/staged-archives" nodeAffinity: required: nodeSelectorTerms: @@ -281,7 +267,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp.dataDirHost" $root }}/staged-streams + path: "/tmp/clp/staged-streams" nodeAffinity: required: nodeSelectorTerms: diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index ffd87b17ab..0b414da61d 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -4,10 +4,6 @@ fullnameOverride: "" # Whether to allow sbin script accesses to services on the node host. allowSbinScripts: true -storage: - # Base directory on host when using local PVs. - localPathBase: "/tmp/clp" - securityContext: # UID/GID for first-party CLP services firstParty: @@ -105,7 +101,7 @@ clpConfig: type: "fs" # NOTE: This directory must not overlap with any path used in CLP's execution container. An # error will be raised if so. - directory: "var/data/archives" + directory: "/tmp/clp/var/data/archives" # Retention period for archives, in minutes. Set to null to disable automatic deletion. retention_period: null @@ -133,7 +129,7 @@ clpConfig: type: "fs" # NOTE: This directory must not overlap with any path used in CLP's execution container. An # error will be raised if so. - directory: "var/data/streams" + directory: "/tmp/clp/var/data/streams" # How large each stream file should be before being split into a new stream file target_uncompressed_size: 134217728 # 128 MB @@ -165,18 +161,18 @@ clpConfig: # it doesn't exist. # NOTE: This directory must not overlap with any path used in CLP's execution container. An error # will be raised if so. - data_directory: "var/data" + data_directory: "/tmp/clp/var/data" # Location where logs are stored. It will be created if it doesn't exist. # NOTE: This directory must not overlap with any path used in CLP's execution container. An error # will be raised if so. - logs_directory: "var/log" + logs_directory: "/tmp/clp/var/log" # Location where temporary runtime data are stored. It will be created if # it doesn't exist. # NOTE: This directory must not overlap with any path used in CLP's execution container. An error # will be raised if so. - tmp_directory: "var/tmp" + tmp_directory: "/tmp/clp/var/tmp" # Location of the AWS tools' config files (e.g., `~/.aws`) aws_config_directory: null From 832eb1935e35d96783f8c5bba5d99420f57ccbb4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 18:04:12 -0500 Subject: [PATCH 026/155] refactor(helm): Consolidate PersistentVolume templates into `createLocalPv`. --- .../package-helm/templates/_helpers.tpl | 193 ++---------------- .../templates/api-server-logs-pv.yaml | 9 +- .../package-helm/templates/archives-pv.yaml | 9 +- .../compression-scheduler-logs-pv.yaml | 9 +- .../templates/compression-worker-logs-pv.yaml | 9 +- ...compression-worker-staged-archives-pv.yaml | 9 +- .../templates/database-logs-pv.yaml | 9 +- .../package-helm/templates/database-pv.yaml | 9 +- .../templates/garbage-collector-logs-pv.yaml | 9 +- .../templates/mcp-server-logs-pv.yaml | 11 +- .../templates/query-scheduler-logs-pv.yaml | 9 +- .../templates/query-worker-logs-pv.yaml | 9 +- .../query-worker-staged-streams-pv.yaml | 9 +- .../package-helm/templates/queue-logs-pv.yaml | 9 +- .../package-helm/templates/redis-logs-pv.yaml | 9 +- .../package-helm/templates/redis-pv.yaml | 9 +- .../templates/reducer-logs-pv.yaml | 9 +- .../templates/results-cache-logs-pv.yaml | 9 +- .../templates/results-cache-pv.yaml | 9 +- .../package-helm/templates/streams-pv.yaml | 9 +- .../package-helm/templates/tmp-pv.yaml | 9 +- 21 files changed, 175 insertions(+), 200 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 0db2fb59b7..249bcadaba 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -79,26 +79,31 @@ failureThreshold: 3 {{- end }} {{/* -Creates a local PersistentVolume for data storage. +Creates a local PersistentVolume. +Parameters (dict): + root: Root template context + name: PV name (full) + component: Component label + capacity: Storage capacity + accessModes: Access modes (list) + hostPath: Absolute path on host */}} -{{- define "clp.createLocalDataPv" -}} -{{- $root := index . 0 -}} -{{- $component := index . 1 -}} +{{- define "clp.createLocalPv" -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp.fullname" $root }}-data-pv-{{ $component }} + name: {{ .name }} labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: {{ $component | quote }} + {{- include "clp.labels" .root | nindent 4 }} + app.kubernetes.io/component: {{ .component | quote }} spec: capacity: - storage: "20Gi" - accessModes: ["ReadWriteOnce"] + storage: {{ .capacity }} + accessModes: {{ .accessModes }} persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ $root.Values.clpConfig.data_directory }}/{{ $component }} + path: {{ .hostPath }} nodeAffinity: required: nodeSelectorTerms: @@ -107,172 +112,4 @@ spec: operator: "Exists" {{- end }} -{{/* -Creates a local PersistentVolume for logs storage. -*/}} -{{- define "clp.createLocalLogsPv" -}} -{{- $root := index . 0 -}} -{{- $component := index . 1 -}} -apiVersion: "v1" -kind: "PersistentVolume" -metadata: - name: {{ include "clp.fullname" $root }}-logs-pv-{{ $component }} - labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: {{ $component | quote }} -spec: - capacity: - storage: "5Gi" - accessModes: ["ReadWriteOnce"] - persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" - local: - path: {{ $root.Values.clpConfig.logs_directory }}/{{ $component }} - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: "Exists" -{{- end }} - -{{/* -Creates a local PersistentVolume for streams storage (ReadWriteMany). -*/}} -{{- define "clp.createStreamsPv" -}} -{{- $root := . -}} -apiVersion: "v1" -kind: "PersistentVolume" -metadata: - name: {{ include "clp.fullname" $root }}-streams-pv - labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: "streams" -spec: - capacity: - storage: "20Gi" - accessModes: ["ReadWriteMany"] - persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" - local: - path: {{ $root.Values.clpConfig.stream_output.storage.directory }} - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: "Exists" -{{- end }} - -{{/* -Creates a local PersistentVolume for tmp storage (ReadWriteMany). -*/}} -{{- define "clp.createTmpPv" -}} -{{- $root := . -}} -apiVersion: "v1" -kind: "PersistentVolume" -metadata: - name: {{ include "clp.fullname" $root }}-tmp-pv - labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: "tmp" -spec: - capacity: - storage: "10Gi" - accessModes: ["ReadWriteMany"] - persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" - local: - path: {{ .Values.clpConfig.tmp_directory }} - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: "Exists" -{{- end }} - -{{/* -Creates a local PersistentVolume for archives storage (ReadWriteMany). -*/}} -{{- define "clp.createArchivesPv" -}} -{{- $root := . -}} -apiVersion: "v1" -kind: "PersistentVolume" -metadata: - name: {{ include "clp.fullname" $root }}-archives-pv - labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: "archives" -spec: - capacity: - storage: "50Gi" - accessModes: ["ReadWriteMany"] - persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" - local: - path: {{ $root.Values.clpConfig.archive_output.storage.directory }} - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: "Exists" -{{- end }} - -{{/* -Creates a local PersistentVolume for staged-archives storage (ReadWriteMany). -*/}} -{{- define "clp.createStagedArchivesPv" -}} -{{- $root := . -}} -apiVersion: "v1" -kind: "PersistentVolume" -metadata: - name: {{ include "clp.fullname" $root }}-staged-archives-pv - labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: "staged-archives" -spec: - capacity: - storage: "20Gi" - accessModes: ["ReadWriteMany"] - persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" - local: - path: "/tmp/clp/staged-archives" - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: "Exists" -{{- end }} - -{{/* -Creates a local PersistentVolume for staged-streams storage (ReadWriteMany). -*/}} -{{- define "clp.createStagedStreamsPv" -}} -{{- $root := . -}} -apiVersion: "v1" -kind: "PersistentVolume" -metadata: - name: {{ include "clp.fullname" $root }}-staged-streams-pv - labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: "staged-streams" -spec: - capacity: - storage: "20Gi" - accessModes: ["ReadWriteMany"] - persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" - local: - path: "/tmp/clp/staged-streams" - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: "Exists" -{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index c717de287a..f229201f9f 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "api-server") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-api-server-logs-pv" (include "clp.fullname" .)) + "component" "api-server" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/api-server" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/archives-pv.yaml b/tools/deployment/package-helm/templates/archives-pv.yaml index 84edd505ec..bfd618723e 100644 --- a/tools/deployment/package-helm/templates/archives-pv.yaml +++ b/tools/deployment/package-helm/templates/archives-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createArchivesPv" . }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-archives-pv" (include "clp.fullname" .)) + "component" "archives" + "capacity" "50Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" .Values.clpConfig.archive_output.storage.directory +) }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 05becd1f26..14812f7384 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "compression-scheduler") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-compression-scheduler-logs-pv" (include "clp.fullname" .)) + "component" "compression-scheduler" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/compression-scheduler" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index 737e205c52..a1f5903e4f 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "compression-worker") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-compression-worker-logs-pv" (include "clp.fullname" .)) + "component" "compression-worker" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/compression-worker" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index 28b5053f43..c10c71642c 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createStagedArchivesPv" . }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-staged-archives-pv" (include "clp.fullname" .)) + "component" "staged-archives" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" "/tmp/clp/var/data/staged-archives" +) }} diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml index ada6357ba0..92d8f548f9 100644 --- a/tools/deployment/package-helm/templates/database-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "database") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-database-logs-pv" (include "clp.fullname" .)) + "component" "database" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/database" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/database-pv.yaml b/tools/deployment/package-helm/templates/database-pv.yaml index a72331b6f1..cf8bebe068 100644 --- a/tools/deployment/package-helm/templates/database-pv.yaml +++ b/tools/deployment/package-helm/templates/database-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalDataPv" (list . "database") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-database-data-pv" (include "clp.fullname" .)) + "component" "database" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/database" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index a55e94608f..4b1e709b6f 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "garbage-collector") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-garbage-collector-logs-pv" (include "clp.fullname" .)) + "component" "garbage-collector" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/garbage-collector" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index 48fd58f61f..ee6b204e65 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -1,3 +1,8 @@ -{{- if .Values.clpConfig.mcp_server }} -{{- include "clp.createLocalLogsPv" (list . "mcp-server") }} -{{- end }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-mcp-server-logs-pv" (include "clp.fullname" .)) + "component" "mcp-server" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/mcp-server" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index ff15477c5e..cf9b994d63 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "query-scheduler") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-query-scheduler-logs-pv" (include "clp.fullname" .)) + "component" "query-scheduler" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/query-scheduler" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index 9faec0d79d..fd52f15714 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "query-worker") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-query-worker-logs-pv" (include "clp.fullname" .)) + "component" "query-worker" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/query-worker" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index ae7e52821b..134ec7d14c 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createStagedStreamsPv" . }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-staged-streams-pv" (include "clp.fullname" .)) + "component" "staged-streams" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" "/tmp/clp/var/data/staged-streams" +) }} diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml index 57a08a0e5b..0db0788eaa 100644 --- a/tools/deployment/package-helm/templates/queue-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "queue") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-queue-logs-pv" (include "clp.fullname" .)) + "component" "queue" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/queue" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml index 5dcc2821de..d123c21d11 100644 --- a/tools/deployment/package-helm/templates/redis-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "redis") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-redis-logs-pv" (include "clp.fullname" .)) + "component" "redis" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/redis" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/redis-pv.yaml b/tools/deployment/package-helm/templates/redis-pv.yaml index 418696c290..139cfbed4e 100644 --- a/tools/deployment/package-helm/templates/redis-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalDataPv" (list . "redis") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-redis-data-pv" (include "clp.fullname" .)) + "component" "redis" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/redis" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index e94ed53016..898c88d8c4 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "reducer") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-reducer-logs-pv" (include "clp.fullname" .)) + "component" "reducer" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/reducer" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index 5547f06955..89a8fbf892 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "results-cache") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-results-cache-logs-pv" (include "clp.fullname" .)) + "component" "results-cache" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/results-cache" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/results-cache-pv.yaml b/tools/deployment/package-helm/templates/results-cache-pv.yaml index bbaacab351..a74fa2cb66 100644 --- a/tools/deployment/package-helm/templates/results-cache-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalDataPv" (list . "results-cache") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-results-cache-data-pv" (include "clp.fullname" .)) + "component" "results-cache" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/results-cache" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/streams-pv.yaml b/tools/deployment/package-helm/templates/streams-pv.yaml index 7f773161a0..477c23153d 100644 --- a/tools/deployment/package-helm/templates/streams-pv.yaml +++ b/tools/deployment/package-helm/templates/streams-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createStreamsPv" . }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-streams-pv" (include "clp.fullname" .)) + "component" "streams" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" .Values.clpConfig.stream_output.storage.directory +) }} diff --git a/tools/deployment/package-helm/templates/tmp-pv.yaml b/tools/deployment/package-helm/templates/tmp-pv.yaml index 1864277a59..2ddba79f4c 100644 --- a/tools/deployment/package-helm/templates/tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/tmp-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createTmpPv" . }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-tmp-pv" (include "clp.fullname" .)) + "component" "tmp" + "capacity" "10Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" .Values.clpConfig.tmp_directory +) }} From d2b4671abcb5da3d2aec8df5972063583d88753e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 23:58:12 -0500 Subject: [PATCH 027/155] refactor(helm): Rename PVC and PV template files for consistency. --- .../templates/{api-server-pvc.yaml => api-server-logs-pvc.yaml} | 0 ...ion-scheduler-pvc.yaml => compression-scheduler-logs-pvc.yaml} | 0 .../templates/{database-pv.yaml => database-data-pv.yaml} | 0 ...garbage-collector-pvc.yaml => garbage-collector-logs-pvc.yaml} | 0 .../templates/{mcp-server-pvc.yaml => mcp-server-logs-pvc.yaml} | 0 .../{query-scheduler-pvc.yaml => query-scheduler-logs-pvc.yaml} | 0 .../package-helm/templates/{redis-pv.yaml => redis-data-pv.yaml} | 0 .../templates/{reducer-pvc.yaml => reducer-logs-pvc.yaml} | 0 .../{results-cache-pv.yaml => results-cache-data-pv.yaml} | 0 .../package-helm/templates/{webui-pvc.yaml => streams-pvc.yaml} | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename tools/deployment/package-helm/templates/{api-server-pvc.yaml => api-server-logs-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{compression-scheduler-pvc.yaml => compression-scheduler-logs-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{database-pv.yaml => database-data-pv.yaml} (100%) rename tools/deployment/package-helm/templates/{garbage-collector-pvc.yaml => garbage-collector-logs-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{mcp-server-pvc.yaml => mcp-server-logs-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{query-scheduler-pvc.yaml => query-scheduler-logs-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{redis-pv.yaml => redis-data-pv.yaml} (100%) rename tools/deployment/package-helm/templates/{reducer-pvc.yaml => reducer-logs-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{results-cache-pv.yaml => results-cache-data-pv.yaml} (100%) rename tools/deployment/package-helm/templates/{webui-pvc.yaml => streams-pvc.yaml} (100%) diff --git a/tools/deployment/package-helm/templates/api-server-pvc.yaml b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/api-server-pvc.yaml rename to tools/deployment/package-helm/templates/api-server-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/compression-scheduler-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/compression-scheduler-pvc.yaml rename to tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/database-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml similarity index 100% rename from tools/deployment/package-helm/templates/database-pv.yaml rename to tools/deployment/package-helm/templates/database-data-pv.yaml diff --git a/tools/deployment/package-helm/templates/garbage-collector-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/garbage-collector-pvc.yaml rename to tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/mcp-server-pvc.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/mcp-server-pvc.yaml rename to tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/query-scheduler-pvc.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/query-scheduler-pvc.yaml rename to tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/redis-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml similarity index 100% rename from tools/deployment/package-helm/templates/redis-pv.yaml rename to tools/deployment/package-helm/templates/redis-data-pv.yaml diff --git a/tools/deployment/package-helm/templates/reducer-pvc.yaml b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/reducer-pvc.yaml rename to tools/deployment/package-helm/templates/reducer-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/results-cache-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml similarity index 100% rename from tools/deployment/package-helm/templates/results-cache-pv.yaml rename to tools/deployment/package-helm/templates/results-cache-data-pv.yaml diff --git a/tools/deployment/package-helm/templates/webui-pvc.yaml b/tools/deployment/package-helm/templates/streams-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/webui-pvc.yaml rename to tools/deployment/package-helm/templates/streams-pvc.yaml From 8985355147ea5ed443ce6d5fc620d5373d8c64a9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 00:04:57 -0500 Subject: [PATCH 028/155] refactor(helm): Simplify PV and secret object names for consistency. --- tools/deployment/package-helm/templates/api-server-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/archives-pv.yaml | 2 +- .../package-helm/templates/compression-scheduler-logs-pv.yaml | 2 +- .../package-helm/templates/compression-worker-logs-pv.yaml | 2 +- .../templates/compression-worker-staged-archives-pv.yaml | 2 +- tools/deployment/package-helm/templates/database-data-pv.yaml | 2 +- tools/deployment/package-helm/templates/database-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/database-secret.yaml | 2 +- .../package-helm/templates/garbage-collector-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml | 2 +- .../package-helm/templates/query-scheduler-logs-pv.yaml | 2 +- .../deployment/package-helm/templates/query-worker-logs-pv.yaml | 2 +- .../package-helm/templates/query-worker-staged-streams-pv.yaml | 2 +- tools/deployment/package-helm/templates/queue-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/queue-secret.yaml | 2 +- tools/deployment/package-helm/templates/redis-data-pv.yaml | 2 +- tools/deployment/package-helm/templates/redis-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/redis-secret.yaml | 2 +- tools/deployment/package-helm/templates/reducer-logs-pv.yaml | 2 +- .../package-helm/templates/results-cache-data-pv.yaml | 2 +- .../package-helm/templates/results-cache-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/streams-pv.yaml | 2 +- tools/deployment/package-helm/templates/tmp-pv.yaml | 2 +- 23 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index f229201f9f..b0201e5b7d 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-api-server-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-api-server-logs" (include "clp.fullname" .)) "component" "api-server" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/archives-pv.yaml b/tools/deployment/package-helm/templates/archives-pv.yaml index bfd618723e..11072b5c9d 100644 --- a/tools/deployment/package-helm/templates/archives-pv.yaml +++ b/tools/deployment/package-helm/templates/archives-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-archives-pv" (include "clp.fullname" .)) + "name" (printf "%s-archives" (include "clp.fullname" .)) "component" "archives" "capacity" "50Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 14812f7384..6687600ebe 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-compression-scheduler-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-compression-scheduler-logs" (include "clp.fullname" .)) "component" "compression-scheduler" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index a1f5903e4f..c474d42bda 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-compression-worker-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-compression-worker-logs" (include "clp.fullname" .)) "component" "compression-worker" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index c10c71642c..6a52ad3e9c 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-staged-archives-pv" (include "clp.fullname" .)) + "name" (printf "%s-staged-archives" (include "clp.fullname" .)) "component" "staged-archives" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/database-data-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml index cf8bebe068..df4d58c1c2 100644 --- a/tools/deployment/package-helm/templates/database-data-pv.yaml +++ b/tools/deployment/package-helm/templates/database-data-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-database-data-pv" (include "clp.fullname" .)) + "name" (printf "%s-database-data" (include "clp.fullname" .)) "component" "database" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml index 92d8f548f9..0fdc598ee0 100644 --- a/tools/deployment/package-helm/templates/database-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-database-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-database-logs" (include "clp.fullname" .)) "component" "database" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/database-secret.yaml b/tools/deployment/package-helm/templates/database-secret.yaml index f2814d6877..64235ff4a7 100644 --- a/tools/deployment/package-helm/templates/database-secret.yaml +++ b/tools/deployment/package-helm/templates/database-secret.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" kind: "Secret" metadata: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database labels: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "database" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index 4b1e709b6f..eb3b1e4cc1 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-garbage-collector-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-garbage-collector-logs" (include "clp.fullname" .)) "component" "garbage-collector" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index ee6b204e65..469e292729 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-mcp-server-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-mcp-server-logs" (include "clp.fullname" .)) "component" "mcp-server" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index cf9b994d63..e4bc56060c 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-query-scheduler-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-query-scheduler-logs" (include "clp.fullname" .)) "component" "query-scheduler" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index fd52f15714..c4dc136698 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-query-worker-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-query-worker-logs" (include "clp.fullname" .)) "component" "query-worker" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index 134ec7d14c..993cb3bca1 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-staged-streams-pv" (include "clp.fullname" .)) + "name" (printf "%s-staged-streams" (include "clp.fullname" .)) "component" "staged-streams" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml index 0db0788eaa..b6ff44e134 100644 --- a/tools/deployment/package-helm/templates/queue-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-queue-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-queue-logs" (include "clp.fullname" .)) "component" "queue" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/queue-secret.yaml b/tools/deployment/package-helm/templates/queue-secret.yaml index 8b909ce22a..704e443b64 100644 --- a/tools/deployment/package-helm/templates/queue-secret.yaml +++ b/tools/deployment/package-helm/templates/queue-secret.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" kind: "Secret" metadata: - name: {{ include "clp.fullname" . }}-queue-secret + name: {{ include "clp.fullname" . }}-queue labels: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "queue" diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml index 139cfbed4e..bf170d9f8c 100644 --- a/tools/deployment/package-helm/templates/redis-data-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-redis-data-pv" (include "clp.fullname" .)) + "name" (printf "%s-redis-data" (include "clp.fullname" .)) "component" "redis" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml index d123c21d11..3a1fc890d3 100644 --- a/tools/deployment/package-helm/templates/redis-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-redis-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-redis-logs" (include "clp.fullname" .)) "component" "redis" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/redis-secret.yaml b/tools/deployment/package-helm/templates/redis-secret.yaml index 4923e0ce5d..244ccb5291 100644 --- a/tools/deployment/package-helm/templates/redis-secret.yaml +++ b/tools/deployment/package-helm/templates/redis-secret.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" kind: "Secret" metadata: - name: {{ include "clp.fullname" . }}-redis-secret + name: {{ include "clp.fullname" . }}-redis labels: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "redis" diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index 898c88d8c4..0e560d674f 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-reducer-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-reducer-logs" (include "clp.fullname" .)) "component" "reducer" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml index a74fa2cb66..52502fa827 100644 --- a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-results-cache-data-pv" (include "clp.fullname" .)) + "name" (printf "%s-results-cache-data" (include "clp.fullname" .)) "component" "results-cache" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index 89a8fbf892..61c8efa80b 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-results-cache-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-results-cache-logs" (include "clp.fullname" .)) "component" "results-cache" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/streams-pv.yaml b/tools/deployment/package-helm/templates/streams-pv.yaml index 477c23153d..f998090001 100644 --- a/tools/deployment/package-helm/templates/streams-pv.yaml +++ b/tools/deployment/package-helm/templates/streams-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-streams-pv" (include "clp.fullname" .)) + "name" (printf "%s-streams" (include "clp.fullname" .)) "component" "streams" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/tmp-pv.yaml b/tools/deployment/package-helm/templates/tmp-pv.yaml index 2ddba79f4c..43bdd2ca8c 100644 --- a/tools/deployment/package-helm/templates/tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/tmp-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-tmp-pv" (include "clp.fullname" .)) + "name" (printf "%s-tmp" (include "clp.fullname" .)) "component" "tmp" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") From 75c0101b49fa10969df79c5dd509e26086f004da Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 01:01:57 -0500 Subject: [PATCH 029/155] refactor(helm): Consolidate PVC definitions into `createPvc` and streamline component naming. --- .../package-helm/templates/_helpers.tpl | 28 +++++++++++++++++++ .../templates/api-server-deployment.yaml | 4 +-- .../templates/api-server-logs-pv.yaml | 2 +- .../templates/api-server-logs-pvc.yaml | 23 ++++----------- .../package-helm/templates/archives-pvc.yaml | 23 ++++----------- .../compression-scheduler-deployment.yaml | 4 +-- .../compression-scheduler-logs-pv.yaml | 2 +- .../compression-scheduler-logs-pvc.yaml | 23 ++++----------- .../templates/compression-worker-logs-pv.yaml | 2 +- .../compression-worker-logs-pvc.yaml | 23 ++++----------- ...ompression-worker-staged-archives-pvc.yaml | 23 ++++----------- .../templates/database-statefulset.yaml | 6 ++-- .../templates/db-table-creator-job.yaml | 4 +-- .../garbage-collector-deployment.yaml | 4 +-- .../templates/garbage-collector-logs-pv.yaml | 2 +- .../templates/garbage-collector-logs-pvc.yaml | 23 ++++----------- .../templates/mcp-server-deployment.yaml | 4 +-- .../templates/mcp-server-logs-pv.yaml | 2 +- .../templates/mcp-server-logs-pvc.yaml | 23 ++++----------- .../templates/query-scheduler-deployment.yaml | 4 +-- .../templates/query-scheduler-logs-pv.yaml | 2 +- .../templates/query-scheduler-logs-pvc.yaml | 23 ++++----------- .../templates/query-worker-archives-pvc.yaml | 17 ----------- .../templates/query-worker-logs-pv.yaml | 2 +- .../templates/query-worker-logs-pvc.yaml | 23 ++++----------- .../query-worker-staged-streams-pvc.yaml | 23 ++++----------- .../templates/queue-statefulset.yaml | 4 +-- .../templates/redis-statefulset.yaml | 2 +- .../templates/reducer-logs-pv.yaml | 2 +- .../templates/reducer-logs-pvc.yaml | 23 ++++----------- .../package-helm/templates/streams-pvc.yaml | 23 ++++----------- .../package-helm/templates/tmp-pvc.yaml | 23 ++++----------- .../templates/webui-deployment.yaml | 4 +-- 33 files changed, 134 insertions(+), 266 deletions(-) delete mode 100644 tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 249bcadaba..87fd33bc2d 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -112,4 +112,32 @@ spec: operator: "Exists" {{- end }} +{{/* +Creates a PersistentVolumeClaim. +Parameters (dict): + root: Root template context + component: Component label (used in name, labels, and selectors) + capacity: Storage capacity + accessModes: Access modes (list) +*/}} +{{- define "clp.createPvc" -}} +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" .root }}-{{ .component }} + labels: + {{- include "clp.labels" .root | nindent 4 }} + app.kubernetes.io/component: {{ .component | quote }} +spec: + accessModes: {{ .accessModes }} + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" .root | nindent 6 }} + app.kubernetes.io/component: {{ .component | quote }} + resources: + requests: + storage: {{ .capacity }} +{{- end }} + diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index a55769e02c..8bcf8f55f0 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -43,12 +43,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "CLP_LOGS_DIR" value: "/var/log/api_server" diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index b0201e5b7d..977be57d6f 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-api-server-logs" (include "clp.fullname" .)) - "component" "api-server" + "component" "api-server-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/api-server" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml index 53db492b5a..72d6f4702c 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-api-server-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "api-server" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "api-server" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "api-server-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/archives-pvc.yaml b/tools/deployment/package-helm/templates/archives-pvc.yaml index 2534d4ca78..c5b1bac44f 100644 --- a/tools/deployment/package-helm/templates/archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/archives-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-archives - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "archives" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "archives" - resources: - requests: - storage: "50Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "archives" + "capacity" "50Gi" + "accessModes" (list "ReadWriteMany") +) }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 9365b60327..510332695e 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -51,12 +51,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 6687600ebe..475a1ecc6b 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-compression-scheduler-logs" (include "clp.fullname" .)) - "component" "compression-scheduler" + "component" "compression-scheduler-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression-scheduler" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml index 9e5a5170d9..63011569c5 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-compression-scheduler-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "compression-scheduler" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "compression-scheduler" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "compression-scheduler-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index c474d42bda..7d8aeae9ba 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-compression-worker-logs" (include "clp.fullname" .)) - "component" "compression-worker" + "component" "compression-worker-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression-worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml index 617290ea74..45b11b1fad 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-compression-worker-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "compression-worker" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "compression-worker" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "compression-worker-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml index 541516ed3b..77410397a8 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-staged-archives - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "staged-archives" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "staged-archives" - resources: - requests: - storage: "20Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "staged-archives" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") +) }} diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 911adbd117..a193f83a96 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -32,17 +32,17 @@ spec: - name: "MYSQL_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "MYSQL_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "MYSQL_ROOT_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" ports: - name: "database" diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index f8f89a260a..4178adff85 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -32,12 +32,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 2369b9538a..e47bc8b8bd 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -43,12 +43,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "CLP_HOME" value: "/opt/clp" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index eb3b1e4cc1..8eaa2c9ea2 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-garbage-collector-logs" (include "clp.fullname" .)) - "component" "garbage-collector" + "component" "garbage-collector-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/garbage-collector" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml index ec16d99ae7..714e35bd94 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-garbage-collector-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "garbage-collector" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "garbage-collector" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "garbage-collector-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 436719558f..e651e7fb63 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -44,12 +44,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.mcp_server.logging_level | quote }} diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index 469e292729..da16ab4e3f 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-mcp-server-logs" (include "clp.fullname" .)) - "component" "mcp-server" + "component" "mcp-server-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/mcp-server" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml index 4ab1171bc2..f648fa9ccd 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml @@ -1,19 +1,8 @@ {{- if .Values.clpConfig.mcp_server }} -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-mcp-server-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "mcp-server" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "mcp-server" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "mcp-server-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} {{- end }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 8728244139..46a6299234 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -53,12 +53,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index e4bc56060c..8ec5ed13d4 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-query-scheduler-logs" (include "clp.fullname" .)) - "component" "query-scheduler" + "component" "query-scheduler-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query-scheduler" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml index 7b1c09aba6..bf9794ada9 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-query-scheduler-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "query-scheduler" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "query-scheduler" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "query-scheduler-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml deleted file mode 100644 index 20b4dd1c75..0000000000 --- a/tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-staged-streams - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "staged-streams" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "staged-streams" - resources: - requests: - storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index c4dc136698..007505daae 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-query-worker-logs" (include "clp.fullname" .)) - "component" "query-worker" + "component" "query-worker-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query-worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml index b3948b64aa..bfbc07a9d1 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-query-worker-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "query-worker" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "query-worker" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "query-worker-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml index 20b4dd1c75..195d94319e 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-staged-streams - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "staged-streams" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "staged-streams" - resources: - requests: - storage: "20Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "staged-streams" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") +) }} diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index 7c7245a5e7..406e423e34 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -30,12 +30,12 @@ spec: - name: "RABBITMQ_DEFAULT_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-queue-secret + name: {{ include "clp.fullname" . }}-queue key: "username" - name: "RABBITMQ_DEFAULT_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-queue-secret + name: {{ include "clp.fullname" . }}-queue key: "password" - name: "RABBITMQ_LOGS" value: "/var/log/rabbitmq/rabbitmq.log" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index c9ef5b8223..b938f9bd0e 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -34,7 +34,7 @@ spec: - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-redis-secret + name: {{ include "clp.fullname" . }}-redis key: "password" ports: - name: "redis" diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index 0e560d674f..3fd67c21b5 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-reducer-logs" (include "clp.fullname" .)) - "component" "reducer" + "component" "reducer-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/reducer" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml index e4502864c7..cd4eec1792 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-reducer-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "reducer" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "reducer" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "reducer-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/streams-pvc.yaml b/tools/deployment/package-helm/templates/streams-pvc.yaml index dfde2e91e1..bd754e6c42 100644 --- a/tools/deployment/package-helm/templates/streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/streams-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-streams - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "streams" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "streams" - resources: - requests: - storage: "20Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "streams" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") +) }} diff --git a/tools/deployment/package-helm/templates/tmp-pvc.yaml b/tools/deployment/package-helm/templates/tmp-pvc.yaml index c0e4cffad0..abdddcc887 100644 --- a/tools/deployment/package-helm/templates/tmp-pvc.yaml +++ b/tools/deployment/package-helm/templates/tmp-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-tmp - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "tmp" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "tmp" - resources: - requests: - storage: "10Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "tmp" + "capacity" "10Gi" + "accessModes" (list "ReadWriteMany") +) }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 23c2b57320..6ece053202 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -43,12 +43,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "HOST" value: "0.0.0.0" From 1476b718332e43624afde031b50af937c6dddda0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 02:18:51 -0500 Subject: [PATCH 030/155] Add user-logs PVC and PV definitions. --- .../templates/compression-scheduler-deployment.yaml | 5 +++++ tools/deployment/package-helm/templates/user-logs-pv.yaml | 8 ++++++++ .../deployment/package-helm/templates/user-logs-pvc.yaml | 6 ++++++ tools/deployment/package-helm/test.sh | 1 + 4 files changed, 20 insertions(+) create mode 100644 tools/deployment/package-helm/templates/user-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/user-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 510332695e..a0af4204ca 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -73,6 +73,8 @@ spec: readOnly: true - name: "compression-scheduler-logs" mountPath: "/var/log/compression_scheduler" + - name: "user-logs" + mountPath: "/var/log/user" - name: "logs-input" mountPath: "/mnt/logs" readOnly: true @@ -88,6 +90,9 @@ spec: - name: "compression-scheduler-logs" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-compression-scheduler-logs + - name: "user-logs" + persistentVolumeClaim: + claimName: {{ include "clp.fullname" . }}-user-logs - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" diff --git a/tools/deployment/package-helm/templates/user-logs-pv.yaml b/tools/deployment/package-helm/templates/user-logs-pv.yaml new file mode 100644 index 0000000000..b90b351a48 --- /dev/null +++ b/tools/deployment/package-helm/templates/user-logs-pv.yaml @@ -0,0 +1,8 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-user-logs" (include "clp.fullname" .)) + "component" "user-logs" + "capacity" "10Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" (printf "%s/user" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/user-logs-pvc.yaml b/tools/deployment/package-helm/templates/user-logs-pvc.yaml new file mode 100644 index 0000000000..2fb0688dc2 --- /dev/null +++ b/tools/deployment/package-helm/templates/user-logs-pvc.yaml @@ -0,0 +1,6 @@ +{{- include "clp.createPvc" (dict + "root" . + "component" "user-logs" + "capacity" "10Gi" + "accessModes" (list "ReadWriteMany") +) }} diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 33fbc61651..73927a489a 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -6,6 +6,7 @@ kind delete cluster --name clp-test sudo rm -rf /tmp/clp mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker,reducer,garbage-collector,api-server,mcp-server} mkdir -p /tmp/clp/var/data/{archives,staged-archives,staged-streams,streams} +mkdir -p /tmp/clp/var/log/user mkdir -p /tmp/clp/var/tmp mkdir -p /tmp/clp/samples From ad82e2565587073c90d73acae2025780e8b56957 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:04:31 -0500 Subject: [PATCH 031/155] fix(helm): Remove unnecessary image tag override in test script. --- tools/deployment/package-helm/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 73927a489a..d1b85b6dcc 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -40,5 +40,5 @@ cp /home/junhao/samples/postgresql.jsonl /tmp/clp/postgresql.jsonl helm uninstall test || true sleep 2 -helm install test . --set image.clpPackage.tag=nightly +helm install test . ls -l /tmp/clp/var/data/ From 29fe3c0126ec04139c4d33899f297ba5eefa3908 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:04:37 -0500 Subject: [PATCH 032/155] feat(helm): Add conditional database image selection based on configuration. --- .../package-helm/templates/database-statefulset.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index a193f83a96..65f9a21904 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -24,7 +24,12 @@ spec: fsGroup: {{ .Values.securityContext.thirdParty.gid }} containers: - name: "database" - image: "mariadb:10-jammy" + image: >- + {{- if eq .Values.clpConfig.database.type "mysql" }} + mysql:8.0.23 + {{- else }} + mariadb:10-jammy + {{- end }} imagePullPolicy: "Always" env: - name: "MYSQL_DATABASE" From c5bb582a6bb4d4d5023b1f0ed8d386f7ab4d9400 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:13:42 -0500 Subject: [PATCH 033/155] feat(helm): Add persistentVolumeClaimRetentionPolicy to statefulsets. --- .../deployment/package-helm/templates/database-statefulset.yaml | 2 ++ tools/deployment/package-helm/templates/queue-statefulset.yaml | 2 ++ tools/deployment/package-helm/templates/redis-statefulset.yaml | 2 ++ .../package-helm/templates/results-cache-statefulset.yaml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 65f9a21904..b1365ab412 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -77,6 +77,8 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" volumeClaimTemplates: - metadata: name: "database-data" diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index 406e423e34..bea559736a 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -53,6 +53,8 @@ spec: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: <<: *queue-health-check + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" volumeClaimTemplates: - metadata: name: "queue-logs" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index b938f9bd0e..245601fb68 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -68,6 +68,8 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" volumeClaimTemplates: - metadata: name: "redis-data" diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 818093fde7..441603be4e 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -60,6 +60,8 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" volumeClaimTemplates: - metadata: name: "results-cache-data" From db9cef95d80b01ced073a83a49f2845d0abf2c86 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:13:52 -0500 Subject: [PATCH 034/155] feat(helm): Add ttlSecondsAfterFinished to job templates. --- .../deployment/package-helm/templates/db-table-creator-job.yaml | 1 + .../templates/results-cache-indices-creator-job.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index 4178adff85..49f242a6e9 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -6,6 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "db-table-creator" spec: + ttlSecondsAfterFinished: 300 template: metadata: labels: diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index 4b45368f3f..5ff701fa3a 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -6,6 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "results-cache-indices-creator" spec: + ttlSecondsAfterFinished: 300 template: metadata: labels: From 1503c494bda9a53779424ca82344e018dd665e40 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:37:31 -0500 Subject: [PATCH 035/155] feat(deployment): set up initial Helm chart for CLP package: - Add database StatefulSet and db-table-creator Job. - Introduce `lint:helm` task to lint the chart using chart-testing (ct). - Update CI workflow to include Helm chart linting. --- .github/workflows/clp-lint.yaml | 4 + chart_schema.yaml | 38 +++++ .../clp_package_utils/general.py | 4 +- .../clp-py-utils/clp_py_utils/clp_config.py | 4 +- .../src/etc/credentials.template.yml | 4 +- ct.yaml | 8 + taskfiles/lint.yaml | 37 +++-- taskfiles/toolchains.yaml | 38 +++++ tools/deployment/package-helm/.helmignore | 23 +++ tools/deployment/package-helm/Chart.yaml | 17 +++ .../package-helm/templates/_helpers.tpl | 141 ++++++++++++++++++ .../package-helm/templates/configmap.yaml | 33 ++++ .../templates/database-data-pv.yaml | 8 + .../templates/database-logs-pv.yaml | 8 + .../templates/database-secret.yaml | 11 ++ .../templates/database-service.yaml | 22 +++ .../templates/database-statefulset.yaml | 106 +++++++++++++ .../templates/db-table-creator-job.yaml | 59 ++++++++ tools/deployment/package-helm/test.sh | 26 ++++ tools/deployment/package-helm/values.yaml | 47 ++++++ tools/deployment/presto-clp/scripts/init.py | 6 +- 21 files changed, 625 insertions(+), 19 deletions(-) create mode 100644 chart_schema.yaml create mode 100644 ct.yaml create mode 100644 tools/deployment/package-helm/.helmignore create mode 100644 tools/deployment/package-helm/Chart.yaml create mode 100644 tools/deployment/package-helm/templates/_helpers.tpl create mode 100644 tools/deployment/package-helm/templates/configmap.yaml create mode 100644 tools/deployment/package-helm/templates/database-data-pv.yaml create mode 100644 tools/deployment/package-helm/templates/database-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/database-secret.yaml create mode 100644 tools/deployment/package-helm/templates/database-service.yaml create mode 100644 tools/deployment/package-helm/templates/database-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/db-table-creator-job.yaml create mode 100755 tools/deployment/package-helm/test.sh create mode 100644 tools/deployment/package-helm/values.yaml diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml index 1d97737e74..933ac97325 100644 --- a/.github/workflows/clp-lint.yaml +++ b/.github/workflows/clp-lint.yaml @@ -78,3 +78,7 @@ jobs: - name: "Lint .yaml files" shell: "bash" run: "task lint:check-yaml" + + - name: "Lint Helm charts" + shell: "bash" + run: "task lint:check-helm" diff --git a/chart_schema.yaml b/chart_schema.yaml new file mode 100644 index 0000000000..4f1d69b73b --- /dev/null +++ b/chart_schema.yaml @@ -0,0 +1,38 @@ +# yamllint disable rule:document-start +name: "str()" +home: "str(required=False)" +version: "str()" +apiVersion: "str()" +appVersion: "any(str(), num(), required=False)" +description: "str(required=False)" +keywords: "list(str(), required=False)" +sources: "list(str(), required=False)" +maintainers: "list(include('maintainer'), required=False)" +dependencies: "list(include('dependency'), required=False)" +icon: "str(required=False)" +engine: "str(required=False)" +condition: "str(required=False)" +tags: "str(required=False)" +deprecated: "bool(required=False)" +kubeVersion: "str(required=False)" +annotations: "map(str(), str(), required=False)" +type: "str(required=False)" +--- +maintainer: + name: "str()" + email: "str(required=False)" + url: "str(required=False)" +--- +dependency: + name: "str()" + version: "str()" + repository: "str(required=False)" + condition: "str(required=False)" + tags: "list(str(), required=False)" + enabled: "bool(required=False)" + import-values: "list(any(str(), include('import-value')), required=False)" + alias: "str(required=False)" +--- +import-value: + child: "str()" + parent: "str()" diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a1c5fb5e15..d0ad00034c 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -453,8 +453,8 @@ def load_config_file( def generate_credentials_file(credentials_file_path: pathlib.Path): credentials = { - DB_COMPONENT_NAME: {"user": "clp-user", "password": secrets.token_urlsafe(8)}, - QUEUE_COMPONENT_NAME: {"user": "clp-user", "password": secrets.token_urlsafe(8)}, + DB_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)}, + QUEUE_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)}, REDIS_COMPONENT_NAME: {"password": secrets.token_urlsafe(16)}, } diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index c35a30dc0f..9e791754b0 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -232,7 +232,7 @@ def load_credentials_from_file(self, credentials_file_path: pathlib.Path): if config is None: raise ValueError(f"Credentials file '{credentials_file_path}' is empty.") try: - self.username = get_config_value(config, f"{DB_COMPONENT_NAME}.user") + self.username = get_config_value(config, f"{DB_COMPONENT_NAME}.username") self.password = get_config_value(config, f"{DB_COMPONENT_NAME}.password") except KeyError as ex: raise ValueError( @@ -360,7 +360,7 @@ def load_credentials_from_file(self, credentials_file_path: pathlib.Path): if config is None: raise ValueError(f"Credentials file '{credentials_file_path}' is empty.") try: - self.username = get_config_value(config, f"{QUEUE_COMPONENT_NAME}.user") + self.username = get_config_value(config, f"{QUEUE_COMPONENT_NAME}.username") self.password = get_config_value(config, f"{QUEUE_COMPONENT_NAME}.password") except KeyError as ex: raise ValueError( diff --git a/components/package-template/src/etc/credentials.template.yml b/components/package-template/src/etc/credentials.template.yml index 9a0dd73fa2..eb2f9b73f7 100644 --- a/components/package-template/src/etc/credentials.template.yml +++ b/components/package-template/src/etc/credentials.template.yml @@ -1,11 +1,11 @@ ## Database credentials #database: -# user: "user" +# username: "clp-user" # password: "pass" # ## Queue credentials #queue: -# user: "user" +# username: "clp-user" # password: "pass" # ## Redis credentials diff --git a/ct.yaml b/ct.yaml new file mode 100644 index 0000000000..159fec6d15 --- /dev/null +++ b/ct.yaml @@ -0,0 +1,8 @@ +chart-dirs: + - "tools/deployment" +validate-maintainers: true +validate-chart-schema: true +check-version-increment: true +validate-yaml: true +lint-conf: ".yamllint.yml" +chart-yaml-schema: "chart_schema.yaml" diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 29b6dcd254..74b792b91b 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -14,6 +14,7 @@ tasks: check: cmds: - task: "check-cpp-full" + - task: "check-helm" - task: "check-js" - task: "check-py" - task: "check-rust" @@ -32,6 +33,7 @@ tasks: fix: cmds: - task: "fix-cpp-full" + - task: "fix-helm" - task: "fix-js" - task: "fix-py" - task: "fix-rust" @@ -54,6 +56,18 @@ tasks: - task: "fix-rust" - task: "fix-yaml" + helm: + aliases: + - "check-helm" + - "fix-helm" + deps: + - "toolchains:helm" + - "venv" + cmds: + - |- + . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + ct lint + check-js: sources: &js_source_files - "{{.G_BUILD_DIR}}/webui-node-modules.md5" @@ -98,16 +112,19 @@ tasks: - |- . "{{.G_LINT_VENV_DIR}}/bin/activate" yamllint --strict \ - .github \ - .yamllint.yml \ - components/core/.clang-format \ - components/core/config \ - components/core/tools/ \ - components/package-template/src/etc \ - docs \ - taskfile.yaml \ - taskfiles \ - tools/deployment + ".github" \ + ".yamllint.yml" \ + "components/core/.clang-format" \ + "components/core/config" \ + "components/core/tools/" \ + "components/package-template/src/etc" \ + "chart_schema.yaml" \ + "ct.yaml" \ + "docs" \ + "taskfile.yaml" \ + "taskfiles" \ + "tools/deployment/package" \ + "tools/deployment/presto-clp" check-cpp-format: sources: &cpp_source_files diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 53a6c08f30..438e5ea23f 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -1,9 +1,12 @@ version: "3" includes: + remote: "../tools/yscope-dev-utils/exports/taskfiles/utils/remote.yaml" utils: "../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" vars: + G_HELM_TOOLCHAIN_DIR: "{{.G_BUILD_DIR}}/toolchains/helm" + G_HELM_TOOLCHAIN_ENV_FILE: "{{.G_HELM_TOOLCHAIN_DIR}}/env" G_RUST_TOOLCHAIN_DIR: "{{.G_BUILD_DIR}}/toolchains/rust" G_RUST_TOOLCHAIN_ENV_FILE: "{{.G_RUST_TOOLCHAIN_DIR}}/env" @@ -15,6 +18,41 @@ vars: G_RUST_TOOLCHAIN_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/toolchains-rust.md5" tasks: + helm: + run: "once" + vars: + CHART_TESTING_VERSION: "3.14.0" + cmds: + - "mkdir -p '{{.G_HELM_TOOLCHAIN_DIR}}'" + - task: "remote:download-and-extract-tar" + vars: + OUTPUT_DIR: "{{.G_BUILD_DIR}}/toolchains/helm" + URL: "https://github.com/helm/chart-testing/releases/download/v{{.CHART_TESTING_VERSION}}\ + /chart-testing_{{.CHART_TESTING_VERSION}}_{{OS}}_{{ARCH}}.tar.gz" + FILE_SHA256: >- + {{- if and (eq OS "linux") (eq ARCH "amd64") -}} + d16f0583616885423826241164ce1f6589c6fe5332fa74f374ebd2bd3cb3fe1f + {{- else if and (eq OS "darwin") (eq ARCH "arm64") -}} + db10dbbb42b110c7a5da5a3202908f32ad2ca6ad600d423426dc7886d09aad07 + {{- else -}} + unsupported + {{- end }} + INCLUDE_PATTERNS: ["ct"] + NUM_COMPONENTS_TO_STRIP: "0" + - |- + export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" + export HELM_INSTALL_DIR="{{.G_HELM_TOOLCHAIN_DIR}}" + curl --proto "=https" --tlsv1.2 --silent --show-error --fail \ + https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-4 | bash -s -- --no-sudo + + # Create an environment file to configure the custom Helm toolchain location. + - |- + cat < "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + #!/bin/sh + export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" + EOF + chmod +x "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + rust: # We use a label to uniquely identify this task and its checksum under `.task/checksums/`. label: "toolchains-rust" diff --git a/tools/deployment/package-helm/.helmignore b/tools/deployment/package-helm/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/tools/deployment/package-helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml new file mode 100644 index 0000000000..ba2ac6ee3d --- /dev/null +++ b/tools/deployment/package-helm/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: "v2" +name: "clp" +description: "A Helm chart for CLP (Compressed Log Processor) package deployment" +type: "application" +version: "0.1.0" +appVersion: "0.6.1-dev" +home: "https://github.com/y-scope/clp" +icon: "https://github.com/y-scope/clp/raw/main/docs/src/clp-logo.png" +sources: ["https://github.com/y-scope/clp"] +maintainers: [{name: "y-scope", email: "dev@yscope.com"}] +keywords: + - "analytics" + - "compression" + - "log-management" + - "log-parser" + - "logging" + - "search" diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl new file mode 100644 index 0000000000..658953f248 --- /dev/null +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -0,0 +1,141 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "clp.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "clp.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "clp.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "clp.labels" -}} +helm.sh/chart: {{ include "clp.chart" . }} +{{ include "clp.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "clp.selectorLabels" -}} +app.kubernetes.io/name: {{ include "clp.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Image reference for CLP Package +*/}} +{{- define "clp.image.ref" -}} +{{- $tag := .Values.image.clpPackage.tag | default .Chart.AppVersion }} +{{- printf "%s:%s" .Values.image.clpPackage.repository $tag }} +{{- end }} + +{{/* +Creates timings for readiness probes (faster checks for quicker startup). +*/}} +{{- define "clp.readinessProbeTimings" -}} +initialDelaySeconds: 1 +periodSeconds: 1 +timeoutSeconds: 1 +failureThreshold: 3 +{{- end }} + +{{/* +Creates timings for liveness probes. +*/}} +{{- define "clp.livenessProbeTimings" -}} +initialDelaySeconds: 30 +periodSeconds: 10 +timeoutSeconds: 5 +failureThreshold: 3 +{{- end }} + +{{/* +Creates a local PersistentVolume. +Parameters (dict): + root: Root template context + name: PV name (full) + component: Component label + capacity: Storage capacity + accessModes: Access modes (list) + hostPath: Absolute path on host +*/}} +{{- define "clp.createLocalPv" -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ .name }} + labels: + {{- include "clp.labels" .root | nindent 4 }} + app.kubernetes.io/component: {{ .component | quote }} +spec: + capacity: + storage: {{ .capacity }} + accessModes: {{ .accessModes }} + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ .hostPath }} + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + +{{/* +Creates a PersistentVolumeClaim. +Parameters (dict): + root: Root template context + component: Component label (used in name, labels, and selectors) + capacity: Storage capacity + accessModes: Access modes (list) +*/}} +{{- define "clp.createPvc" -}} +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" .root }}-{{ .component }} + labels: + {{- include "clp.labels" .root | nindent 4 }} + app.kubernetes.io/component: {{ .component | quote }} +spec: + accessModes: {{ .accessModes }} + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" .root | nindent 6 }} + app.kubernetes.io/component: {{ .component | quote }} + resources: + requests: + storage: {{ .capacity }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml new file mode 100644 index 0000000000..ee9fc207ae --- /dev/null +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -0,0 +1,33 @@ +apiVersion: "v1" +kind: "ConfigMap" +metadata: + name: {{ include "clp.fullname" . }}-config + labels: + {{- include "clp.labels" . | nindent 4 }} +data: + clp-config.yml: | + data_directory: "/var/data" + database: + auto_commit: false + compress: true + host: {{ include "clp.fullname" . }}-database + name: {{ .Values.clpConfig.database.name | quote }} + port: 3306 + ssl_cert: null + type: {{ .Values.clpConfig.database.type | quote }} + logs_directory: "/var/log" + package: + query_engine: {{ .Values.clpConfig.package.query_engine | quote }} + storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} + + mysql-logging.cnf: | + [mysqld] + # https://dev.mysql.com/doc/refman/8.0/en/log-destinations.html + # Enable the general log + general_log=1 + # Set the destination file + general_log_file=/var/log/mysql/mysql.log + + # https://dev.mysql.com/doc/refman/8.0/en/error-log-destination-configuration.html + # Send error logs to file + log_error=/var/log/mysql/mysql-error.log diff --git a/tools/deployment/package-helm/templates/database-data-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml new file mode 100644 index 0000000000..df4d58c1c2 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-data-pv.yaml @@ -0,0 +1,8 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-database-data" (include "clp.fullname" .)) + "component" "database" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/database" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml new file mode 100644 index 0000000000..0fdc598ee0 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -0,0 +1,8 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-database-logs" (include "clp.fullname" .)) + "component" "database" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/database" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/database-secret.yaml b/tools/deployment/package-helm/templates/database-secret.yaml new file mode 100644 index 0000000000..64235ff4a7 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp.fullname" . }}-database + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +type: "Opaque" +stringData: + username: {{ .Values.credentials.database.username | quote }} + password: {{ .Values.credentials.database.password | quote }} diff --git a/tools/deployment/package-helm/templates/database-service.yaml b/tools/deployment/package-helm/templates/database-service.yaml new file mode 100644 index 0000000000..fe6f2f823e --- /dev/null +++ b/tools/deployment/package-helm/templates/database-service.yaml @@ -0,0 +1,22 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-database + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +spec: + {{- if .Values.allowSbinScripts }} + type: "NodePort" + {{- else }} + clusterIP: "None" + {{- end }} + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "database" + ports: + - port: 3306 + targetPort: "database" + {{- if .Values.allowSbinScripts }} + nodePort: {{ .Values.clpConfig.database.port }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml new file mode 100644 index 0000000000..b1365ab412 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -0,0 +1,106 @@ +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp.fullname" . }}-database + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +spec: + serviceName: {{ include "clp.fullname" . }}-database + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "database" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "database" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "database" + image: >- + {{- if eq .Values.clpConfig.database.type "mysql" }} + mysql:8.0.23 + {{- else }} + mariadb:10-jammy + {{- end }} + imagePullPolicy: "Always" + env: + - name: "MYSQL_DATABASE" + value: {{ .Values.clpConfig.database.name | quote }} + - name: "MYSQL_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "username" + - name: "MYSQL_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "password" + - name: "MYSQL_ROOT_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "password" + ports: + - name: "database" + containerPort: 3306 + volumeMounts: + - name: "database-data" + mountPath: "/var/lib/mysql" + - name: "database-logs" + mountPath: "/var/log/mysql" + - name: "config" + mountPath: "/etc/mysql/conf.d/logging.cnf" + subPath: "mysql-logging.cnf" + readOnly: true + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: &database-health-check + command: [ + "mysqladmin", "ping", + "--silent", + "-h", "127.0.0.1" + ] + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: + <<: *database-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" + volumeClaimTemplates: + - metadata: + name: "database-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "database" + resources: + requests: + storage: "20Gi" + - metadata: + name: "database-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "database" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml new file mode 100644 index 0000000000..49f242a6e9 --- /dev/null +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -0,0 +1,59 @@ +apiVersion: "batch/v1" +kind: "Job" +metadata: + name: {{ include "clp.fullname" . }}-db-table-creator + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "db-table-creator" +spec: + ttlSecondsAfterFinished: 300 + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "db-table-creator" + spec: + restartPolicy: "OnFailure" + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + containers: + - name: "db-table-creator" + image: "{{ include "clp.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "password" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + command: [ + "python3", "-u", + "-m", "clp_py_utils.create-db-tables", + "--config", "/etc/clp-config.yml", + "--storage-engine", {{ .Values.clpConfig.package.storage_engine | quote }} + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh new file mode 100755 index 0000000000..73587b1570 --- /dev/null +++ b/tools/deployment/package-helm/test.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# TODO: to be deleted / migrated to integration tests + +kind delete cluster --name clp-test +rm -rf /tmp/clp +mkdir -p /tmp/clp/var/{data,log}/database + +cat < Date: Mon, 17 Nov 2025 03:40:05 -0500 Subject: [PATCH 036/155] chore(deployment): remove pod watch from test script. --- tools/deployment/package-helm/test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 73587b1570..1111d8d985 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -23,4 +23,3 @@ EOF helm uninstall test || true sleep 2 helm install test . -kubectl get pods -w \ No newline at end of file From 0b4c4fd54717028178eec46e7832a2a4b13dd5b1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:49:00 -0500 Subject: [PATCH 037/155] chore(deployment): update chart-testing config to set target branch. --- ct.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ct.yaml b/ct.yaml index 159fec6d15..dd82566d74 100644 --- a/ct.yaml +++ b/ct.yaml @@ -1,8 +1,9 @@ chart-dirs: - "tools/deployment" -validate-maintainers: true -validate-chart-schema: true +chart-yaml-schema: "chart_schema.yaml" check-version-increment: true -validate-yaml: true lint-conf: ".yamllint.yml" -chart-yaml-schema: "chart_schema.yaml" +target-branch: HEAD +validate-chart-schema: true +validate-maintainers: true +validate-yaml: true From f12557de3dd867210891ea5d74de4e1fd167525a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:51:45 -0500 Subject: [PATCH 038/155] quote string --- ct.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ct.yaml b/ct.yaml index dd82566d74..0333e87bc8 100644 --- a/ct.yaml +++ b/ct.yaml @@ -3,7 +3,7 @@ chart-dirs: chart-yaml-schema: "chart_schema.yaml" check-version-increment: true lint-conf: ".yamllint.yml" -target-branch: HEAD +target-branch: "HEAD" validate-chart-schema: true validate-maintainers: true validate-yaml: true From 5713eb31c0b6475aabdcfc351645553dcc56af1e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 04:00:06 -0500 Subject: [PATCH 039/155] fix missing yamale --- lint-requirements.txt | 8 +++++++- taskfiles/lint.yaml | 1 + taskfiles/toolchains.yaml | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index a5e2ef60d9..2cab04f19a 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,6 +1,12 @@ -black>=24.4.2 +# C++ clang-format>=21.1.2 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 + +# Python +black>=24.4.2 ruff>=0.4.4 + +# YAML yamllint>=1.35.1 +yamale>=6.0.0 \ No newline at end of file diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 74b792b91b..8f13303004 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -65,6 +65,7 @@ tasks: - "venv" cmds: - |- + . "{{.G_LINT_VENV_DIR}}/bin/activate" . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" ct lint diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 438e5ea23f..1df385ef74 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -47,7 +47,7 @@ tasks: # Create an environment file to configure the custom Helm toolchain location. - |- - cat < "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + cat <<"EOF" > "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" #!/bin/sh export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" EOF From fa2522eea0c170aa52354ad489ee8506b916786e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:51:45 -0500 Subject: [PATCH 040/155] quote string --- ct.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ct.yaml b/ct.yaml index 159fec6d15..0333e87bc8 100644 --- a/ct.yaml +++ b/ct.yaml @@ -1,8 +1,9 @@ chart-dirs: - "tools/deployment" -validate-maintainers: true -validate-chart-schema: true +chart-yaml-schema: "chart_schema.yaml" check-version-increment: true -validate-yaml: true lint-conf: ".yamllint.yml" -chart-yaml-schema: "chart_schema.yaml" +target-branch: "HEAD" +validate-chart-schema: true +validate-maintainers: true +validate-yaml: true From 8d176fecc8c6ba37e15887220fbc4278f0eb03ca Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 04:00:06 -0500 Subject: [PATCH 041/155] fix missing yamale --- taskfiles/lint.yaml | 25 +++++++++++++------------ taskfiles/toolchains.yaml | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 21597d13ec..8f13303004 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -56,6 +56,19 @@ tasks: - task: "fix-rust" - task: "fix-yaml" + helm: + aliases: + - "check-helm" + - "fix-helm" + deps: + - "toolchains:helm" + - "venv" + cmds: + - |- + . "{{.G_LINT_VENV_DIR}}/bin/activate" + . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + ct lint + check-js: sources: &js_source_files - "{{.G_BUILD_DIR}}/webui-node-modules.md5" @@ -91,18 +104,6 @@ tasks: RUFF_CHECK_FLAGS: "--fix" RUFF_FORMAT_FLAGS: "" - helm: - aliases: - - "check-helm" - - "fix-helm" - deps: - - "toolchains:helm" - - "venv" - cmds: - - |- - . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" - ct lint - yaml: aliases: - "check-yaml" diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 438e5ea23f..1df385ef74 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -47,7 +47,7 @@ tasks: # Create an environment file to configure the custom Helm toolchain location. - |- - cat < "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + cat <<"EOF" > "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" #!/bin/sh export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" EOF From e1fe43659fee74eb68e5674a90f3733596b02288 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 04:17:17 -0500 Subject: [PATCH 042/155] chore(ci): Set fetch-depth to 0 in clp-lint workflow. --- .github/workflows/clp-lint.yaml | 1 + ct.yaml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml index 933ac97325..68eb65a617 100644 --- a/.github/workflows/clp-lint.yaml +++ b/.github/workflows/clp-lint.yaml @@ -25,6 +25,7 @@ jobs: steps: - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" with: + fetch-depth: 0 submodules: "recursive" - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" diff --git a/ct.yaml b/ct.yaml index 0333e87bc8..11f80fb3ad 100644 --- a/ct.yaml +++ b/ct.yaml @@ -3,7 +3,6 @@ chart-dirs: chart-yaml-schema: "chart_schema.yaml" check-version-increment: true lint-conf: ".yamllint.yml" -target-branch: "HEAD" validate-chart-schema: true validate-maintainers: true validate-yaml: true From e4c0f2218866cdbba5a9f8d5504d84c991f4f200 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 04:23:16 -0500 Subject: [PATCH 043/155] fix(deployment): Adjust readinessProbe timeout and failure threshold. --- tools/deployment/package-helm/templates/_helpers.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 658953f248..314b97131a 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -64,8 +64,8 @@ Creates timings for readiness probes (faster checks for quicker startup). {{- define "clp.readinessProbeTimings" -}} initialDelaySeconds: 1 periodSeconds: 1 -timeoutSeconds: 1 -failureThreshold: 3 +timeoutSeconds: 5 +failureThreshold: 10 {{- end }} {{/* From 7fe32b774262cbe8db5602e44e07b5cab05fc6c9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 05:04:48 -0500 Subject: [PATCH 044/155] chore(ci): Add comment explaining fetch-depth requirement in clp-lint workflow. --- .github/workflows/clp-lint.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml index 68eb65a617..264d5e21c1 100644 --- a/.github/workflows/clp-lint.yaml +++ b/.github/workflows/clp-lint.yaml @@ -25,9 +25,12 @@ jobs: steps: - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" with: - fetch-depth: 0 submodules: "recursive" + # Fetch all history for all branches; otherwise, `helm lint` would complain about not + # finding the `origin/main` branch. + fetch-depth: 0 + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" with: python-version: "3.11" From 6c89ed656d8238299a91315c5116bdaf7fccbb07 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 06:26:06 -0500 Subject: [PATCH 045/155] Reformat database Job initContainer command for readability. --- .../package-helm/templates/db-table-creator-job.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index 49f242a6e9..66cbace10b 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -21,10 +21,14 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo waiting for database; + sleep 2; + done containers: - name: "db-table-creator" image: "{{ include "clp.image.ref" . }}" From e40d239ba49fdc871676b293742240d26f3eb6e9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 11:50:22 -0500 Subject: [PATCH 046/155] Update database livenessProbe to include authentication parameters. --- .../templates/database-statefulset.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index b1365ab412..82df032604 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -64,11 +64,15 @@ spec: livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: &database-health-check - command: [ - "mysqladmin", "ping", - "--silent", - "-h", "127.0.0.1" - ] + command: + - "sh" + - "-c" + - >- + mysqladmin ping + --host=127.0.0.1 + --port=3306 + --user=${MYSQL_USER} + --password=${MYSQL_PASSWORD} readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: From f17264ea11a577075f71b8b356f4f7fb8f0e47f6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 11:52:24 -0500 Subject: [PATCH 047/155] alphabetize --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index 2cab04f19a..49300e3f8c 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -8,5 +8,5 @@ black>=24.4.2 ruff>=0.4.4 # YAML +yamale>=6.0.0 yamllint>=1.35.1 -yamale>=6.0.0 \ No newline at end of file From f51b97eabb0f76411ef3b03fe13ed8e2c5f0fbbf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 11:52:34 -0500 Subject: [PATCH 048/155] Adjust probe timings for readiness and liveness checks. --- tools/deployment/package-helm/templates/_helpers.tpl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 314b97131a..33687f6f97 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -62,9 +62,9 @@ Image reference for CLP Package Creates timings for readiness probes (faster checks for quicker startup). */}} {{- define "clp.readinessProbeTimings" -}} -initialDelaySeconds: 1 -periodSeconds: 1 -timeoutSeconds: 5 +initialDelaySeconds: 6 +periodSeconds: 2 +timeoutSeconds: 2 failureThreshold: 10 {{- end }} @@ -72,9 +72,9 @@ failureThreshold: 10 Creates timings for liveness probes. */}} {{- define "clp.livenessProbeTimings" -}} -initialDelaySeconds: 30 +initialDelaySeconds: 60 periodSeconds: 10 -timeoutSeconds: 5 +timeoutSeconds: 4 failureThreshold: 3 {{- end }} From 8ed32a7f020ff4e4954fc2813ae43b862cd6dd23 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 12:19:01 -0500 Subject: [PATCH 049/155] add url to maintainers - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index ba2ac6ee3d..e7dfe42d3c 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -7,7 +7,7 @@ appVersion: "0.6.1-dev" home: "https://github.com/y-scope/clp" icon: "https://github.com/y-scope/clp/raw/main/docs/src/clp-logo.png" sources: ["https://github.com/y-scope/clp"] -maintainers: [{name: "y-scope", email: "dev@yscope.com"}] +maintainers: [{name: "y-scope", email: "dev@yscope.com", url: "https://yscope.com"}] keywords: - "analytics" - "compression" From 45674e2503dac25cb65d1193d5d69cda5f9fb866 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 12:14:18 -0500 Subject: [PATCH 050/155] Standardize version field placement in Chart.yaml. --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index e7dfe42d3c..f156ec8a54 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: "v2" name: "clp" +version: "0.1.0" description: "A Helm chart for CLP (Compressed Log Processor) package deployment" type: "application" -version: "0.1.0" appVersion: "0.6.1-dev" home: "https://github.com/y-scope/clp" icon: "https://github.com/y-scope/clp/raw/main/docs/src/clp-logo.png" From 2ad9b0aa0020ccf1cd0082d3bfe3843275c1ef40 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 12:44:18 -0500 Subject: [PATCH 051/155] Adjust livenessProbe periodSeconds to 30. --- tools/deployment/package-helm/templates/_helpers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 33687f6f97..3920d4eb18 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -73,7 +73,7 @@ Creates timings for liveness probes. */}} {{- define "clp.livenessProbeTimings" -}} initialDelaySeconds: 60 -periodSeconds: 10 +periodSeconds: 30 timeoutSeconds: 4 failureThreshold: 3 {{- end }} From 5d5bd27894910920e1671945708420cfc5905905 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 14:15:45 -0500 Subject: [PATCH 052/155] Swap liveness and readiness probe configurations in database-statefulset.yaml --- .../package-helm/templates/database-statefulset.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 82df032604..06347e5062 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -61,8 +61,8 @@ spec: mountPath: "/etc/mysql/conf.d/logging.cnf" subPath: "mysql-logging.cnf" readOnly: true - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &database-health-check command: - "sh" @@ -73,8 +73,8 @@ spec: --port=3306 --user=${MYSQL_USER} --password=${MYSQL_PASSWORD} - readinessProbe: - {{- include "clp.readinessProbeTimings" . | nindent 12 }} + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: <<: *database-health-check volumes: From fb2d28757927700a4311ceb1860da97e0130038d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 22 Nov 2025 02:28:39 -0500 Subject: [PATCH 053/155] Update all `.yml` extensions to `.yaml` for consistency --- ct.yaml | 2 +- tools/deployment/package-helm/templates/configmap.yaml | 2 +- .../package-helm/templates/db-table-creator-job.yaml | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ct.yaml b/ct.yaml index 11f80fb3ad..95717fd2b6 100644 --- a/ct.yaml +++ b/ct.yaml @@ -2,7 +2,7 @@ chart-dirs: - "tools/deployment" chart-yaml-schema: "chart_schema.yaml" check-version-increment: true -lint-conf: ".yamllint.yml" +lint-conf: ".yamllint.yaml" validate-chart-schema: true validate-maintainers: true validate-yaml: true diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index ee9fc207ae..6925e5e174 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "clp.labels" . | nindent 4 }} data: - clp-config.yml: | + clp-config.yaml: | data_directory: "/var/data" database: auto_commit: false diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index 66cbace10b..d3b6c13a9e 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -48,13 +48,13 @@ spec: value: "/opt/clp/lib/python3/site-packages" volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true command: [ "python3", "-u", "-m", "clp_py_utils.create-db-tables", - "--config", "/etc/clp-config.yml", + "--config", "/etc/clp-config.yaml", "--storage-engine", {{ .Values.clpConfig.package.storage_engine | quote }} ] volumes: From a155ba35942717dffaf8c7bad991fce522aadf91 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 02:22:26 -0500 Subject: [PATCH 054/155] Update configuration file extensions from .yml to .yaml for consistency --- .../package-helm/templates/api-server-deployment.yaml | 6 +++--- .../templates/compression-scheduler-deployment.yaml | 6 +++--- .../templates/compression-worker-deployment.yaml | 6 +++--- .../templates/garbage-collector-deployment.yaml | 6 +++--- .../package-helm/templates/mcp-server-deployment.yaml | 6 +++--- .../package-helm/templates/query-scheduler-deployment.yaml | 6 +++--- .../package-helm/templates/query-worker-deployment.yaml | 6 +++--- .../package-helm/templates/reducer-deployment.yaml | 6 +++--- 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 8bcf8f55f0..d0a91e097a 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -59,8 +59,8 @@ spec: containerPort: 3001 volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "api-server-logs" mountPath: "/var/log/api_server" @@ -68,7 +68,7 @@ spec: "/opt/clp/bin/api_server", "--host", "0.0.0.0", "--port", "3001", - "--config", "/etc/clp-config.yml" + "--config", "/etc/clp-config.yaml" ] livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index a0af4204ca..26fd11e590 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -68,8 +68,8 @@ spec: value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "compression-scheduler-logs" mountPath: "/var/log/compression_scheduler" @@ -81,7 +81,7 @@ spec: command: [ "python3", "-u", "-m", "job_orchestration.scheduler.compress.compression_scheduler", - "--config", "/etc/clp-config.yml" + "--config", "/etc/clp-config.yaml" ] volumes: - name: "config" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 6aee42b056..3baef2dbe3 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -43,7 +43,7 @@ spec: - name: "BROKER_URL" value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_CONFIG_PATH" - value: "/etc/clp-config.yml" + value: "/etc/clp-config.yaml" - name: "CLP_HOME" value: "/opt/clp" - name: "CLP_LOGGING_LEVEL" @@ -58,8 +58,8 @@ spec: value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "compression-worker-logs" mountPath: "/var/log/compression_worker" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index e47bc8b8bd..72f7d1cbf3 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -60,8 +60,8 @@ spec: value: "/opt/clp/lib/python3/site-packages" volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "garbage-collector-logs" mountPath: "/var/log/garbage_collector" @@ -72,7 +72,7 @@ spec: command: [ "python3", "-u", "-m", "job_orchestration.garbage_collector.garbage_collector", - "--config", "/etc/clp-config.yml" + "--config", "/etc/clp-config.yaml" ] volumes: - name: "config" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index e651e7fb63..89b23451d1 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -62,8 +62,8 @@ spec: containerPort: 8000 volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "mcp-server-logs" mountPath: "/var/log/mcp_server" @@ -72,7 +72,7 @@ spec: "-m", "clp_mcp_server.clp_mcp_server", "--host", "0.0.0.0", "--port", "8000", - "--config-path", "/etc/clp-config.yml" + "--config-path", "/etc/clp-config.yaml" ] livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 46a6299234..2ac200a42e 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -73,15 +73,15 @@ spec: containerPort: 7000 volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "query-scheduler-logs" mountPath: "/var/log/query_scheduler" command: [ "python3", "-u", "-m", "job_orchestration.scheduler.query.query_scheduler", - "--config", "/etc/clp-config.yml" + "--config", "/etc/clp-config.yaml" ] volumes: - name: "config" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 6f913e2a86..d91d970c4d 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -43,7 +43,7 @@ spec: - name: "BROKER_URL" value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_CONFIG_PATH" - value: "/etc/clp-config.yml" + value: "/etc/clp-config.yaml" - name: "CLP_HOME" value: "/opt/clp" - name: "CLP_LOGGING_LEVEL" @@ -58,8 +58,8 @@ spec: value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "query-worker-logs" mountPath: "/var/log/query_worker" diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index edd1693849..fbfeb6ac26 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -52,15 +52,15 @@ spec: value: "/opt/clp/lib/python3/site-packages" volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "reducer-logs" mountPath: "/var/log/reducer" command: [ "python3", "-u", "-m", "job_orchestration.reducer.reducer", - "--config", "/etc/clp-config.yml", + "--config", "/etc/clp-config.yaml", "--concurrency", "{{ .Values.workerConcurrency }}", "--upsert-interval", "{{ .Values.clpConfig.reducer.upsert_interval }}" ] From be9afb3fb1a8a6b9615520f6ed14a32a522d7b0f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 02:30:41 -0500 Subject: [PATCH 055/155] Refactor liveness and readiness probes across multiple deployment files for consistency --- .../templates/api-server-deployment.yaml | 10 ++++---- .../templates/database-statefulset.yaml | 3 +-- .../templates/mcp-server-deployment.yaml | 10 ++++---- .../templates/queue-statefulset.yaml | 11 ++++----- .../templates/redis-statefulset.yaml | 24 +++++++++---------- .../templates/results-cache-statefulset.yaml | 11 ++++----- .../templates/webui-deployment.yaml | 9 ++++--- 7 files changed, 35 insertions(+), 43 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index d0a91e097a..310b74b54c 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -70,16 +70,14 @@ spec: "--port", "3001", "--config", "/etc/clp-config.yaml" ] - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} - httpGet: - path: "/health" - port: 3001 readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} - httpGet: + httpGet: &api-server-health-check path: "/health" port: 3001 + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + httpGet: *api-server-health-check volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 06347e5062..a8859e9f24 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -75,8 +75,7 @@ spec: --password=${MYSQL_PASSWORD} livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} - exec: - <<: *database-health-check + exec: *database-health-check volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 89b23451d1..46793e3e16 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -74,16 +74,14 @@ spec: "--port", "8000", "--config-path", "/etc/clp-config.yaml" ] - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} - httpGet: - path: "/health" - port: 8000 readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} - httpGet: + httpGet: &mcp-server-health-check path: "/health" port: 8000 + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + httpGet: *mcp-server-health-check volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index bea559736a..94e8ea9bdc 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -45,14 +45,13 @@ spec: volumeMounts: - name: "queue-logs" mountPath: "/var/log/rabbitmq" - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} - exec: &queue-health-check - command: ["rabbitmq-diagnostics", "check_running"] readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} - exec: - <<: *queue-health-check + exec: &queue-health-check + command: [ "rabbitmq-diagnostics", "check_running" ] + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: *queue-health-check persistentVolumeClaimRetentionPolicy: whenDeleted: "Delete" volumeClaimTemplates: diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 245601fb68..4e6efd906b 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -48,20 +48,20 @@ spec: mountPath: "/etc/redis/redis.conf" subPath: "redis.conf" readOnly: true - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} - exec: &redis-health-check - command: - - "redis-cli" - - "-h" - - "127.0.0.1" - - "-p" - - "6379" - - "-a" - - "$(REDIS_PASSWORD)" - - "PING" readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: &redis-health-check + command: + - "sh" + - "-c" + - >- + redis-cli + -h 127.0.0.1 + -p 6379 + -a "${REDIS_PASSWORD}" + ping + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: <<: *redis-health-check volumes: diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 441603be4e..aea5c5b84a 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -43,8 +43,8 @@ spec: mountPath: "/etc/mongo/mongod.conf" subPath: "mongod.conf" readOnly: true - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &results-cache-health-check command: - "mongosh" @@ -52,10 +52,9 @@ spec: - "--quiet" - "--eval" - "db.runCommand('ping').ok" - readinessProbe: - {{- include "clp.readinessProbeTimings" . | nindent 12 }} - exec: - <<: *results-cache-health-check + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: *results-cache-health-check volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 6ece053202..2263dca7d0 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -78,14 +78,13 @@ spec: "/opt/clp/bin/node-22", "/opt/clp/var/www/webui/server/dist/src/main.js" ] - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} - tcpSocket: - port: 4000 readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} - tcpSocket: + tcpSocket: &webui-health-check port: 4000 + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + tcpSocket: *webui-health-check volumes: - name: "client-settings" configMap: From 387ad2fdaa36ab31b543ee05cc22c056a3ebb977 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 02:35:47 -0500 Subject: [PATCH 056/155] Add workerConcurrency setting to values.yaml --- tools/deployment/package-helm/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index c4c8d239a9..43d440cb7f 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -18,6 +18,8 @@ image: pullPolicy: "IfNotPresent" tag: "nightly" +workerConcurrency: 8 + clpConfig: package: storage_engine: "clp-s" From d2a43572f413a61dab27cac9c2b861bdfc3ebd0e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 02:38:21 -0500 Subject: [PATCH 057/155] Set replicas to 1 for compression and query workers --- .../package-helm/templates/compression-worker-deployment.yaml | 2 +- .../package-helm/templates/query-worker-deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 3baef2dbe3..b3540eb310 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-worker" spec: - replicas: {{ .Values.workerConcurrency }} + replicas: 1 selector: matchLabels: {{- include "clp.selectorLabels" . | nindent 6 }} diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index d91d970c4d..d96e79c62f 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-worker" spec: - replicas: {{ .Values.workerConcurrency }} + replicas: 1 selector: matchLabels: {{- include "clp.selectorLabels" . | nindent 6 }} From d4f3c81217d47ca7514587aa06e00e48de14ce73 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 20:27:16 -0500 Subject: [PATCH 058/155] upgrade yamale to 6.1.0 - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index 49300e3f8c..1f6f2d86fc 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -8,5 +8,5 @@ black>=24.4.2 ruff>=0.4.4 # YAML -yamale>=6.0.0 +yamale>=6.1.0 yamllint>=1.35.1 From 1515d0dbb713ab72e3bf73b8c2bc8c367adf633b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 20:39:38 -0500 Subject: [PATCH 059/155] Add chart-testing SHA256 checksum for Linux ARM64 architecture in toolchains.yaml --- taskfiles/toolchains.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 1df385ef74..2472f3f43d 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -32,6 +32,8 @@ tasks: FILE_SHA256: >- {{- if and (eq OS "linux") (eq ARCH "amd64") -}} d16f0583616885423826241164ce1f6589c6fe5332fa74f374ebd2bd3cb3fe1f + {{- else if and (eq OS "linux") (eq ARCH "arm64") -}} + 7b035bbbd0768fba33983e81c15445e33e9c8281d329100852a342ecfd0b16b7 {{- else if and (eq OS "darwin") (eq ARCH "arm64") -}} db10dbbb42b110c7a5da5a3202908f32ad2ca6ad600d423426dc7886d09aad07 {{- else -}} From dd35d68879a7cd81123013b983cd00ed280c665c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 20:41:12 -0500 Subject: [PATCH 060/155] Remove redundant chart schema and lint validation check settings because they are by default true --- ct.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ct.yaml b/ct.yaml index 95717fd2b6..b3232c1b6e 100644 --- a/ct.yaml +++ b/ct.yaml @@ -1,8 +1,4 @@ chart-dirs: - "tools/deployment" chart-yaml-schema: "chart_schema.yaml" -check-version-increment: true lint-conf: ".yamllint.yaml" -validate-chart-schema: true -validate-maintainers: true -validate-yaml: true From 5296f312a9ec527c07c8b426d7e06c2c8b384efb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 07:14:52 -0500 Subject: [PATCH 061/155] Remove the unneeded `chmod u+x` from the helm env source script - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- taskfiles/toolchains.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 1a232a9dce..270943f6d7 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -53,7 +53,6 @@ tasks: #!/bin/sh export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" EOF - chmod +x "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" rust: # We use a label to uniquely identify this task and its checksum under `.task/checksums/`. From 2c412518a4d3ce8984631273053641df754b841f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 07:24:24 -0500 Subject: [PATCH 062/155] use chart-yaml-schema from the downloaded toolchain --- chart_schema.yaml | 38 -------------------------------------- ct.yaml | 2 +- taskfiles/toolchains.yaml | 2 +- 3 files changed, 2 insertions(+), 40 deletions(-) delete mode 100644 chart_schema.yaml diff --git a/chart_schema.yaml b/chart_schema.yaml deleted file mode 100644 index 4f1d69b73b..0000000000 --- a/chart_schema.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# yamllint disable rule:document-start -name: "str()" -home: "str(required=False)" -version: "str()" -apiVersion: "str()" -appVersion: "any(str(), num(), required=False)" -description: "str(required=False)" -keywords: "list(str(), required=False)" -sources: "list(str(), required=False)" -maintainers: "list(include('maintainer'), required=False)" -dependencies: "list(include('dependency'), required=False)" -icon: "str(required=False)" -engine: "str(required=False)" -condition: "str(required=False)" -tags: "str(required=False)" -deprecated: "bool(required=False)" -kubeVersion: "str(required=False)" -annotations: "map(str(), str(), required=False)" -type: "str(required=False)" ---- -maintainer: - name: "str()" - email: "str(required=False)" - url: "str(required=False)" ---- -dependency: - name: "str()" - version: "str()" - repository: "str(required=False)" - condition: "str(required=False)" - tags: "list(str(), required=False)" - enabled: "bool(required=False)" - import-values: "list(any(str(), include('import-value')), required=False)" - alias: "str(required=False)" ---- -import-value: - child: "str()" - parent: "str()" diff --git a/ct.yaml b/ct.yaml index b3232c1b6e..d50cc5563f 100644 --- a/ct.yaml +++ b/ct.yaml @@ -1,4 +1,4 @@ chart-dirs: - "tools/deployment" -chart-yaml-schema: "chart_schema.yaml" +chart-yaml-schema: "build/toolchains/helm/etc/chart_schema.yaml" lint-conf: ".yamllint.yaml" diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 270943f6d7..7e97ac55c1 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -39,7 +39,7 @@ tasks: {{- else -}} unsupported {{- end }} - INCLUDE_PATTERNS: ["ct"] + INCLUDE_PATTERNS: ["ct", "etc/chart_schema.yaml"] NUM_COMPONENTS_TO_STRIP: "0" - |- export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" From 093113611604144d0b0c91d0ca737c6a5e7a28ad Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 07:44:32 -0500 Subject: [PATCH 063/155] Remove chart_schema.yaml from lint.yaml --- taskfiles/lint.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 6187f0bdca..9654a83216 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -119,7 +119,6 @@ tasks: "components/core/config" \ "components/core/tools/" \ "components/package-template/src/etc" \ - "chart_schema.yaml" \ "ct.yaml" \ "docs" \ "taskfile.yaml" \ From dd70f5c2a819fbb0c9676e7aa5d1fb80f937abfa Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:36:45 -0500 Subject: [PATCH 064/155] Assign `nodeRole` for database persistent volumes to target control-plane nodes. Update helper template to include `nodeRole` field with fallback logic. --- tools/deployment/package-helm/templates/_helpers.tpl | 8 +++++++- .../package-helm/templates/database-data-pv.yaml | 1 + .../package-helm/templates/database-logs-pv.yaml | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 3920d4eb18..4d44c0472d 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -87,6 +87,9 @@ Parameters (dict): capacity: Storage capacity accessModes: Access modes (list) hostPath: Absolute path on host + nodeRole: Node role for affinity + Targets nodes with label "node-role.kubernetes.io/" + Always falls back to "node-role.kubernetes.io/control-plane" */}} {{- define "clp.createLocalPv" -}} apiVersion: "v1" @@ -108,7 +111,10 @@ spec: required: nodeSelectorTerms: - matchExpressions: - - key: "kubernetes.io/hostname" + - key: {{ printf "node-role.kubernetes.io/%s" .nodeRole | quote }} + operator: "Exists" + - matchExpressions: + - key: "node-role.kubernetes.io/control-plane" operator: "Exists" {{- end }} diff --git a/tools/deployment/package-helm/templates/database-data-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml index df4d58c1c2..98beaaf866 100644 --- a/tools/deployment/package-helm/templates/database-data-pv.yaml +++ b/tools/deployment/package-helm/templates/database-data-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-database-data" (include "clp.fullname" .)) "component" "database" + "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/database" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml index 0fdc598ee0..27ad28d197 100644 --- a/tools/deployment/package-helm/templates/database-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-database-logs" (include "clp.fullname" .)) "component" "database" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/database" .Values.clpConfig.logs_directory) From ea062688bf7edda734ba8b33f1e7d6213148373b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:37:33 -0500 Subject: [PATCH 065/155] Improve waiting prompts - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- .../deployment/package-helm/templates/db-table-creator-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index d3b6c13a9e..e397494f8d 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -26,7 +26,7 @@ spec: - "-c" - >- until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo waiting for database; + do echo Waiting for database to be ready...; sleep 2; done containers: From ba5d4dfe6eef2f288d6cec568335019555828874 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:37:50 -0500 Subject: [PATCH 066/155] update helm chart description - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index f156ec8a54..26b6fe1201 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v2" name: "clp" version: "0.1.0" -description: "A Helm chart for CLP (Compressed Log Processor) package deployment" +description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.6.1-dev" home: "https://github.com/y-scope/clp" From 8b44dd63e52b4dbb43c03de3f9914288d7ddabcf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:38:08 -0500 Subject: [PATCH 067/155] update docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/deployment/package-helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 1ed16596db..6b9ba6fd06 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -1,7 +1,7 @@ nameOverride: "" fullnameOverride: "" -# Whether to allow sbin script accesses to services on the node host. +# Whether to allow scripts in `sbin` access to services on the host. allowSbinScripts: true securityContext: From 9ff1611293617a3f8f1773c8573a3a31ea3ea24a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:38:36 -0500 Subject: [PATCH 068/155] Rename `allowSbinScripts` to `allowHostAccessForSbinScripts` for clearer intent in Helm values and templates. --- tools/deployment/package-helm/templates/database-service.yaml | 4 ++-- tools/deployment/package-helm/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package-helm/templates/database-service.yaml b/tools/deployment/package-helm/templates/database-service.yaml index fe6f2f823e..f940b65f86 100644 --- a/tools/deployment/package-helm/templates/database-service.yaml +++ b/tools/deployment/package-helm/templates/database-service.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "database" spec: - {{- if .Values.allowSbinScripts }} + {{- if .Values.allowHostAccessForSbinScripts }} type: "NodePort" {{- else }} clusterIP: "None" @@ -17,6 +17,6 @@ spec: ports: - port: 3306 targetPort: "database" - {{- if .Values.allowSbinScripts }} + {{- if .Values.allowHostAccessForSbinScripts }} nodePort: {{ .Values.clpConfig.database.port }} {{- end }} diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 6b9ba6fd06..20de735f43 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -2,7 +2,7 @@ nameOverride: "" fullnameOverride: "" # Whether to allow scripts in `sbin` access to services on the host. -allowSbinScripts: true +allowHostAccessForSbinScripts: true securityContext: firstParty: From 014097123366402c351b5f55b8bf772c19a49dc5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:48:32 -0500 Subject: [PATCH 069/155] update test script shebang, update TODO comment, set eu pipefail - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/deployment/package-helm/test.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 1111d8d985..c1c45cb9f7 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -1,6 +1,10 @@ -#!/bin/bash +#!/usr/bin/env bash -# TODO: to be deleted / migrated to integration tests +# TODO: Migrate into integration test + +set -o errexit +set -o nounset +set -o pipefail kind delete cluster --name clp-test rm -rf /tmp/clp From 44bd1af1ee16e3bf801323ae0580e1ffd88afc3b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:45:04 -0500 Subject: [PATCH 070/155] Make image pull policy configurable for db-table-creator job in Helm templates. --- .../deployment/package-helm/templates/db-table-creator-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index e397494f8d..a3ece1f23e 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -32,7 +32,7 @@ spec: containers: - name: "db-table-creator" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "CLP_DB_USER" valueFrom: From 72013ed41c04dd86a00369ba9e8447e15b1c5ff3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:57:36 -0500 Subject: [PATCH 071/155] Add JSDoc-style comments for helper templates in Helm chart --- .../package-helm/templates/_helpers.tpl | 64 ++++++++++++------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 4d44c0472d..19bbb3e238 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -1,14 +1,18 @@ {{/* -Expand the name of the chart. +Expands the name of the chart. + +@return {string} The chart name (truncated to 63 characters) */}} {{- define "clp.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} {{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. +Creates a default fully qualified app name. We truncate at 63 chars because +some Kubernetes name fields are limited to this (by the DNS naming spec). If +release name contains chart name it will be used as a full name. + +@return {string} The fully qualified app name (truncated to 63 characters) */}} {{- define "clp.fullname" -}} {{- if .Values.fullnameOverride }} @@ -24,14 +28,18 @@ If release name contains chart name it will be used as a full name. {{- end }} {{/* -Create chart name and version as used by the chart label. +Creates chart name and version as used by the chart label. + +@return {string} Chart name and version (truncated to 63 characters) */}} {{- define "clp.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* -Common labels +Creates common labels for all resources. + +@return {string} YAML-formatted common labels */}} {{- define "clp.labels" -}} helm.sh/chart: {{ include "clp.chart" . }} @@ -43,7 +51,9 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} {{/* -Selector labels +Creates selector labels for matching resources. + +@return {string} YAML-formatted selector labels */}} {{- define "clp.selectorLabels" -}} app.kubernetes.io/name: {{ include "clp.name" . }} @@ -51,7 +61,9 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* -Image reference for CLP Package +Creates image reference for CLP Package. + +@return {string} Full image reference (repository:tag) */}} {{- define "clp.image.ref" -}} {{- $tag := .Values.image.clpPackage.tag | default .Chart.AppVersion }} @@ -60,6 +72,8 @@ Image reference for CLP Package {{/* Creates timings for readiness probes (faster checks for quicker startup). + +@return {string} YAML-formatted readiness probe timing configuration */}} {{- define "clp.readinessProbeTimings" -}} initialDelaySeconds: 6 @@ -70,6 +84,8 @@ failureThreshold: 10 {{/* Creates timings for liveness probes. + +@return {string} YAML-formatted liveness probe timing configuration */}} {{- define "clp.livenessProbeTimings" -}} initialDelaySeconds: 60 @@ -80,16 +96,17 @@ failureThreshold: 3 {{/* Creates a local PersistentVolume. -Parameters (dict): - root: Root template context - name: PV name (full) - component: Component label - capacity: Storage capacity - accessModes: Access modes (list) - hostPath: Absolute path on host - nodeRole: Node role for affinity - Targets nodes with label "node-role.kubernetes.io/" - Always falls back to "node-role.kubernetes.io/control-plane" + +@param {object} root Root template context +@param {string} name PV name +@param {string} component Component label +@param {string} capacity Storage capacity +@param {string[]} accessModes Access modes (list) +@param {string} hostPath Absolute path on host +@param {string} nodeRole Node role for affinity. Targets nodes with label + "node-role.kubernetes.io/". Always falls back to + "node-role.kubernetes.io/control-plane" +@return {string} YAML-formatted PersistentVolume resource */}} {{- define "clp.createLocalPv" -}} apiVersion: "v1" @@ -120,11 +137,12 @@ spec: {{/* Creates a PersistentVolumeClaim. -Parameters (dict): - root: Root template context - component: Component label (used in name, labels, and selectors) - capacity: Storage capacity - accessModes: Access modes (list) + +@param {object} root Root template context +@param {string} component Component label +@param {string} capacity Storage capacity +@param {string[]} accessModes +@return {string} YAML-formatted PersistentVolumeClaim resource */}} {{- define "clp.createPvc" -}} apiVersion: "v1" From 1452795e79e427c83fe288d177ba6d888d7dd7d9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 27 Nov 2025 06:41:10 -0500 Subject: [PATCH 072/155] improve docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/deployment/package-helm/templates/_helpers.tpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 19bbb3e238..51c0258c89 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -8,9 +8,9 @@ Expands the name of the chart. {{- end }} {{/* -Creates a default fully qualified app name. We truncate at 63 chars because -some Kubernetes name fields are limited to this (by the DNS naming spec). If -release name contains chart name it will be used as a full name. +Creates a default fully qualified app name. We truncate at 63 chars because some Kubernetes name +fields are limited to this (by the DNS naming spec). If release name contains chart name it will be +used as a full name. @return {string} The fully qualified app name (truncated to 63 characters) */}} @@ -61,7 +61,7 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* -Creates image reference for CLP Package. +Creates image reference for the CLP Package. @return {string} Full image reference (repository:tag) */}} From 480c14b594a7ed7071d0ae4aa73ffae688c3fec5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 27 Nov 2025 04:56:26 -0500 Subject: [PATCH 073/155] Remove unused `createPvc` helper definition from Helm templates --- .../package-helm/templates/_helpers.tpl | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 51c0258c89..92b6eae5d9 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -134,32 +134,3 @@ spec: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" {{- end }} - -{{/* -Creates a PersistentVolumeClaim. - -@param {object} root Root template context -@param {string} component Component label -@param {string} capacity Storage capacity -@param {string[]} accessModes -@return {string} YAML-formatted PersistentVolumeClaim resource -*/}} -{{- define "clp.createPvc" -}} -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" .root }}-{{ .component }} - labels: - {{- include "clp.labels" .root | nindent 4 }} - app.kubernetes.io/component: {{ .component | quote }} -spec: - accessModes: {{ .accessModes }} - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" .root | nindent 6 }} - app.kubernetes.io/component: {{ .component | quote }} - resources: - requests: - storage: {{ .capacity }} -{{- end }} From 41428566dbd3156cdf82d9c0efcca536b2df3494 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 27 Nov 2025 06:09:21 -0500 Subject: [PATCH 074/155] Update helm toolchain output directory to use `G_HELM_TOOLCHAIN_DIR` --- taskfiles/toolchains.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 7e97ac55c1..a40d1bd5d5 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -26,7 +26,7 @@ tasks: - "mkdir -p '{{.G_HELM_TOOLCHAIN_DIR}}'" - task: "remote:download-and-extract-tar" vars: - OUTPUT_DIR: "{{.G_BUILD_DIR}}/toolchains/helm" + OUTPUT_DIR: "{{.G_HELM_TOOLCHAIN_DIR}}" URL: "https://github.com/helm/chart-testing/releases/download/v{{.CHART_TESTING_VERSION}}\ /chart-testing_{{.CHART_TESTING_VERSION}}_{{OS}}_{{ARCH}}.tar.gz" FILE_SHA256: >- From 036222fd98b7a049410a02edbecb7028fea75709 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 27 Nov 2025 06:20:04 -0500 Subject: [PATCH 075/155] Remove `ct.yaml` and inline configuration in `lint.yaml` for clarity --- ct.yaml | 4 ---- taskfiles/lint.yaml | 6 ++++-- 2 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 ct.yaml diff --git a/ct.yaml b/ct.yaml deleted file mode 100644 index d50cc5563f..0000000000 --- a/ct.yaml +++ /dev/null @@ -1,4 +0,0 @@ -chart-dirs: - - "tools/deployment" -chart-yaml-schema: "build/toolchains/helm/etc/chart_schema.yaml" -lint-conf: ".yamllint.yaml" diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 9654a83216..fd8f58d790 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -67,7 +67,10 @@ tasks: - |- . "{{.G_LINT_VENV_DIR}}/bin/activate" . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" - ct lint + ct lint \ + --chart-dirs="tools/deployment" \ + --chart-yaml-schema="{{.G_HELM_TOOLCHAIN_DIR}}/etc/chart_schema.yaml" \ + --lint-conf=".yamllint.yaml" check-js: sources: &js_source_files @@ -119,7 +122,6 @@ tasks: "components/core/config" \ "components/core/tools/" \ "components/package-template/src/etc" \ - "ct.yaml" \ "docs" \ "taskfile.yaml" \ "taskfiles" \ From bfa2ea351d4eaceed1954d8dcb49a4bd3f60fbb7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 14:44:01 -0500 Subject: [PATCH 076/155] update values.yaml with the latest clp-config.yaml --- tools/deployment/package-helm/values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 00996b5dbb..d1e53c69cd 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -34,10 +34,12 @@ clpConfig: # this directory will be ignored. directory: "/" + bundled: ["database", "queue", "redis", "results_cache"] + database: type: "mariadb" # "mariadb" or "mysql" host: "localhost" - port: 30306 + port: 3306 name: "clp-db" compression_scheduler: From 93fcbfe841c40b611b405e1ccf6789f588b6baf9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 14:50:16 -0500 Subject: [PATCH 077/155] Assign `nodeRole` to persistent volumes for control-plane targeting. --- tools/deployment/package-helm/templates/api-server-logs-pv.yaml | 1 + tools/deployment/package-helm/templates/archives-pv.yaml | 1 + .../package-helm/templates/compression-scheduler-logs-pv.yaml | 1 + .../package-helm/templates/compression-worker-logs-pv.yaml | 1 + .../templates/compression-worker-staged-archives-pv.yaml | 1 + .../package-helm/templates/garbage-collector-logs-pv.yaml | 1 + tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml | 1 + .../package-helm/templates/query-scheduler-logs-pv.yaml | 1 + .../deployment/package-helm/templates/query-worker-logs-pv.yaml | 1 + .../package-helm/templates/query-worker-staged-streams-pv.yaml | 1 + tools/deployment/package-helm/templates/queue-logs-pv.yaml | 1 + tools/deployment/package-helm/templates/redis-data-pv.yaml | 1 + tools/deployment/package-helm/templates/redis-logs-pv.yaml | 1 + tools/deployment/package-helm/templates/reducer-logs-pv.yaml | 1 + .../deployment/package-helm/templates/results-cache-data-pv.yaml | 1 + .../deployment/package-helm/templates/results-cache-logs-pv.yaml | 1 + tools/deployment/package-helm/templates/streams-pv.yaml | 1 + tools/deployment/package-helm/templates/tmp-pv.yaml | 1 + tools/deployment/package-helm/templates/user-logs-pv.yaml | 1 + 19 files changed, 19 insertions(+) diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index 977be57d6f..74d372325c 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-api-server-logs" (include "clp.fullname" .)) "component" "api-server-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/api-server" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/archives-pv.yaml b/tools/deployment/package-helm/templates/archives-pv.yaml index 11072b5c9d..d1a043d64f 100644 --- a/tools/deployment/package-helm/templates/archives-pv.yaml +++ b/tools/deployment/package-helm/templates/archives-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-archives" (include "clp.fullname" .)) "component" "archives" + "nodeRole" "control-plane" "capacity" "50Gi" "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.archive_output.storage.directory diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 475a1ecc6b..48a46f4f8a 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-compression-scheduler-logs" (include "clp.fullname" .)) "component" "compression-scheduler-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression-scheduler" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index 7d8aeae9ba..3c451126d2 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-compression-worker-logs" (include "clp.fullname" .)) "component" "compression-worker-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression-worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index 6a52ad3e9c..11dcbd1d8a 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-staged-archives" (include "clp.fullname" .)) "component" "staged-archives" + "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") "hostPath" "/tmp/clp/var/data/staged-archives" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index 8eaa2c9ea2..75f3a6a3e5 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-garbage-collector-logs" (include "clp.fullname" .)) "component" "garbage-collector-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/garbage-collector" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index da16ab4e3f..01bbf18ab7 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-mcp-server-logs" (include "clp.fullname" .)) "component" "mcp-server-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/mcp-server" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index 8ec5ed13d4..d7b62604c4 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-query-scheduler-logs" (include "clp.fullname" .)) "component" "query-scheduler-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query-scheduler" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index 007505daae..e92ee8b03f 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-query-worker-logs" (include "clp.fullname" .)) "component" "query-worker-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query-worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index 993cb3bca1..95a641be53 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-staged-streams" (include "clp.fullname" .)) "component" "staged-streams" + "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") "hostPath" "/tmp/clp/var/data/staged-streams" diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml index b6ff44e134..cbc4fa3008 100644 --- a/tools/deployment/package-helm/templates/queue-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-queue-logs" (include "clp.fullname" .)) "component" "queue" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/queue" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml index bf170d9f8c..96fa611225 100644 --- a/tools/deployment/package-helm/templates/redis-data-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-redis-data" (include "clp.fullname" .)) "component" "redis" + "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/redis" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml index 3a1fc890d3..b3308cbfc9 100644 --- a/tools/deployment/package-helm/templates/redis-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-redis-logs" (include "clp.fullname" .)) "component" "redis" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/redis" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index 3fd67c21b5..12c67a778f 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-reducer-logs" (include "clp.fullname" .)) "component" "reducer-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/reducer" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml index 52502fa827..8c67722348 100644 --- a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-results-cache-data" (include "clp.fullname" .)) "component" "results-cache" + "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/results-cache" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index 61c8efa80b..2050d3a033 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-results-cache-logs" (include "clp.fullname" .)) "component" "results-cache" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/results-cache" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/streams-pv.yaml b/tools/deployment/package-helm/templates/streams-pv.yaml index f998090001..df211cf22f 100644 --- a/tools/deployment/package-helm/templates/streams-pv.yaml +++ b/tools/deployment/package-helm/templates/streams-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-streams" (include "clp.fullname" .)) "component" "streams" + "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.stream_output.storage.directory diff --git a/tools/deployment/package-helm/templates/tmp-pv.yaml b/tools/deployment/package-helm/templates/tmp-pv.yaml index 43bdd2ca8c..1add0502bd 100644 --- a/tools/deployment/package-helm/templates/tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/tmp-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-tmp" (include "clp.fullname" .)) "component" "tmp" + "nodeRole" "control-plane" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.tmp_directory diff --git a/tools/deployment/package-helm/templates/user-logs-pv.yaml b/tools/deployment/package-helm/templates/user-logs-pv.yaml index b90b351a48..9c43c56e00 100644 --- a/tools/deployment/package-helm/templates/user-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/user-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-user-logs" (include "clp.fullname" .)) "component" "user-logs" + "nodeRole" "control-plane" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") "hostPath" (printf "%s/user" .Values.clpConfig.logs_directory) From 7a1fff495d655d8ce60cf0bbe3cec8b261fc3c01 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 15:00:43 -0500 Subject: [PATCH 078/155] add back helper to create PersistentVolumeClaim templates. --- .../package-helm/templates/_helpers.tpl | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 92b6eae5d9..1a6946fbe2 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -101,7 +101,7 @@ Creates a local PersistentVolume. @param {string} name PV name @param {string} component Component label @param {string} capacity Storage capacity -@param {string[]} accessModes Access modes (list) +@param {string[]} accessModes Access modes @param {string} hostPath Absolute path on host @param {string} nodeRole Node role for affinity. Targets nodes with label "node-role.kubernetes.io/". Always falls back to @@ -134,3 +134,32 @@ spec: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" {{- end }} + +{{/* +Creates a PersistentVolumeClaim for the given component. + +@param {object} root Root template context +@param {string} component Component label +@param {string} capacity Storage capacity +@param {string[]} accessModes Access modes +@return {string} YAML-formatted PersistentVolumeClaim resource +*/}} +{{- define "clp.createPvc" -}} +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" .root }}-{{ .component }} + labels: + {{- include "clp.labels" .root | nindent 4 }} + app.kubernetes.io/component: {{ .component | quote }} +spec: + accessModes: {{ .accessModes }} + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" .root | nindent 6 }} + app.kubernetes.io/component: {{ .component | quote }} + resources: + requests: + storage: {{ .capacity }} +{{- end }} From ba6abb40d4217022507ece163ee0a751185d4cad Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 15:01:28 -0500 Subject: [PATCH 079/155] fix(helm): Rename values flag for sbin script access in results cache service. --- .../package-helm/templates/results-cache-service.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/results-cache-service.yaml b/tools/deployment/package-helm/templates/results-cache-service.yaml index 2c8e540fd8..0a6a10f4bb 100644 --- a/tools/deployment/package-helm/templates/results-cache-service.yaml +++ b/tools/deployment/package-helm/templates/results-cache-service.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "results-cache" spec: - {{- if .Values.allowSbinScripts }} + {{- if .Values.allowHostAccessForSbinScripts }} type: "NodePort" {{- else }} clusterIP: "None" @@ -17,6 +17,6 @@ spec: ports: - port: 27017 targetPort: "mongodb" - {{- if .Values.allowSbinScripts }} + {{- if .Values.allowHostAccessForSbinScripts }} nodePort: {{ .Values.clpConfig.results_cache.port }} {{- end }} From b25b2433811539ea842b16ad65aa84caed00ae52 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 15:51:08 -0500 Subject: [PATCH 080/155] refactor(helm): Update initContainer command syntax for better readability. --- .../templates/api-server-deployment.yaml | 24 ++++++++----- .../compression-scheduler-deployment.yaml | 36 ++++++++++++------- .../compression-worker-deployment.yaml | 24 ++++++++----- .../garbage-collector-deployment.yaml | 24 ++++++++----- .../templates/mcp-server-deployment.yaml | 24 ++++++++----- .../templates/query-scheduler-deployment.yaml | 36 ++++++++++++------- .../templates/query-worker-deployment.yaml | 24 ++++++++----- .../templates/reducer-deployment.yaml | 24 ++++++++----- .../results-cache-indices-creator-job.yaml | 12 ++++--- .../templates/webui-deployment.yaml | 24 ++++++++----- 10 files changed, 168 insertions(+), 84 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 310b74b54c..7b8a6870fc 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -25,16 +25,24 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo Waiting for database to be ready...; + sleep 2; + done - name: "wait-for-results-cache" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done containers: - name: "api-server" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 26fd11e590..ba21533462 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -25,22 +25,34 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo Waiting for database to be ready...; + sleep 2; + done - name: "wait-for-queue" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-queue 5672; + do echo Waiting for queue to be ready...; + sleep 2; + done - name: "wait-for-redis" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-redis 6379; + do echo Waiting for redis to be ready...; + sleep 2; + done containers: - name: "compression-scheduler" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index b3540eb310..e703c8f251 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -25,16 +25,24 @@ spec: initContainers: - name: "wait-for-queue" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-queue 5672; + do echo Waiting for queue to be ready...; + sleep 2; + done - name: "wait-for-redis" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-redis 6379; + do echo Waiting for redis to be ready...; + sleep 2; + done containers: - name: "compression-worker" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 72f7d1cbf3..8afba4cfe5 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -25,16 +25,24 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo Waiting for database to be ready...; + sleep 2; + done - name: "wait-for-results-cache" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done containers: - name: "garbage-collector" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 46793e3e16..8426f38c00 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -26,16 +26,24 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo Waiting for database to be ready...; + sleep 2; + done - name: "wait-for-results-cache" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done containers: - name: "mcp-server" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 2ac200a42e..6a4a6a6559 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -27,22 +27,34 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo Waiting for database to be ready...; + sleep 2; + done - name: "wait-for-queue" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-queue 5672; + do echo Waiting for queue to be ready...; + sleep 2; + done - name: "wait-for-redis" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-redis 6379; + do echo Waiting for redis to be ready...; + sleep 2; + done containers: - name: "query-scheduler" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index d96e79c62f..a9301c0ae1 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -25,16 +25,24 @@ spec: initContainers: - name: "wait-for-queue" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-queue 5672; + do echo Waiting for queue to be ready...; + sleep 2; + done - name: "wait-for-redis" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-redis 6379; + do echo Waiting for redis to be ready...; + sleep 2; + done containers: - name: "query-worker" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index fbfeb6ac26..3df0db7023 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -27,16 +27,24 @@ spec: initContainers: - name: "wait-for-query-scheduler" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-query-scheduler 7000; do echo waiting for query-scheduler; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-query-scheduler 7000; + do echo Waiting for query-scheduler to be ready...; + sleep 2; + done - name: "wait-for-results-cache" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done containers: - name: "reducer" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index 5ff701fa3a..7cd1075d6c 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -21,10 +21,14 @@ spec: initContainers: - name: "wait-for-results-cache" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done containers: - name: "results-cache-indices-creator" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 2263dca7d0..9ba41c4c00 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -25,16 +25,24 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo Waiting for database to be ready...; + sleep 2; + done - name: "wait-for-results-cache" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done containers: - name: "webui" image: "{{ include "clp.image.ref" . }}" From 4ef25b513321376309b168875c83de5faed1163e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 15:54:19 -0500 Subject: [PATCH 081/155] refactor(helm): Use configurable image pull policy for deployments. --- .../package-helm/templates/api-server-deployment.yaml | 2 +- .../templates/compression-scheduler-deployment.yaml | 2 +- .../package-helm/templates/compression-worker-deployment.yaml | 2 +- .../package-helm/templates/garbage-collector-deployment.yaml | 2 +- .../package-helm/templates/mcp-server-deployment.yaml | 2 +- .../package-helm/templates/query-scheduler-deployment.yaml | 2 +- .../package-helm/templates/query-worker-deployment.yaml | 2 +- tools/deployment/package-helm/templates/reducer-deployment.yaml | 2 +- .../templates/results-cache-indices-creator-job.yaml | 2 +- tools/deployment/package-helm/templates/webui-deployment.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 7b8a6870fc..7f8e5fb1a6 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -46,7 +46,7 @@ spec: containers: - name: "api-server" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "CLP_DB_USER" valueFrom: diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index ba21533462..1de5e362ea 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -56,7 +56,7 @@ spec: containers: - name: "compression-scheduler" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "BROKER_URL" value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index e703c8f251..92db721a17 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -46,7 +46,7 @@ spec: containers: - name: "compression-worker" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "BROKER_URL" value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 8afba4cfe5..a4fcaab0cf 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -46,7 +46,7 @@ spec: containers: - name: "garbage-collector" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "CLP_DB_USER" valueFrom: diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 8426f38c00..beb0c1e1b9 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -47,7 +47,7 @@ spec: containers: - name: "mcp-server" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "CLP_DB_USER" valueFrom: diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 6a4a6a6559..b6c6a2990e 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -58,7 +58,7 @@ spec: containers: - name: "query-scheduler" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "BROKER_URL" value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index a9301c0ae1..ba395d4d29 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -46,7 +46,7 @@ spec: containers: - name: "query-worker" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "BROKER_URL" value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index 3df0db7023..edde1c8fcd 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -48,7 +48,7 @@ spec: containers: - name: "reducer" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "CLP_HOME" value: "/opt/clp" diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index 7cd1075d6c..51c2b822db 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -32,7 +32,7 @@ spec: containers: - name: "results-cache-indices-creator" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 9ba41c4c00..06fed0405d 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -46,7 +46,7 @@ spec: containers: - name: "webui" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "CLP_DB_USER" valueFrom: From 0ca31739c7f47c66232d3dde8dba60fb41a6b30f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 17:59:59 -0500 Subject: [PATCH 082/155] revert db port --- tools/deployment/package-helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index d1e53c69cd..e45b76b2a8 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -39,7 +39,7 @@ clpConfig: database: type: "mariadb" # "mariadb" or "mysql" host: "localhost" - port: 3306 + port: 30306 name: "clp-db" compression_scheduler: From 399373febb6e6dee38f5c656e735708ca2b03c38 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 02:42:58 -0500 Subject: [PATCH 083/155] refactor(helm): Remove redundant host field from configuration values. --- tools/deployment/package-helm/templates/configmap.yaml | 4 ++-- tools/deployment/package-helm/values.yaml | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 71db36d4d5..e3ab999c33 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -8,7 +8,7 @@ data: clp-config.yaml: | api_server: default_max_num_query_results: {{ .Values.clpConfig.api_server.default_max_num_query_results }} - host: {{ .Values.clpConfig.api_server.host | quote }} + host: "localhost" port: 3001 query_job_polling: initial_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.initial_backoff_ms }} @@ -87,7 +87,7 @@ data: target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size }} tmp_directory: "/var/tmp" webui: - host: {{ .Values.clpConfig.webui.host | quote }} + host: "localhost" port: 4000 rate_limit: {{ .Values.clpConfig.webui.rate_limit }} results_metadata_collection_name: {{ .Values.clpConfig.webui.results_metadata_collection_name | quote }} diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index e45b76b2a8..f49fae664e 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -38,7 +38,6 @@ clpConfig: database: type: "mariadb" # "mariadb" or "mysql" - host: "localhost" port: 30306 name: "clp-db" @@ -47,30 +46,25 @@ clpConfig: logging_level: "INFO" query_scheduler: - host: "localhost" port: 7000 jobs_poll_delay: 0.1 # seconds num_archives_to_search_per_sub_job: 16 logging_level: "INFO" queue: - host: "localhost" port: 5672 redis: - host: "localhost" port: 6379 query_backend_database: 0 compression_backend_database: 1 reducer: - host: "localhost" base_port: 14009 logging_level: "INFO" upsert_interval: 100 # milliseconds results_cache: - host: "localhost" port: 30017 db_name: "clp-query-results" stream_collection_name: "stream-files" @@ -85,13 +79,11 @@ clpConfig: logging_level: "INFO" webui: - host: "localhost" port: 30400 results_metadata_collection_name: "results-metadata" rate_limit: 1000 mcp_server: null - # host: "localhost" # port: 30800 # logging_level: "INFO" @@ -145,7 +137,6 @@ clpConfig: # API server config api_server: - host: "localhost" port: 30301 default_max_num_query_results: 1000 query_job_polling: @@ -154,7 +145,6 @@ clpConfig: # Presto client config presto: null - # host: "localhost" # port: 30889 # Location where other data (besides archives) are stored. It will be created if From 0bb706bfc3559797863b4a62ab24034ff5c62301 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 02:44:07 -0500 Subject: [PATCH 084/155] remove empty line --- tools/deployment/package-helm/templates/configmap.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index e3ab999c33..6ad7cf6b5b 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -51,7 +51,6 @@ data: package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} - presto: {{ .Values.clpConfig.presto }} query_scheduler: host: {{ include "clp.fullname" . }}-query-scheduler From 7f90a853738fbfbf69ce66742d6c25c165bd04a6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 02:48:49 -0500 Subject: [PATCH 085/155] Update image tag to 'main' and pull policy to 'Always' in values.yaml. --- tools/deployment/package-helm/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index f49fae664e..532c55d828 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -15,8 +15,8 @@ securityContext: image: clpPackage: repository: "ghcr.io/y-scope/clp/clp-package" - pullPolicy: "IfNotPresent" - tag: "nightly" + pullPolicy: "Always" + tag: "main" workerConcurrency: 8 From 39fdcd604fa6b99febc767ec095cfc3088f566e4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 03:16:58 -0500 Subject: [PATCH 086/155] feat(deployment): Add redis, queue (RabbitMQ), and results cache (MongoDB) as `StatefulSet` into the CLP Package Helm chart. --- .../package-helm/templates/_helpers.tpl | 2 +- .../package-helm/templates/configmap.yaml | 30 ++++++ .../templates/database-statefulset.yaml | 3 +- .../package-helm/templates/queue-logs-pv.yaml | 9 ++ .../package-helm/templates/queue-secret.yaml | 11 +++ .../package-helm/templates/queue-service.yaml | 15 +++ .../templates/queue-statefulset.yaml | 69 +++++++++++++ .../package-helm/templates/redis-data-pv.yaml | 9 ++ .../package-helm/templates/redis-logs-pv.yaml | 9 ++ .../package-helm/templates/redis-secret.yaml | 10 ++ .../package-helm/templates/redis-service.yaml | 15 +++ .../templates/redis-statefulset.yaml | 97 +++++++++++++++++++ .../templates/results-cache-data-pv.yaml | 9 ++ .../results-cache-indices-creator-job.yaml | 44 +++++++++ .../templates/results-cache-logs-pv.yaml | 9 ++ .../templates/results-cache-service.yaml | 22 +++++ .../templates/results-cache-statefulset.yaml | 88 +++++++++++++++++ tools/deployment/package-helm/test.sh | 5 +- tools/deployment/package-helm/values.yaml | 23 ++++- 19 files changed, 472 insertions(+), 7 deletions(-) create mode 100644 tools/deployment/package-helm/templates/queue-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/queue-secret.yaml create mode 100644 tools/deployment/package-helm/templates/queue-service.yaml create mode 100644 tools/deployment/package-helm/templates/queue-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/redis-data-pv.yaml create mode 100644 tools/deployment/package-helm/templates/redis-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/redis-secret.yaml create mode 100644 tools/deployment/package-helm/templates/redis-service.yaml create mode 100644 tools/deployment/package-helm/templates/redis-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-data-pv.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-service.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-statefulset.yaml diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 92b6eae5d9..6aa0e2b873 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -101,7 +101,7 @@ Creates a local PersistentVolume. @param {string} name PV name @param {string} component Component label @param {string} capacity Storage capacity -@param {string[]} accessModes Access modes (list) +@param {string[]} accessModes Access modes @param {string} hostPath Absolute path on host @param {string} nodeRole Node role for affinity. Targets nodes with label "node-role.kubernetes.io/". Always falls back to diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 6925e5e174..f58670e2bd 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -19,6 +19,20 @@ data: package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} + queue: + host: {{ include "clp.fullname" . }}-queue + port: 5672 + redis: + compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} + host: {{ include "clp.fullname" . }}-redis + port: 6379 + query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} + results_cache: + db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} + host: {{ include "clp.fullname" . }}-results-cache + port: 27017 + retention_period: {{ .Values.clpConfig.results_cache.retention_period }} + stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} mysql-logging.cnf: | [mysqld] @@ -31,3 +45,19 @@ data: # https://dev.mysql.com/doc/refman/8.0/en/error-log-destination-configuration.html # Send error logs to file log_error=/var/log/mysql/mysql-error.log + + redis.conf: | + # For full reference of configuration options see + # https://raw.githubusercontent.com/redis/redis/7.2/redis.conf + + databases 2 + logfile "/var/log/redis/redis.log" + + mongod.conf: | + replication: + replSetName: "rs0" + systemLog: + destination: file + path: /var/log/mongodb/mongod.log + logAppend: true + timeStampFormat: iso8601-local \ No newline at end of file diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 06347e5062..a8859e9f24 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -75,8 +75,7 @@ spec: --password=${MYSQL_PASSWORD} livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} - exec: - <<: *database-health-check + exec: *database-health-check volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml new file mode 100644 index 0000000000..cbc4fa3008 --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-queue-logs" (include "clp.fullname" .)) + "component" "queue" + "nodeRole" "control-plane" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/queue" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/queue-secret.yaml b/tools/deployment/package-helm/templates/queue-secret.yaml new file mode 100644 index 0000000000..704e443b64 --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp.fullname" . }}-queue + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +type: "Opaque" +stringData: + username: {{ .Values.credentials.queue.username | quote }} + password: {{ .Values.credentials.queue.password | quote }} diff --git a/tools/deployment/package-helm/templates/queue-service.yaml b/tools/deployment/package-helm/templates/queue-service.yaml new file mode 100644 index 0000000000..2971725fdb --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-queue + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "queue" + ports: + - port: 5672 + targetPort: "amqp" diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml new file mode 100644 index 0000000000..94e8ea9bdc --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -0,0 +1,69 @@ +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp.fullname" . }}-queue + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +spec: + serviceName: {{ include "clp.fullname" . }}-queue + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "queue" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "queue" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "queue" + image: "rabbitmq:3.9.8" + imagePullPolicy: "Always" + env: + - name: "RABBITMQ_DEFAULT_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-queue + key: "username" + - name: "RABBITMQ_DEFAULT_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-queue + key: "password" + - name: "RABBITMQ_LOGS" + value: "/var/log/rabbitmq/rabbitmq.log" + ports: + - name: "amqp" + containerPort: 5672 + volumeMounts: + - name: "queue-logs" + mountPath: "/var/log/rabbitmq" + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: &queue-health-check + command: [ "rabbitmq-diagnostics", "check_running" ] + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: *queue-health-check + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" + volumeClaimTemplates: + - metadata: + name: "queue-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "queue" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml new file mode 100644 index 0000000000..96fa611225 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-redis-data" (include "clp.fullname" .)) + "component" "redis" + "nodeRole" "control-plane" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/redis" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml new file mode 100644 index 0000000000..b3308cbfc9 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-redis-logs" (include "clp.fullname" .)) + "component" "redis" + "nodeRole" "control-plane" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/redis" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/redis-secret.yaml b/tools/deployment/package-helm/templates/redis-secret.yaml new file mode 100644 index 0000000000..244ccb5291 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp.fullname" . }}-redis + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +type: "Opaque" +stringData: + password: {{ .Values.credentials.redis.password | quote }} diff --git a/tools/deployment/package-helm/templates/redis-service.yaml b/tools/deployment/package-helm/templates/redis-service.yaml new file mode 100644 index 0000000000..5cd8e61386 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-redis + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "redis" + ports: + - port: 6379 + targetPort: "redis" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml new file mode 100644 index 0000000000..4e6efd906b --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -0,0 +1,97 @@ +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp.fullname" . }}-redis + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +spec: + serviceName: {{ include "clp.fullname" . }}-redis + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "redis" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "redis" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "redis" + image: "redis:7.2.4" + imagePullPolicy: "Always" + command: [ + "redis-server", "/etc/redis/redis.conf", + "--requirepass", "$(REDIS_PASSWORD)" + ] + env: + - name: "REDIS_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-redis + key: "password" + ports: + - name: "redis" + containerPort: 6379 + volumeMounts: + - name: "redis-data" + mountPath: "/data" + - name: "redis-logs" + mountPath: "/var/log/redis" + - name: "config" + mountPath: "/etc/redis/redis.conf" + subPath: "redis.conf" + readOnly: true + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: &redis-health-check + command: + - "sh" + - "-c" + - >- + redis-cli + -h 127.0.0.1 + -p 6379 + -a "${REDIS_PASSWORD}" + ping + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: + <<: *redis-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" + volumeClaimTemplates: + - metadata: + name: "redis-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "redis" + resources: + requests: + storage: "10Gi" + - metadata: + name: "redis-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "redis" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml new file mode 100644 index 0000000000..8c67722348 --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-results-cache-data" (include "clp.fullname" .)) + "component" "results-cache" + "nodeRole" "control-plane" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/results-cache" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml new file mode 100644 index 0000000000..51c2b822db --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -0,0 +1,44 @@ +apiVersion: "batch/v1" +kind: "Job" +metadata: + name: {{ include "clp.fullname" . }}-results-cache-indices-creator + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache-indices-creator" +spec: + ttlSecondsAfterFinished: 300 + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "results-cache-indices-creator" + spec: + restartPolicy: "OnFailure" + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-results-cache" + image: "busybox:latest" + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done + containers: + - name: "results-cache-indices-creator" + image: "{{ include "clp.image.ref" . }}" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" + env: + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + command: [ + "python3", "-u", + "-m", "clp_py_utils.initialize-results-cache", + "--uri", "mongodb://{{ include "clp.fullname" . }}-results-cache:27017/{{ .Values.clpConfig.results_cache.db_name }}", + "--stream-collection", {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} + ] diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml new file mode 100644 index 0000000000..2050d3a033 --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-results-cache-logs" (include "clp.fullname" .)) + "component" "results-cache" + "nodeRole" "control-plane" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/results-cache" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/results-cache-service.yaml b/tools/deployment/package-helm/templates/results-cache-service.yaml new file mode 100644 index 0000000000..0a6a10f4bb --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-service.yaml @@ -0,0 +1,22 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-results-cache + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + {{- if .Values.allowHostAccessForSbinScripts }} + type: "NodePort" + {{- else }} + clusterIP: "None" + {{- end }} + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" + ports: + - port: 27017 + targetPort: "mongodb" + {{- if .Values.allowHostAccessForSbinScripts }} + nodePort: {{ .Values.clpConfig.results_cache.port }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml new file mode 100644 index 0000000000..aea5c5b84a --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -0,0 +1,88 @@ +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp.fullname" . }}-results-cache + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + serviceName: {{ include "clp.fullname" . }}-results-cache + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "results-cache" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "results-cache" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "results-cache" + image: "mongo:7.0.1" + imagePullPolicy: "Always" + command: [ + "mongod", + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0" + ] + ports: + - name: "mongodb" + containerPort: 27017 + volumeMounts: + - name: "results-cache-data" + mountPath: "/data/db" + - name: "results-cache-logs" + mountPath: "/var/log/mongodb" + - name: "config" + mountPath: "/etc/mongo/mongod.conf" + subPath: "mongod.conf" + readOnly: true + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: &results-cache-health-check + command: + - "mongosh" + - "127.0.0.1:27017/test" + - "--quiet" + - "--eval" + - "db.runCommand('ping').ok" + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: *results-cache-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" + volumeClaimTemplates: + - metadata: + name: "results-cache-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "results-cache" + resources: + requests: + storage: "20Gi" + - metadata: + name: "results-cache-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "results-cache" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index c1c45cb9f7..f2f9664170 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -8,7 +8,7 @@ set -o pipefail kind delete cluster --name clp-test rm -rf /tmp/clp -mkdir -p /tmp/clp/var/{data,log}/database +mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache} cat < Date: Sat, 29 Nov 2025 03:25:10 -0500 Subject: [PATCH 087/155] chore(deployment): Bump Helm chart version to 0.1.1. --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 26b6fe1201..6ad8ec4576 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.1.0" +version: "0.1.1" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.6.1-dev" From 4fb6dd41ea57af4de2b9bfcff4448b5db26d855b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 03:33:03 -0500 Subject: [PATCH 088/155] add empty newline at EOF --- tools/deployment/package-helm/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index f58670e2bd..f5ae83a580 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -60,4 +60,4 @@ data: destination: file path: /var/log/mongodb/mongod.log logAppend: true - timeStampFormat: iso8601-local \ No newline at end of file + timeStampFormat: iso8601-local From 1f4df0c21098d2446ff24bcd7ab718cac46cd653 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 03:34:06 -0500 Subject: [PATCH 089/155] docs(deployment): Reorder parameters in Helm helper function documentation. --- tools/deployment/package-helm/templates/_helpers.tpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 6aa0e2b873..6577538785 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -100,12 +100,12 @@ Creates a local PersistentVolume. @param {object} root Root template context @param {string} name PV name @param {string} component Component label -@param {string} capacity Storage capacity -@param {string[]} accessModes Access modes -@param {string} hostPath Absolute path on host @param {string} nodeRole Node role for affinity. Targets nodes with label "node-role.kubernetes.io/". Always falls back to "node-role.kubernetes.io/control-plane" +@param {string} capacity Storage capacity +@param {string[]} accessModes Access modes +@param {string} hostPath Absolute path on host @return {string} YAML-formatted PersistentVolume resource */}} {{- define "clp.createLocalPv" -}} From 683f03b70b251c0e71445f898a157d9e2ed7bced Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 03:44:04 -0500 Subject: [PATCH 090/155] fix(helm): Simplify syntax for redis livenessProbe exec configuration. --- tools/deployment/package-helm/templates/redis-statefulset.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 4e6efd906b..ae5db90e3d 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -62,8 +62,7 @@ spec: ping livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} - exec: - <<: *redis-health-check + exec: *redis-health-check volumes: - name: "config" configMap: From 33ea69a2669d6f5ac77f4259c5b2bcc3af68f25c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 14:03:18 -0500 Subject: [PATCH 091/155] Update port names for services and deployments. --- .../package-helm/templates/api-server-deployment.yaml | 2 +- tools/deployment/package-helm/templates/api-server-service.yaml | 2 +- .../package-helm/templates/mcp-server-deployment.yaml | 2 +- tools/deployment/package-helm/templates/mcp-server-service.yaml | 2 +- tools/deployment/package-helm/templates/webui-deployment.yaml | 2 +- tools/deployment/package-helm/templates/webui-service.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 7f8e5fb1a6..7a97719902 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -63,7 +63,7 @@ spec: - name: "RUST_LOG" value: "DEBUG" ports: - - name: "http" + - name: "api-server" containerPort: 3001 volumeMounts: - name: "config" diff --git a/tools/deployment/package-helm/templates/api-server-service.yaml b/tools/deployment/package-helm/templates/api-server-service.yaml index 7c24c0f6e5..500b5cd6b6 100644 --- a/tools/deployment/package-helm/templates/api-server-service.yaml +++ b/tools/deployment/package-helm/templates/api-server-service.yaml @@ -12,5 +12,5 @@ spec: app.kubernetes.io/component: "api-server" ports: - port: 3001 - targetPort: "http" + targetPort: "api-server" nodePort: {{ .Values.clpConfig.api_server.port }} diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index beb0c1e1b9..bfe5273352 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -66,7 +66,7 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" ports: - - name: "http" + - name: "mcp-server" containerPort: 8000 volumeMounts: - name: "config" diff --git a/tools/deployment/package-helm/templates/mcp-server-service.yaml b/tools/deployment/package-helm/templates/mcp-server-service.yaml index ee2ae9fbe9..9acbe48f9e 100644 --- a/tools/deployment/package-helm/templates/mcp-server-service.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-service.yaml @@ -13,6 +13,6 @@ spec: app.kubernetes.io/component: "mcp-server" ports: - port: 8000 - targetPort: "http" + targetPort: "mcp-server" nodePort: {{ .Values.clpConfig.mcp_server.port }} {{- end }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 06fed0405d..4fc2f28e7d 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -69,7 +69,7 @@ spec: - name: "RATE_LIMIT" value: "1000" ports: - - name: "http" + - name: "webui" containerPort: 4000 volumeMounts: - name: "client-settings" diff --git a/tools/deployment/package-helm/templates/webui-service.yaml b/tools/deployment/package-helm/templates/webui-service.yaml index ebe1ea2074..aa1e885680 100644 --- a/tools/deployment/package-helm/templates/webui-service.yaml +++ b/tools/deployment/package-helm/templates/webui-service.yaml @@ -12,5 +12,5 @@ spec: app.kubernetes.io/component: "webui" ports: - port: 4000 - targetPort: "http" + targetPort: "webui" nodePort: {{ .Values.clpConfig.webui.port }} From 7a52e6c5125744520e3846f372374bced3687e54 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 14:03:59 -0500 Subject: [PATCH 092/155] chore(helm): Bump chart version to 0.1.2-dev.3 --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 9469a9b469..0a9c85ff60 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.1.2-dev.2" +version: "0.1.2-dev.3" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.7.1-dev" From abb1cf87b35b5f6b4679a7ae6aa4d7cfa8eb2c45 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 14:41:15 -0500 Subject: [PATCH 093/155] feat(helm): Add `max_concurrent_tasks_per_job` parameter to compression_scheduler configuration --- tools/deployment/package-helm/templates/configmap.yaml | 1 + tools/deployment/package-helm/values.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 6ad7cf6b5b..7cc8861103 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -27,6 +27,7 @@ data: compression_scheduler: jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} + max_concurrent_tasks_per_job: {{ .Values.clpConfig.compression_scheduler.max_concurrent_tasks_per_job }} compression_worker: logging_level: {{ .Values.clpConfig.compression_worker.logging_level | quote }} data_directory: "/var/data" diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index d0311cf337..6ee186ff1e 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -44,6 +44,7 @@ clpConfig: compression_scheduler: jobs_poll_delay: 0.1 # seconds logging_level: "INFO" + max_concurrent_tasks_per_job: 0 # A value of 0 disables the limit query_scheduler: port: 7000 From 3bf2d940781c0c0fbaf8cd97ec97a58ecd6a3482 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 14:50:36 -0500 Subject: [PATCH 094/155] Update configmap webui settings --- .../deployment/package-helm/templates/configmap.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 7cc8861103..8a732a2a8b 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -124,10 +124,11 @@ data: { "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, + "LogsInputType": {{ .Values.clpConfig.logs_input.type | quote }}, "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, - "SqlDbClpArchivesTableName": "", + "SqlDbClpArchivesTableName": "clp_archives", "SqlDbClpDatasetsTableName": "clp_datasets", - "SqlDbClpFilesTableName": "", + "SqlDbClpFilesTableName": "clp_files", "SqlDbClpTablePrefix": "clp_", "SqlDbCompressionJobsTableName": "compression_jobs" } @@ -138,6 +139,7 @@ data: "SqlDbPort": 3306, "SqlDbName": {{ .Values.clpConfig.database.name | quote }}, "SqlDbQueryJobsTableName": "query_jobs", + "SqlDbCompressionJobsTableName": "compression_jobs", "MongoDbHost": "{{ include "clp.fullname" . }}-results-cache", "MongoDbPort": 27017, "MongoDbName": {{ .Values.clpConfig.results_cache.db_name | quote }}, @@ -150,7 +152,13 @@ data: "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, + "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level }}, + "ArchiveOutputTargetArchiveSize": {{ .Values.clpConfig.archive_output.target_archive_size }}, + "ArchiveOutputTargetDictionariesSize": {{ .Values.clpConfig.archive_output.target_dictionaries_size }}, + "ArchiveOutputTargetEncodedFileSize": {{ .Values.clpConfig.archive_output.target_encoded_file_size }}, + "ArchiveOutputTargetSegmentSize": {{ .Values.clpConfig.archive_output.target_segment_size }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, + "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, "PrestoHost": null, "PrestoPort": null } From 0195c05172e364fd7d4075aff2a1fe89212599f2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 14:54:45 -0500 Subject: [PATCH 095/155] Make `StreamTargetUncompressedSize` configurable via values file --- tools/deployment/package-helm/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 8a732a2a8b..7d3a5f8463 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -148,7 +148,7 @@ data: "ClientDir": "/opt/clp/var/www/webui/client", "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer", "StreamFilesDir": "/var/data/streams", - "StreamTargetUncompressedSize": 134217728, + "StreamTargetUncompressedSize": {{ .Values.clpConfig.stream_output.target_uncompressed_size }}, "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, From af2a04b585a7ead2c5d8d8e7ce3aee35e62b0911 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 15:06:08 -0500 Subject: [PATCH 096/155] avoid encoding u64 as float --- .../package-helm/templates/configmap.yaml | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 7d3a5f8463..e204cfdb97 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -7,7 +7,8 @@ metadata: data: clp-config.yaml: | api_server: - default_max_num_query_results: {{ .Values.clpConfig.api_server.default_max_num_query_results }} + default_max_num_query_results: {{ + .Values.clpConfig.api_server.default_max_num_query_results }} host: "localhost" port: 3001 query_job_polling: @@ -19,15 +20,19 @@ data: storage: directory: "/var/data/archives" type: "fs" - target_archive_size: {{ .Values.clpConfig.archive_output.target_archive_size }} - target_dictionaries_size: {{ .Values.clpConfig.archive_output.target_dictionaries_size }} - target_encoded_file_size: {{ .Values.clpConfig.archive_output.target_encoded_file_size }} - target_segment_size: {{ .Values.clpConfig.archive_output.target_segment_size }} + target_archive_size: {{ .Values.clpConfig.archive_output.target_archive_size | int }} + target_dictionaries_size: {{ .Values.clpConfig.archive_output.target_dictionaries_size + | int }} + target_encoded_file_size: {{ .Values.clpConfig.archive_output.target_encoded_file_size + | int }} + target_segment_size: {{ .Values.clpConfig.archive_output.target_segment_size + | int }} aws_config_directory: {{ .Values.clpConfig.aws_config_directory }} compression_scheduler: jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} - max_concurrent_tasks_per_job: {{ .Values.clpConfig.compression_scheduler.max_concurrent_tasks_per_job }} + max_concurrent_tasks_per_job: {{ + .Values.clpConfig.compression_scheduler.max_concurrent_tasks_per_job }} compression_worker: logging_level: {{ .Values.clpConfig.compression_worker.logging_level | quote }} data_directory: "/var/data" @@ -57,7 +62,8 @@ data: host: {{ include "clp.fullname" . }}-query-scheduler jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} - num_archives_to_search_per_sub_job: {{ .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job }} + num_archives_to_search_per_sub_job: {{ + .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job }} port: 7000 query_worker: logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }} @@ -84,13 +90,14 @@ data: storage: directory: "/var/data/streams" type: "fs" - target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size }} + target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }} tmp_directory: "/var/tmp" webui: host: "localhost" port: 4000 rate_limit: {{ .Values.clpConfig.webui.rate_limit }} - results_metadata_collection_name: {{ .Values.clpConfig.webui.results_metadata_collection_name | quote }} + results_metadata_collection_name: {{ .Values.clpConfig.webui.results_metadata_collection_name + | quote }} mysql-logging.cnf: | [mysqld] @@ -125,7 +132,8 @@ data: "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, "LogsInputType": {{ .Values.clpConfig.logs_input.type | quote }}, - "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, + "MongoDbSearchResultsMetadataCollectionName": + {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, "SqlDbClpArchivesTableName": "clp_archives", "SqlDbClpDatasetsTableName": "clp_datasets", "SqlDbClpFilesTableName": "clp_files", @@ -143,20 +151,26 @@ data: "MongoDbHost": "{{ include "clp.fullname" . }}-results-cache", "MongoDbPort": 27017, "MongoDbName": {{ .Values.clpConfig.results_cache.db_name | quote }}, - "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, + "MongoDbSearchResultsMetadataCollectionName": + {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, "MongoDbStreamFilesCollectionName": "stream-files", "ClientDir": "/opt/clp/var/www/webui/client", "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer", "StreamFilesDir": "/var/data/streams", - "StreamTargetUncompressedSize": {{ .Values.clpConfig.stream_output.target_uncompressed_size }}, + "StreamTargetUncompressedSize": + {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }}, "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level }}, - "ArchiveOutputTargetArchiveSize": {{ .Values.clpConfig.archive_output.target_archive_size }}, - "ArchiveOutputTargetDictionariesSize": {{ .Values.clpConfig.archive_output.target_dictionaries_size }}, - "ArchiveOutputTargetEncodedFileSize": {{ .Values.clpConfig.archive_output.target_encoded_file_size }}, - "ArchiveOutputTargetSegmentSize": {{ .Values.clpConfig.archive_output.target_segment_size }}, + "ArchiveOutputTargetArchiveSize": + {{ .Values.clpConfig.archive_output.target_archive_size | int }}, + "ArchiveOutputTargetDictionariesSize": + {{ .Values.clpConfig.archive_output.target_dictionaries_size | int }}, + "ArchiveOutputTargetEncodedFileSize": + {{ .Values.clpConfig.archive_output.target_encoded_file_size | int }}, + "ArchiveOutputTargetSegmentSize": + {{ .Values.clpConfig.archive_output.target_segment_size | int }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, "PrestoHost": null, From 0a8707df1fa3c6040ad9c02f02050d31f3cf4623 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 15:19:09 -0500 Subject: [PATCH 097/155] feat(helm): Add conditional storage_engine configuration to configmap --- tools/deployment/package-helm/templates/configmap.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index e204cfdb97..8830c2098a 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -6,6 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} data: clp-config.yaml: | + {{- if eq .Values.clpConfig.package.storage_engine "clp-s" }} api_server: default_max_num_query_results: {{ .Values.clpConfig.api_server.default_max_num_query_results }} @@ -14,6 +15,9 @@ data: query_job_polling: initial_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.initial_backoff_ms }} max_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.max_backoff_ms }} + {{- else }} + api_server: null + {{- end }} archive_output: compression_level: {{ .Values.clpConfig.archive_output.compression_level }} retention_period: {{ .Values.clpConfig.archive_output.retention_period }} From eeb0fc290346454cac5479dfcf40a44415eced94 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 15:41:11 -0500 Subject: [PATCH 098/155] fix(helm): Update reducer base_port to a fixed value in configmap and reducer-service --- tools/deployment/package-helm/templates/configmap.yaml | 2 +- tools/deployment/package-helm/templates/reducer-service.yaml | 4 ++-- tools/deployment/package-helm/values.yaml | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 8830c2098a..d595a36acd 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -80,7 +80,7 @@ data: port: 6379 query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} reducer: - base_port: {{ .Values.clpConfig.reducer.base_port }} + base_port: 14009 host: {{ include "clp.fullname" . }}-reducer logging_level: {{ .Values.clpConfig.reducer.logging_level | quote }} upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval }} diff --git a/tools/deployment/package-helm/templates/reducer-service.yaml b/tools/deployment/package-helm/templates/reducer-service.yaml index 5c3fc6d7b1..8f0ee60d8d 100644 --- a/tools/deployment/package-helm/templates/reducer-service.yaml +++ b/tools/deployment/package-helm/templates/reducer-service.yaml @@ -13,6 +13,6 @@ spec: ports: {{- range $i := until (int .Values.workerConcurrency) }} - name: "reducer-{{ $i }}" - port: {{ add $.Values.clpConfig.reducer.base_port $i }} - targetPort: {{ add $.Values.clpConfig.reducer.base_port $i }} + port: {{ add 14009 $i }} + targetPort: {{ add 14009 $i }} {{- end }} diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 6ee186ff1e..8172f14d65 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -57,7 +57,6 @@ clpConfig: compression_backend_database: 1 reducer: - base_port: 14009 logging_level: "INFO" upsert_interval: 100 # milliseconds From 696ea80ee543a249808cea95549c1b205e18c234 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 15:45:36 -0500 Subject: [PATCH 099/155] fix(helm): Replace hardcoded sample file copy with dynamic download and extraction in test script --- tools/deployment/package-helm/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 7c8a9bdf7c..d5a1689a12 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -13,6 +13,8 @@ mkdir -p /tmp/clp/var/data/{archives,staged-archives,staged-streams,streams} mkdir -p /tmp/clp/var/log/user mkdir -p /tmp/clp/var/tmp mkdir -p /tmp/clp/samples +wget -O - https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1 \ + | tar xz -C /tmp/clp/samples & cat < Date: Mon, 8 Dec 2025 16:10:08 -0500 Subject: [PATCH 100/155] remove compression-scheduler-service.yaml --- .../templates/compression-scheduler-service.yaml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 tools/deployment/package-helm/templates/compression-scheduler-service.yaml diff --git a/tools/deployment/package-helm/templates/compression-scheduler-service.yaml b/tools/deployment/package-helm/templates/compression-scheduler-service.yaml deleted file mode 100644 index 605daf9055..0000000000 --- a/tools/deployment/package-helm/templates/compression-scheduler-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-compression-scheduler - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "compression-scheduler" -spec: - clusterIP: "None" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "compression-scheduler" - ports: - - port: 7000 - targetPort: 7000 From 7bd5558b5d915766ebe96f13c7cd993be4e8c752 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 16:10:21 -0500 Subject: [PATCH 101/155] Download and extract sample datasets in the background during test setup --- tools/deployment/package-helm/test.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index d5a1689a12..17ec02454f 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -13,8 +13,11 @@ mkdir -p /tmp/clp/var/data/{archives,staged-archives,staged-streams,streams} mkdir -p /tmp/clp/var/log/user mkdir -p /tmp/clp/var/tmp mkdir -p /tmp/clp/samples + +# Download sample datasets in the background wget -O - https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1 \ | tar xz -C /tmp/clp/samples & +SAMPLE_DOWNLOAD_PID=$! cat < Date: Mon, 8 Dec 2025 16:21:40 -0500 Subject: [PATCH 102/155] feat(test): Add hostPath mount for /home in test setup --- tools/deployment/package-helm/test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 17ec02454f..5e9b9fbf67 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -25,6 +25,8 @@ cat < Date: Tue, 9 Dec 2025 03:48:28 -0500 Subject: [PATCH 103/155] fxi: make Web UI rate limit configurable in Helm chart --- tools/deployment/package-helm/templates/webui-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 4fc2f28e7d..97ad133ac6 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -67,7 +67,7 @@ spec: - name: "PORT" value: "4000" - name: "RATE_LIMIT" - value: "1000" + value: {{ .Values.clpConfig.webui.rate_limit | quote }} ports: - name: "webui" containerPort: 4000 From 7d6cf4fe2277c77e081ddaffffed582748a4a321 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Dec 2025 08:19:42 -0500 Subject: [PATCH 104/155] feat(helm): Add support for root database credentials in Helm templates. --- .../package-helm/templates/database-secret.yaml | 2 ++ .../package-helm/templates/database-statefulset.yaml | 2 +- .../package-helm/templates/db-table-creator-job.yaml | 10 ++++++++++ tools/deployment/package-helm/values.yaml | 2 ++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/database-secret.yaml b/tools/deployment/package-helm/templates/database-secret.yaml index 64235ff4a7..5439e66bd6 100644 --- a/tools/deployment/package-helm/templates/database-secret.yaml +++ b/tools/deployment/package-helm/templates/database-secret.yaml @@ -9,3 +9,5 @@ type: "Opaque" stringData: username: {{ .Values.credentials.database.username | quote }} password: {{ .Values.credentials.database.password | quote }} + root_username: {{ .Values.credentials.database.root_username | quote }} + root_password: {{ .Values.credentials.database.root_password | quote }} diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 39fefe81d2..47801e20f7 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -48,7 +48,7 @@ spec: valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "root_password" ports: - name: "database" containerPort: 3306 diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index a3ece1f23e..bc600db429 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -44,6 +44,16 @@ spec: secretKeyRef: name: {{ include "clp.fullname" . }}-database key: "password" + - name: "CLP_DB_ROOT_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "root_username" + - name: "CLP_DB_ROOT_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "root_password" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 8172f14d65..20803830fd 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -167,6 +167,8 @@ credentials: database: username: "clp-user" password: "pass" + root_username: "root" + root_password: "root-pass" queue: username: "clp-user" From 8f695d95705242f1a9c95f926cc0f1b4eebecdb2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 11 Dec 2025 20:22:26 -0500 Subject: [PATCH 105/155] feat(helm): Refactor init containers to use shared `clp.waitFor` helper and add job-watcher role with service account --- .../package-helm/templates/_helpers.tpl | 25 ++++++++++++++ .../templates/api-server-deployment.yaml | 23 ++----------- .../compression-scheduler-deployment.yaml | 34 +++---------------- .../compression-worker-deployment.yaml | 23 ++----------- .../templates/db-table-creator-job.yaml | 12 ++----- .../garbage-collector-deployment.yaml | 23 ++----------- .../templates/job-watcher-role.yaml | 13 +++++++ .../templates/job-watcher-rolebinding.yaml | 14 ++++++++ .../templates/job-watcher-serviceaccount.yaml | 6 ++++ .../templates/mcp-server-deployment.yaml | 23 ++----------- .../templates/query-scheduler-deployment.yaml | 34 +++---------------- .../templates/query-worker-deployment.yaml | 23 ++----------- .../templates/reducer-deployment.yaml | 23 ++----------- .../results-cache-indices-creator-job.yaml | 12 ++----- .../templates/webui-deployment.yaml | 23 ++----------- 15 files changed, 91 insertions(+), 220 deletions(-) create mode 100644 tools/deployment/package-helm/templates/job-watcher-role.yaml create mode 100644 tools/deployment/package-helm/templates/job-watcher-rolebinding.yaml create mode 100644 tools/deployment/package-helm/templates/job-watcher-serviceaccount.yaml diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index c0f7b4de12..d4de98d0e1 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -163,3 +163,28 @@ spec: requests: storage: {{ .capacity }} {{- end }} + +{{/* +Creates an initContainer that waits for a Kubernetes resource to be ready. + +@param {object} root Root template context +@param {string} type Resource type: "service" (waits for pod readiness) or "job" (waits for completion) +@param {string} name For "service": component name (used for pod label selector) + For "job": job name suffix (appended to fullname) +@return {string} YAML-formatted initContainer definition +*/}} +{{- define "clp.waitFor" -}} +- name: "wait-for-{{ .name }}" + image: "bitnami/kubectl:latest" + command: [ + "kubectl", "wait", + {{- if eq .type "service" }} + "--for=condition=ready", + "pod", "--selector", "app.kubernetes.io/component={{ .name }}", + {{- else if eq .type "job" }} + "--for=condition=complete", + "job/{{ include "clp.fullname" .root }}-{{ .name }}", + {{- end }} + "--timeout=300s" + ] +{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 7a97719902..562933dcc3 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -17,32 +17,15 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "api-server" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done - - name: "wait-for-results-cache" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-results-cache 27017; - do echo Waiting for results-cache to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} containers: - name: "api-server" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 1de5e362ea..fa71e57235 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -17,42 +17,16 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "compression-scheduler" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 300 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done - - name: "wait-for-queue" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-queue 5672; - do echo Waiting for queue to be ready...; - sleep 2; - done - - name: "wait-for-redis" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-redis 6379; - do echo Waiting for redis to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} containers: - name: "compression-scheduler" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 92db721a17..9f3f17a913 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -17,32 +17,15 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "compression-worker" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-queue" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-queue 5672; - do echo Waiting for queue to be ready...; - sleep 2; - done - - name: "wait-for-redis" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-redis 6379; - do echo Waiting for redis to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} containers: - name: "compression-worker" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index bc600db429..8772c56fd8 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -13,22 +13,14 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "db-table-creator" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher restartPolicy: "OnFailure" securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "database") | nindent 8 }} containers: - name: "db-table-creator" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index a4fcaab0cf..157bcbee0e 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -17,32 +17,15 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "garbage-collector" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done - - name: "wait-for-results-cache" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-results-cache 27017; - do echo Waiting for results-cache to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} containers: - name: "garbage-collector" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/job-watcher-role.yaml b/tools/deployment/package-helm/templates/job-watcher-role.yaml new file mode 100644 index 0000000000..7a19885ecb --- /dev/null +++ b/tools/deployment/package-helm/templates/job-watcher-role.yaml @@ -0,0 +1,13 @@ +apiVersion: "rbac.authorization.k8s.io/v1" +kind: "Role" +metadata: + name: {{ include "clp.fullname" . }}-job-reader + labels: + {{- include "clp.labels" . | nindent 4 }} +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] diff --git a/tools/deployment/package-helm/templates/job-watcher-rolebinding.yaml b/tools/deployment/package-helm/templates/job-watcher-rolebinding.yaml new file mode 100644 index 0000000000..78a61624b0 --- /dev/null +++ b/tools/deployment/package-helm/templates/job-watcher-rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: "rbac.authorization.k8s.io/v1" +kind: "RoleBinding" +metadata: + name: {{ include "clp.fullname" . }}-job-watcher-binding + labels: + {{- include "clp.labels" . | nindent 4 }} +subjects: + - kind: "ServiceAccount" + name: {{ include "clp.fullname" . }}-job-watcher + namespace: {{ .Release.Namespace }} +roleRef: + kind: "Role" + name: {{ include "clp.fullname" . }}-job-reader + apiGroup: "rbac.authorization.k8s.io" diff --git a/tools/deployment/package-helm/templates/job-watcher-serviceaccount.yaml b/tools/deployment/package-helm/templates/job-watcher-serviceaccount.yaml new file mode 100644 index 0000000000..54f305eeb3 --- /dev/null +++ b/tools/deployment/package-helm/templates/job-watcher-serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: "v1" +kind: "ServiceAccount" +metadata: + name: {{ include "clp.fullname" . }}-job-watcher + labels: + {{- include "clp.labels" . | nindent 4 }} diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index bfe5273352..2efa30a4c0 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -18,32 +18,15 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "mcp-server" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done - - name: "wait-for-results-cache" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-results-cache 27017; - do echo Waiting for results-cache to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} containers: - name: "mcp-server" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index b6c6a2990e..7ee62c3d8d 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -17,6 +17,7 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "query-scheduler" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher hostname: {{ include "clp.fullname" . }}-query-scheduler subdomain: {{ include "clp.fullname" . }}-query-scheduler terminationGracePeriodSeconds: 10 @@ -25,36 +26,9 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done - - name: "wait-for-queue" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-queue 5672; - do echo Waiting for queue to be ready...; - sleep 2; - done - - name: "wait-for-redis" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-redis 6379; - do echo Waiting for redis to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} containers: - name: "query-scheduler" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index ba395d4d29..b461996427 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -17,32 +17,15 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "query-worker" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-queue" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-queue 5672; - do echo Waiting for queue to be ready...; - sleep 2; - done - - name: "wait-for-redis" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-redis 6379; - do echo Waiting for redis to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} containers: - name: "query-worker" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index edde1c8fcd..869eb36fe4 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -17,6 +17,7 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "reducer" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher hostname: {{ include "clp.fullname" . }}-reducer subdomain: {{ include "clp.fullname" . }}-reducer terminationGracePeriodSeconds: 10 @@ -25,26 +26,8 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-query-scheduler" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-query-scheduler 7000; - do echo Waiting for query-scheduler to be ready...; - sleep 2; - done - - name: "wait-for-results-cache" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-results-cache 27017; - do echo Waiting for results-cache to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "query-scheduler") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} containers: - name: "reducer" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index 51c2b822db..f2cb81d75f 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -13,22 +13,14 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "results-cache-indices-creator" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher restartPolicy: "OnFailure" securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-results-cache" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-results-cache 27017; - do echo Waiting for results-cache to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "results-cache") | nindent 8 }} containers: - name: "results-cache-indices-creator" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 97ad133ac6..6ca09c1d38 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -17,32 +17,15 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "webui" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done - - name: "wait-for-results-cache" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-results-cache 27017; - do echo Waiting for results-cache to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} containers: - name: "webui" image: "{{ include "clp.image.ref" . }}" From b231898d1b36eea6de8b7abd116eb933bc006bbc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 12 Dec 2025 02:29:08 -0500 Subject: [PATCH 106/155] feat(helm): Refactor PersistentVolume and PersistentVolumeClaim templates with shared helpers and consistent naming conventions --- .../package-helm/templates/_helpers.tpl | 25 ++++++++-- .../templates/api-server-deployment.yaml | 8 ++-- .../templates/api-server-logs-pv.yaml | 4 +- .../templates/api-server-logs-pvc.yaml | 3 +- .../package-helm/templates/archives-pv.yaml | 9 ---- .../compression-scheduler-deployment.yaml | 24 ++++++---- .../compression-scheduler-logs-pv.yaml | 4 +- .../compression-scheduler-logs-pvc.yaml | 3 +- ...> compression-scheduler-user-logs-pv.yaml} | 4 +- ... compression-scheduler-user-logs-pvc.yaml} | 3 +- .../compression-worker-deployment.yaml | 48 +++++++++++-------- .../templates/compression-worker-logs-pv.yaml | 4 +- .../compression-worker-logs-pvc.yaml | 3 +- ...compression-worker-staged-archives-pv.yaml | 6 +-- ...ompression-worker-staged-archives-pvc.yaml | 3 +- ...pv.yaml => compression-worker-tmp-pv.yaml} | 4 +- ...c.yaml => compression-worker-tmp-pvc.yaml} | 3 +- .../templates/database-data-pv.yaml | 2 +- .../templates/database-logs-pv.yaml | 2 +- .../garbage-collector-deployment.yaml | 28 ++++++----- .../templates/garbage-collector-logs-pv.yaml | 4 +- .../templates/garbage-collector-logs-pvc.yaml | 3 +- .../templates/mcp-server-deployment.yaml | 8 ++-- .../templates/mcp-server-logs-pv.yaml | 6 ++- .../templates/mcp-server-logs-pvc.yaml | 3 +- .../templates/query-scheduler-deployment.yaml | 8 ++-- .../templates/query-scheduler-logs-pv.yaml | 4 +- .../templates/query-scheduler-logs-pvc.yaml | 3 +- .../templates/query-worker-deployment.yaml | 40 +++++++++------- .../templates/query-worker-logs-pv.yaml | 4 +- .../templates/query-worker-logs-pvc.yaml | 3 +- .../query-worker-staged-streams-pv.yaml | 6 +-- .../query-worker-staged-streams-pvc.yaml | 3 +- .../package-helm/templates/queue-logs-pv.yaml | 2 +- .../package-helm/templates/redis-data-pv.yaml | 2 +- .../package-helm/templates/redis-logs-pv.yaml | 2 +- .../templates/reducer-deployment.yaml | 8 ++-- .../templates/reducer-logs-pv.yaml | 4 +- .../templates/reducer-logs-pvc.yaml | 3 +- .../templates/results-cache-data-pv.yaml | 2 +- .../templates/results-cache-logs-pv.yaml | 2 +- .../templates/shared-data-archives-pv.yaml | 9 ++++ ...pvc.yaml => shared-data-archives-pvc.yaml} | 3 +- .../templates/shared-data-streams-pv.yaml | 9 ++++ ...-pvc.yaml => shared-data-streams-pvc.yaml} | 3 +- .../package-helm/templates/streams-pv.yaml | 9 ---- .../templates/webui-deployment.yaml | 10 ++-- 47 files changed, 210 insertions(+), 143 deletions(-) delete mode 100644 tools/deployment/package-helm/templates/archives-pv.yaml rename tools/deployment/package-helm/templates/{user-logs-pv.yaml => compression-scheduler-user-logs-pv.yaml} (70%) rename tools/deployment/package-helm/templates/{tmp-pvc.yaml => compression-scheduler-user-logs-pvc.yaml} (64%) rename tools/deployment/package-helm/templates/{tmp-pv.yaml => compression-worker-tmp-pv.yaml} (71%) rename tools/deployment/package-helm/templates/{user-logs-pvc.yaml => compression-worker-tmp-pvc.yaml} (68%) create mode 100644 tools/deployment/package-helm/templates/shared-data-archives-pv.yaml rename tools/deployment/package-helm/templates/{archives-pvc.yaml => shared-data-archives-pvc.yaml} (69%) create mode 100644 tools/deployment/package-helm/templates/shared-data-streams-pv.yaml rename tools/deployment/package-helm/templates/{streams-pvc.yaml => shared-data-streams-pvc.yaml} (69%) delete mode 100644 tools/deployment/package-helm/templates/streams-pv.yaml diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index d4de98d0e1..b856e3a1bd 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -98,8 +98,8 @@ failureThreshold: 3 Creates a local PersistentVolume. @param {object} root Root template context -@param {string} name PV name -@param {string} component Component label +@param {string} component Component category (e.g., "shared-data", "database") +@param {string} name Storage name (e.g., "archives", "logs", "data") @param {string} nodeRole Node role for affinity. Targets nodes with label "node-role.kubernetes.io/". Always falls back to "node-role.kubernetes.io/control-plane" @@ -112,7 +112,7 @@ Creates a local PersistentVolume. apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ .name }} + name: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} labels: {{- include "clp.labels" .root | nindent 4 }} app.kubernetes.io/component: {{ .component | quote }} @@ -139,7 +139,8 @@ spec: Creates a PersistentVolumeClaim for the given component. @param {object} root Root template context -@param {string} component Component label +@param {string} component Component category (e.g., "shared-data", "database") +@param {string} name Storage name (e.g., "archives", "logs", "data") @param {string} capacity Storage capacity @param {string[]} accessModes Access modes @return {string} YAML-formatted PersistentVolumeClaim resource @@ -148,7 +149,7 @@ Creates a PersistentVolumeClaim for the given component. apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp.fullname" .root }}-{{ .component }} + name: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} labels: {{- include "clp.labels" .root | nindent 4 }} app.kubernetes.io/component: {{ .component | quote }} @@ -164,6 +165,20 @@ spec: storage: {{ .capacity }} {{- end }} +{{/* +Creates a volume definition that references a PersistentVolumeClaim. + +@param {object} root Root template context +@param {string} component Component category (e.g., "shared-data", "database") +@param {string} name Storage name (e.g., "archives", "logs", "data") +@return {string} YAML-formatted volume definition +*/}} +{{- define "clp.pvcVolume" -}} +- name: {{ printf "%s-%s" .component .name | quote }} + persistentVolumeClaim: + claimName: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} +{{- end }} + {{/* Creates an initContainer that waits for a Kubernetes resource to be ready. diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 562933dcc3..2e5f8a8809 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -70,9 +70,11 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} httpGet: *api-server-health-check volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "api-server" + "name" "logs" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "api-server-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-api-server-logs diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index 74d372325c..87b0b609e8 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-api-server-logs" (include "clp.fullname" .)) - "component" "api-server-logs" + "component" "api-server" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml index 72d6f4702c..d63e3dbb3f 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "api-server-logs" + "component" "api-server" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/archives-pv.yaml b/tools/deployment/package-helm/templates/archives-pv.yaml deleted file mode 100644 index d1a043d64f..0000000000 --- a/tools/deployment/package-helm/templates/archives-pv.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- include "clp.createLocalPv" (dict - "root" . - "name" (printf "%s-archives" (include "clp.fullname" .)) - "component" "archives" - "nodeRole" "control-plane" - "capacity" "50Gi" - "accessModes" (list "ReadWriteMany") - "hostPath" .Values.clpConfig.archive_output.storage.directory -) }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index fa71e57235..634a859582 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -53,14 +53,14 @@ spec: - name: "RESULT_BACKEND" value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: + - name: "compression-scheduler-logs" + mountPath: "/var/log/compression_scheduler" + - name: "compression-scheduler-user-logs" + mountPath: "/var/log/user" - name: "config" mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true - - name: "compression-scheduler-logs" - mountPath: "/var/log/compression_scheduler" - - name: "user-logs" - mountPath: "/var/log/user" - name: "logs-input" mountPath: "/mnt/logs" readOnly: true @@ -70,15 +70,19 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "compression-scheduler" + "name" "logs" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "compression-scheduler" + "name" "user-logs" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "compression-scheduler-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-compression-scheduler-logs - - name: "user-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-user-logs - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 48a46f4f8a..3404844121 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-compression-scheduler-logs" (include "clp.fullname" .)) - "component" "compression-scheduler-logs" + "component" "compression-scheduler" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml index 63011569c5..0d19be396a 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-scheduler-logs" + "component" "compression-scheduler" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/user-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml similarity index 70% rename from tools/deployment/package-helm/templates/user-logs-pv.yaml rename to tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml index 9c43c56e00..6653e4290d 100644 --- a/tools/deployment/package-helm/templates/user-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-user-logs" (include "clp.fullname" .)) - "component" "user-logs" + "component" "compression-scheduler" + "name" "user-logs" "nodeRole" "control-plane" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/tmp-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml similarity index 64% rename from tools/deployment/package-helm/templates/tmp-pvc.yaml rename to tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml index abdddcc887..bf56a89311 100644 --- a/tools/deployment/package-helm/templates/tmp-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "tmp" + "component" "compression-scheduler" + "name" "user-logs" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") ) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 9f3f17a913..0bfcd2954b 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -48,21 +48,21 @@ spec: - name: "RESULT_BACKEND" value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: + - name: "compression-worker-logs" + mountPath: "/var/log/compression_worker" + - name: "compression-worker-staged-archives" + mountPath: "/var/data/staged-archives" + - name: "compression-worker-tmp" + mountPath: "/var/tmp" - name: "config" mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true - - name: "compression-worker-logs" - mountPath: "/var/log/compression_worker" - - name: "archives" - mountPath: "/var/data/archives" - name: "logs-input" mountPath: "/mnt/logs" readOnly: true - - name: "staged-archives" - mountPath: "/var/data/staged-archives" - - name: "tmp" - mountPath: "/var/tmp" + - name: "shared-data-archives" + mountPath: "/var/data/archives" command: [ "python3", "-u", "/opt/clp/lib/python3/site-packages/bin/celery", @@ -75,22 +75,30 @@ spec: "-n", "compression-worker" ] volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "compression-worker" + "name" "logs" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "compression-worker" + "name" "staged-archives" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "compression-worker" + "name" "tmp" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "shared-data" + "name" "archives" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "compression-worker-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-compression-worker-logs - - name: "archives" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-archives - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" type: "Directory" - - name: "staged-archives" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-staged-archives - - name: "tmp" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-tmp diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index 3c451126d2..98e58cab64 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-compression-worker-logs" (include "clp.fullname" .)) - "component" "compression-worker-logs" + "component" "compression-worker" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml index 45b11b1fad..b6953b0393 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-worker-logs" + "component" "compression-worker" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index 11dcbd1d8a..b79ea5d0c1 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -1,9 +1,9 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-staged-archives" (include "clp.fullname" .)) - "component" "staged-archives" + "component" "compression-worker" + "name" "staged-archives" "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") - "hostPath" "/tmp/clp/var/data/staged-archives" + "hostPath" (printf "%s/staged-archives" .Values.clpConfig.data_directory) ) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml index 77410397a8..77f0324264 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "staged-archives" + "component" "compression-worker" + "name" "staged-archives" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") ) }} diff --git a/tools/deployment/package-helm/templates/tmp-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml similarity index 71% rename from tools/deployment/package-helm/templates/tmp-pv.yaml rename to tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml index 1add0502bd..18cd4a68b7 100644 --- a/tools/deployment/package-helm/templates/tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-tmp" (include "clp.fullname" .)) - "component" "tmp" + "component" "compression-worker" + "name" "tmp" "nodeRole" "control-plane" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/user-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml similarity index 68% rename from tools/deployment/package-helm/templates/user-logs-pvc.yaml rename to tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml index 2fb0688dc2..51b7bc0d60 100644 --- a/tools/deployment/package-helm/templates/user-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "user-logs" + "component" "compression-worker" + "name" "tmp" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") ) }} diff --git a/tools/deployment/package-helm/templates/database-data-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml index 98beaaf866..1515a853c8 100644 --- a/tools/deployment/package-helm/templates/database-data-pv.yaml +++ b/tools/deployment/package-helm/templates/database-data-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-database-data" (include "clp.fullname" .)) "component" "database" + "name" "data" "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml index 27ad28d197..ea454a8676 100644 --- a/tools/deployment/package-helm/templates/database-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-database-logs" (include "clp.fullname" .)) "component" "database" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 157bcbee0e..bb8022b0ed 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -56,9 +56,9 @@ spec: readOnly: true - name: "garbage-collector-logs" mountPath: "/var/log/garbage_collector" - - name: "archives" + - name: "shared-data-archives" mountPath: "/var/data/archives" - - name: "streams" + - name: "shared-data-streams" mountPath: "/var/data/streams" command: [ "python3", "-u", @@ -66,15 +66,21 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "garbage-collector" + "name" "logs" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "shared-data" + "name" "archives" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "shared-data" + "name" "streams" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "garbage-collector-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-garbage-collector-logs - - name: "archives" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-archives - - name: "streams" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-streams diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index 75f3a6a3e5..3f8ece5a41 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-garbage-collector-logs" (include "clp.fullname" .)) - "component" "garbage-collector-logs" + "component" "garbage-collector" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml index 714e35bd94..f84222d9b8 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "garbage-collector-logs" + "component" "garbage-collector" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 2efa30a4c0..fe355403f7 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -74,10 +74,12 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} httpGet: *mcp-server-health-check volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "mcp-server" + "name" "logs" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "mcp-server-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-mcp-server-logs {{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index 01bbf18ab7..fb1a6a94a5 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -1,9 +1,11 @@ +{{- if .Values.clpConfig.mcp_server }} {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-mcp-server-logs" (include "clp.fullname" .)) - "component" "mcp-server-logs" + "component" "mcp-server" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/mcp-server" .Values.clpConfig.logs_directory) ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml index f648fa9ccd..61219f973f 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml @@ -1,7 +1,8 @@ {{- if .Values.clpConfig.mcp_server }} {{- include "clp.createPvc" (dict "root" . - "component" "mcp-server-logs" + "component" "mcp-server" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 7ee62c3d8d..ed6630d9c3 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -70,9 +70,11 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "query-scheduler" + "name" "logs" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "query-scheduler-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-query-scheduler-logs diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index d7b62604c4..826a98d3ac 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-query-scheduler-logs" (include "clp.fullname" .)) - "component" "query-scheduler-logs" + "component" "query-scheduler" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml index bf9794ada9..559d3518ff 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "query-scheduler-logs" + "component" "query-scheduler" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index b461996427..150b2e65fb 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -54,11 +54,11 @@ spec: readOnly: true - name: "query-worker-logs" mountPath: "/var/log/query_worker" - - name: "archives" - mountPath: "/var/data/archives" - - name: "staged-streams" + - name: "query-worker-staged-streams" mountPath: "/var/data/staged-streams" - - name: "streams" + - name: "shared-data-archives" + mountPath: "/var/data/archives" + - name: "shared-data-streams" mountPath: "/var/data/streams" command: [ "python3", "-u", @@ -72,18 +72,26 @@ spec: "-n", "query-worker" ] volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "query-worker" + "name" "logs" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "query-worker" + "name" "staged-streams" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "shared-data" + "name" "archives" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "shared-data" + "name" "streams" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "query-worker-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-query-worker-logs - - name: "archives" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-archives - - name: "staged-streams" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-staged-streams - - name: "streams" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-streams diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index e92ee8b03f..6e50f43ea1 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-query-worker-logs" (include "clp.fullname" .)) - "component" "query-worker-logs" + "component" "query-worker" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml index bfbc07a9d1..eb2f41e916 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "query-worker-logs" + "component" "query-worker" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index 95a641be53..02b95378a6 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -1,9 +1,9 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-staged-streams" (include "clp.fullname" .)) - "component" "staged-streams" + "component" "query-worker" + "name" "staged-streams" "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") - "hostPath" "/tmp/clp/var/data/staged-streams" + "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) ) }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml index 195d94319e..242fe1ff4a 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "staged-streams" + "component" "query-worker" + "name" "staged-streams" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") ) }} diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml index cbc4fa3008..a24808b632 100644 --- a/tools/deployment/package-helm/templates/queue-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-queue-logs" (include "clp.fullname" .)) "component" "queue" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml index 96fa611225..d5be52b6b1 100644 --- a/tools/deployment/package-helm/templates/redis-data-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-redis-data" (include "clp.fullname" .)) "component" "redis" + "name" "data" "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml index b3308cbfc9..755484bacd 100644 --- a/tools/deployment/package-helm/templates/redis-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-redis-logs" (include "clp.fullname" .)) "component" "redis" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index 869eb36fe4..4ed9d77e5d 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -56,9 +56,11 @@ spec: "--upsert-interval", "{{ .Values.clpConfig.reducer.upsert_interval }}" ] volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "reducer" + "name" "logs" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "reducer-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-reducer-logs diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index 12c67a778f..1d12c20d16 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-reducer-logs" (include "clp.fullname" .)) - "component" "reducer-logs" + "component" "reducer" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml index cd4eec1792..78dbd64288 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "reducer-logs" + "component" "reducer" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml index 8c67722348..c55a858b39 100644 --- a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-results-cache-data" (include "clp.fullname" .)) "component" "results-cache" + "name" "data" "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index 2050d3a033..77a99383a5 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-results-cache-logs" (include "clp.fullname" .)) "component" "results-cache" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml new file mode 100644 index 0000000000..9373c85fcf --- /dev/null +++ b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "component" "shared-data" + "name" "archives" + "nodeRole" "control-plane" + "capacity" "50Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" (printf "%s/archives" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/archives-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml similarity index 69% rename from tools/deployment/package-helm/templates/archives-pvc.yaml rename to tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml index c5b1bac44f..fb53ebd225 100644 --- a/tools/deployment/package-helm/templates/archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "archives" + "component" "shared-data" + "name" "archives" "capacity" "50Gi" "accessModes" (list "ReadWriteMany") ) }} diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml new file mode 100644 index 0000000000..d96a7be523 --- /dev/null +++ b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "component" "shared-data" + "name" "streams" + "nodeRole" "control-plane" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" (printf "%s/streams" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/streams-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml similarity index 69% rename from tools/deployment/package-helm/templates/streams-pvc.yaml rename to tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml index bd754e6c42..4f4e6ea9a0 100644 --- a/tools/deployment/package-helm/templates/streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "streams" + "component" "shared-data" + "name" "streams" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") ) }} diff --git a/tools/deployment/package-helm/templates/streams-pv.yaml b/tools/deployment/package-helm/templates/streams-pv.yaml deleted file mode 100644 index df211cf22f..0000000000 --- a/tools/deployment/package-helm/templates/streams-pv.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- include "clp.createLocalPv" (dict - "root" . - "name" (printf "%s-streams" (include "clp.fullname" .)) - "component" "streams" - "nodeRole" "control-plane" - "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") - "hostPath" .Values.clpConfig.stream_output.storage.directory -) }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 6ca09c1d38..f792887afc 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -63,7 +63,7 @@ spec: mountPath: "/opt/clp/var/www/webui/server/dist/settings.json" subPath: "webui-server-settings.json" readOnly: true - - name: "streams" + - name: "shared-data-streams" mountPath: "/var/data/streams" command: [ "/opt/clp/bin/node-22", @@ -77,12 +77,14 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: *webui-health-check volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "shared-data" + "name" "streams" + ) | nindent 8 }} - name: "client-settings" configMap: name: {{ include "clp.fullname" . }}-config - name: "server-settings" configMap: name: {{ include "clp.fullname" . }}-config - - name: "streams" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-streams From 44b0f17cb41b035d58e7d69c4eb88abeeb735ca9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 12 Dec 2025 02:45:37 -0500 Subject: [PATCH 107/155] feat(helm): Add shared helpers for Celery environment variables and improve readability of templates with consistent formatting --- .../package-helm/templates/_helpers.tpl | 28 +++++++++++++++ .../templates/api-server-deployment.yaml | 12 +++++-- .../compression-scheduler-deployment.yaml | 35 +++++++++++++------ .../compression-worker-deployment.yaml | 9 ++--- .../garbage-collector-deployment.yaml | 12 +++++-- .../templates/mcp-server-deployment.yaml | 12 +++++-- .../templates/query-scheduler-deployment.yaml | 35 +++++++++++++------ .../templates/query-worker-deployment.yaml | 9 ++--- .../templates/reducer-deployment.yaml | 12 +++++-- .../results-cache-indices-creator-job.yaml | 13 +++++-- .../templates/webui-deployment.yaml | 12 +++++-- 11 files changed, 146 insertions(+), 43 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index b856e3a1bd..988bcf65ee 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -179,6 +179,34 @@ Creates a volume definition that references a PersistentVolumeClaim. claimName: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} {{- end }} +{{/* +Creates the BROKER_URL env var for Celery workers. + +@param {object} . Root template context +@return {string} YAML-formatted env var definition +*/}} +{{- define "clp.celeryBrokerUrlEnvVar" -}} +{{- $user := .Values.credentials.queue.username -}} +{{- $pass := .Values.credentials.queue.password -}} +{{- $host := printf "%s-queue" (include "clp.fullname" .) -}} +- name: "BROKER_URL" + value: {{ printf "amqp://%s:%s@%s:5672" $user $pass $host | quote }} +{{- end }} + +{{/* +Creates the RESULT_BACKEND env var for Celery workers. + +@param {object} root Root template context +@param {string} database Redis database number from config +@return {string} YAML-formatted env var definition +*/}} +{{- define "clp.celeryResultBackendEnvVar" -}} +{{- $pass := .root.Values.credentials.redis.password -}} +{{- $host := printf "%s-redis" (include "clp.fullname" .root) -}} +- name: "RESULT_BACKEND" + value: {{ printf "redis://default:%s@%s:6379/%s" $pass $host .database | quote }} +{{- end }} + {{/* Creates an initContainer that waits for a Kubernetes resource to be ready. diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 2e5f8a8809..51a5291a1c 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -24,8 +24,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 8 }} containers: - name: "api-server" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 634a859582..fa7fd55118 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -24,34 +24,47 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "queue" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "redis" + ) | nindent 8 }} containers: - name: "compression-scheduler" image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - - name: "CLP_DB_USER" + {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} + {{- include "clp.celeryResultBackendEnvVar" (dict + "root" . + "database" .Values.clpConfig.redis.compression_backend_database + ) | nindent 12 }} + - name: "CLP_DB_PASS" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "username" - - name: "CLP_DB_PASS" + key: "password" + - name: "CLP_DB_USER" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "username" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} - name: "CLP_LOGS_DIR" value: "/var/log/compression_scheduler" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: - name: "compression-scheduler-logs" mountPath: "/var/log/compression_scheduler" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 0bfcd2954b..7d9a21524b 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -31,8 +31,11 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" + {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} + {{- include "clp.celeryResultBackendEnvVar" (dict + "root" . + "database" .Values.clpConfig.redis.compression_backend_database + ) | nindent 12 }} - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yaml" - name: "CLP_HOME" @@ -45,8 +48,6 @@ spec: value: "/var/log/compression_worker/worker.log" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: - name: "compression-worker-logs" mountPath: "/var/log/compression_worker" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index bb8022b0ed..b5e22ebff2 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -24,8 +24,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 8 }} containers: - name: "garbage-collector" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index fe355403f7..13863f2f60 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -25,8 +25,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 8 }} containers: - name: "mcp-server" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index ed6630d9c3..94d79a28d3 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -26,34 +26,47 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "queue" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "redis" + ) | nindent 8 }} containers: - name: "query-scheduler" image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - - name: "CLP_DB_USER" + {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} + {{- include "clp.celeryResultBackendEnvVar" (dict + "root" . + "database" .Values.clpConfig.redis.query_backend_database + ) | nindent 12 }} + - name: "CLP_DB_PASS" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "username" - - name: "CLP_DB_PASS" + key: "password" + - name: "CLP_DB_USER" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "username" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} - name: "CLP_LOGS_DIR" value: "/var/log/query_scheduler" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" ports: - name: "query-scheduler" containerPort: 7000 diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 150b2e65fb..5329f3b745 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -31,8 +31,11 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" + {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} + {{- include "clp.celeryResultBackendEnvVar" (dict + "root" . + "database" .Values.clpConfig.redis.query_backend_database + ) | nindent 12 }} - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yaml" - name: "CLP_HOME" @@ -45,8 +48,6 @@ spec: value: "/var/log/query_worker/worker.log" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" volumeMounts: - name: "config" mountPath: "/etc/clp-config.yaml" diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index 4ed9d77e5d..f6ee050a35 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -26,8 +26,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "query-scheduler") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "query-scheduler" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 8 }} containers: - name: "reducer" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index f2cb81d75f..0dcf9ee646 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -20,7 +20,11 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "results-cache") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "results-cache" + ) | nindent 8 }} containers: - name: "results-cache-indices-creator" image: "{{ include "clp.image.ref" . }}" @@ -31,6 +35,9 @@ spec: command: [ "python3", "-u", "-m", "clp_py_utils.initialize-results-cache", - "--uri", "mongodb://{{ include "clp.fullname" . }}-results-cache:27017/{{ .Values.clpConfig.results_cache.db_name }}", - "--stream-collection", {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} + "--uri", + "mongodb://{{ include "clp.fullname" . }}-results-cache:27017/{{ + .Values.clpConfig.results_cache.db_name }}", + "--stream-collection", {{ .Values.clpConfig.results_cache.stream_collection_name + | quote }} ] diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index f792887afc..4f0415556f 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -24,8 +24,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 8 }} containers: - name: "webui" image: "{{ include "clp.image.ref" . }}" From a9e6d09d5c62a5822a1f9f40dd0813714cfacbc2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 12 Dec 2025 02:54:34 -0500 Subject: [PATCH 108/155] feat(helm): Update hostPath in PersistentVolume templates for consistency --- tools/deployment/package-helm/templates/api-server-logs-pv.yaml | 2 +- .../package-helm/templates/compression-scheduler-logs-pv.yaml | 2 +- .../package-helm/templates/compression-worker-logs-pv.yaml | 2 +- .../package-helm/templates/garbage-collector-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml | 2 +- .../package-helm/templates/query-scheduler-logs-pv.yaml | 2 +- .../deployment/package-helm/templates/query-worker-logs-pv.yaml | 2 +- .../package-helm/templates/results-cache-data-pv.yaml | 2 +- .../package-helm/templates/results-cache-logs-pv.yaml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index 87b0b609e8..84b44ad906 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/api-server" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/api_server" .Values.clpConfig.logs_directory) ) }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 3404844121..565fc541f1 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/compression-scheduler" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/compression_scheduler" .Values.clpConfig.logs_directory) ) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index 98e58cab64..aa1f3d165e 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/compression-worker" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/compression_worker" .Values.clpConfig.logs_directory) ) }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index 3f8ece5a41..d986b3cf7a 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/garbage-collector" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/garbage_collector" .Values.clpConfig.logs_directory) ) }} diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index fb1a6a94a5..5af6d1c6ed 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -6,6 +6,6 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/mcp-server" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/mcp_server" .Values.clpConfig.logs_directory) ) }} {{- end }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index 826a98d3ac..346b250c2c 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/query-scheduler" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/query_scheduler" .Values.clpConfig.logs_directory) ) }} diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index 6e50f43ea1..65f9ff3eed 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/query-worker" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/query_worker" .Values.clpConfig.logs_directory) ) }} diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml index c55a858b39..a4a32223b5 100644 --- a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/results-cache" .Values.clpConfig.data_directory) + "hostPath" (printf "%s/results_cache" .Values.clpConfig.data_directory) ) }} diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index 77a99383a5..9498d0ad65 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/results-cache" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/results_cache" .Values.clpConfig.logs_directory) ) }} From 92790326fcff6874e78fe6504ebb1069906e1940 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 12 Dec 2025 03:13:39 -0500 Subject: [PATCH 109/155] fix(test): Correct temp directory structure and ensure Redis database value is cast to integer in Helm templates --- tools/deployment/package-helm/templates/_helpers.tpl | 2 +- tools/deployment/package-helm/test.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 988bcf65ee..59d0a84b71 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -204,7 +204,7 @@ Creates the RESULT_BACKEND env var for Celery workers. {{- $pass := .root.Values.credentials.redis.password -}} {{- $host := printf "%s-redis" (include "clp.fullname" .root) -}} - name: "RESULT_BACKEND" - value: {{ printf "redis://default:%s@%s:6379/%s" $pass $host .database | quote }} + value: {{ printf "redis://default:%s@%s:6379/%d" $pass $host (int .database) | quote }} {{- end }} {{/* diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 5e9b9fbf67..8322934ce6 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -8,7 +8,8 @@ set -o pipefail kind delete cluster --name clp-test rm -rf /tmp/clp -mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker,reducer,garbage-collector,api-server,mcp-server} +mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results_cache} +mkdir -p /tmp/clp/var/log/{compression_scheduler,compression_worker,query_scheduler,query_worker,reducer,garbage_collector,api_server,mcp_server} mkdir -p /tmp/clp/var/data/{archives,staged-archives,staged-streams,streams} mkdir -p /tmp/clp/var/log/user mkdir -p /tmp/clp/var/tmp From 8580c113a0a6461b39532e223627ef9b0aa58cd0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 12 Dec 2025 10:17:36 -0500 Subject: [PATCH 110/155] refactor(helm): Reorder volumeMounts in StatefulSet templates for consistency across components --- .../package-helm/templates/api-server-deployment.yaml | 4 ++-- .../package-helm/templates/database-statefulset.yaml | 8 ++++---- .../package-helm/templates/redis-statefulset.yaml | 8 ++++---- .../package-helm/templates/results-cache-statefulset.yaml | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 51a5291a1c..6571f84b9f 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -57,12 +57,12 @@ spec: - name: "api-server" containerPort: 3001 volumeMounts: + - name: "api-server-logs" + mountPath: "/var/log/api_server" - name: "config" mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true - - name: "api-server-logs" - mountPath: "/var/log/api_server" command: [ "/opt/clp/bin/api_server", "--host", "0.0.0.0", diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 47801e20f7..208dd6295d 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -53,14 +53,14 @@ spec: - name: "database" containerPort: 3306 volumeMounts: - - name: "database-data" - mountPath: "/var/lib/mysql" - - name: "database-logs" - mountPath: "/var/log/mysql" - name: "config" mountPath: "/etc/mysql/conf.d/logging.cnf" subPath: "mysql-logging.cnf" readOnly: true + - name: "database-data" + mountPath: "/var/lib/mysql" + - name: "database-logs" + mountPath: "/var/log/mysql" readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &database-health-check diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index d7895ee04a..f0c6b760da 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -41,14 +41,14 @@ spec: - name: "redis" containerPort: 6379 volumeMounts: - - name: "redis-data" - mountPath: "/data" - - name: "redis-logs" - mountPath: "/var/log/redis" - name: "config" mountPath: "/etc/redis/redis.conf" subPath: "redis.conf" readOnly: true + - name: "redis-data" + mountPath: "/data" + - name: "redis-logs" + mountPath: "/var/log/redis" readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &redis-health-check diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index adbe0cae9b..3245780e9b 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -34,14 +34,14 @@ spec: - name: "results-cache" containerPort: 27017 volumeMounts: - - name: "results-cache-data" - mountPath: "/data/db" - - name: "results-cache-logs" - mountPath: "/var/log/mongodb" - name: "config" mountPath: "/etc/mongo/mongod.conf" subPath: "mongod.conf" readOnly: true + - name: "results-cache-data" + mountPath: "/data/db" + - name: "results-cache-logs" + mountPath: "/var/log/mongodb" readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &results-cache-health-check From 1c2669281345b6280386a3f468165bf39f0c816e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 14 Dec 2025 18:31:08 -0500 Subject: [PATCH 111/155] refactor(helm): Update templates to replace `component` with `component_category` for improved clarity and consistency --- .../package-helm/templates/_helpers.tpl | 32 +++++++++---------- .../templates/api-server-deployment.yaml | 2 +- .../templates/api-server-logs-pv.yaml | 2 +- .../templates/api-server-logs-pvc.yaml | 2 +- .../compression-scheduler-deployment.yaml | 4 +-- .../compression-scheduler-logs-pv.yaml | 2 +- .../compression-scheduler-logs-pvc.yaml | 2 +- .../compression-scheduler-user-logs-pv.yaml | 2 +- .../compression-scheduler-user-logs-pvc.yaml | 2 +- .../compression-worker-deployment.yaml | 8 ++--- .../templates/compression-worker-logs-pv.yaml | 2 +- .../compression-worker-logs-pvc.yaml | 2 +- ...compression-worker-staged-archives-pv.yaml | 2 +- ...ompression-worker-staged-archives-pvc.yaml | 2 +- .../templates/compression-worker-tmp-pv.yaml | 2 +- .../templates/compression-worker-tmp-pvc.yaml | 2 +- .../templates/database-data-pv.yaml | 2 +- .../templates/database-logs-pv.yaml | 2 +- .../garbage-collector-deployment.yaml | 6 ++-- .../templates/garbage-collector-logs-pv.yaml | 2 +- .../templates/garbage-collector-logs-pvc.yaml | 2 +- .../templates/mcp-server-deployment.yaml | 2 +- .../templates/mcp-server-logs-pv.yaml | 2 +- .../templates/mcp-server-logs-pvc.yaml | 2 +- .../templates/query-scheduler-deployment.yaml | 2 +- .../templates/query-scheduler-logs-pv.yaml | 2 +- .../templates/query-scheduler-logs-pvc.yaml | 2 +- .../templates/query-worker-deployment.yaml | 8 ++--- .../templates/query-worker-logs-pv.yaml | 2 +- .../templates/query-worker-logs-pvc.yaml | 2 +- .../query-worker-staged-streams-pv.yaml | 2 +- .../query-worker-staged-streams-pvc.yaml | 2 +- .../package-helm/templates/queue-logs-pv.yaml | 2 +- .../package-helm/templates/redis-data-pv.yaml | 2 +- .../package-helm/templates/redis-logs-pv.yaml | 2 +- .../templates/reducer-deployment.yaml | 2 +- .../templates/reducer-logs-pv.yaml | 2 +- .../templates/reducer-logs-pvc.yaml | 2 +- .../templates/results-cache-data-pv.yaml | 2 +- .../templates/results-cache-logs-pv.yaml | 2 +- .../templates/shared-data-archives-pv.yaml | 2 +- .../templates/shared-data-archives-pvc.yaml | 2 +- .../templates/shared-data-streams-pv.yaml | 2 +- .../templates/shared-data-streams-pvc.yaml | 2 +- .../templates/webui-deployment.yaml | 2 +- 45 files changed, 69 insertions(+), 69 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 59d0a84b71..90c444f565 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -98,8 +98,8 @@ failureThreshold: 3 Creates a local PersistentVolume. @param {object} root Root template context -@param {string} component Component category (e.g., "shared-data", "database") -@param {string} name Storage name (e.g., "archives", "logs", "data") +@param {string} component_category (e.g., "shared-data", "database") +@param {string} name (e.g., "archives", "logs", "data") @param {string} nodeRole Node role for affinity. Targets nodes with label "node-role.kubernetes.io/". Always falls back to "node-role.kubernetes.io/control-plane" @@ -112,10 +112,10 @@ Creates a local PersistentVolume. apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} + name: {{ include "clp.fullname" .root }}-{{ .component_category }}-{{ .name }} labels: {{- include "clp.labels" .root | nindent 4 }} - app.kubernetes.io/component: {{ .component | quote }} + app.kubernetes.io/component: {{ .component_category | quote }} spec: capacity: storage: {{ .capacity }} @@ -139,8 +139,8 @@ spec: Creates a PersistentVolumeClaim for the given component. @param {object} root Root template context -@param {string} component Component category (e.g., "shared-data", "database") -@param {string} name Storage name (e.g., "archives", "logs", "data") +@param {string} component_category (e.g., "shared-data", "database") +@param {string} name (e.g., "archives", "logs", "data") @param {string} capacity Storage capacity @param {string[]} accessModes Access modes @return {string} YAML-formatted PersistentVolumeClaim resource @@ -149,17 +149,17 @@ Creates a PersistentVolumeClaim for the given component. apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} + name: {{ include "clp.fullname" .root }}-{{ .component_category }}-{{ .name }} labels: {{- include "clp.labels" .root | nindent 4 }} - app.kubernetes.io/component: {{ .component | quote }} + app.kubernetes.io/component: {{ .component_category | quote }} spec: accessModes: {{ .accessModes }} storageClassName: "local-storage" selector: matchLabels: {{- include "clp.selectorLabels" .root | nindent 6 }} - app.kubernetes.io/component: {{ .component | quote }} + app.kubernetes.io/component: {{ .component_category | quote }} resources: requests: storage: {{ .capacity }} @@ -169,14 +169,14 @@ spec: Creates a volume definition that references a PersistentVolumeClaim. @param {object} root Root template context -@param {string} component Component category (e.g., "shared-data", "database") -@param {string} name Storage name (e.g., "archives", "logs", "data") +@param {string} component_category (e.g., "shared-data", "database") +@param {string} name (e.g., "archives", "logs", "data") @return {string} YAML-formatted volume definition */}} {{- define "clp.pvcVolume" -}} -- name: {{ printf "%s-%s" .component .name | quote }} +- name: {{ printf "%s-%s" .component_category .name | quote }} persistentVolumeClaim: - claimName: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} + claimName: {{ include "clp.fullname" .root }}-{{ .component_category }}-{{ .name }} {{- end }} {{/* @@ -211,9 +211,9 @@ Creates the RESULT_BACKEND env var for Celery workers. Creates an initContainer that waits for a Kubernetes resource to be ready. @param {object} root Root template context -@param {string} type Resource type: "service" (waits for pod readiness) or "job" (waits for completion) -@param {string} name For "service": component name (used for pod label selector) - For "job": job name suffix (appended to fullname) +@param {string} type "service" (waits for pod readiness) or "job" (waits for completion) +@param {string} name For "service": component name + For "job": job name suffix @return {string} YAML-formatted initContainer definition */}} {{- define "clp.waitFor" -}} diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 6571f84b9f..8756623df3 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -80,7 +80,7 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "api-server" + "component_category" "api-server" "name" "logs" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index 84b44ad906..8d51f753fc 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "api-server" + "component_category" "api-server" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml index d63e3dbb3f..d290ac58cf 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "api-server" + "component_category" "api-server" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index fa7fd55118..fee1ba1d85 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -85,12 +85,12 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "compression-scheduler" + "component_category" "compression-scheduler" "name" "logs" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "compression-scheduler" + "component_category" "compression-scheduler" "name" "user-logs" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 565fc541f1..565f5dc40b 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "compression-scheduler" + "component_category" "compression-scheduler" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml index 0d19be396a..37d53a0105 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-scheduler" + "component_category" "compression-scheduler" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml index 6653e4290d..c8667c4597 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "compression-scheduler" + "component_category" "compression-scheduler" "name" "user-logs" "nodeRole" "control-plane" "capacity" "10Gi" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml index bf56a89311..5e0637d2eb 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-scheduler" + "component_category" "compression-scheduler" "name" "user-logs" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 7d9a21524b..8a4e31a972 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -78,22 +78,22 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "logs" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "staged-archives" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "tmp" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "archives" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index aa1f3d165e..4b6d55466b 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml index b6953b0393..136ffbd3ae 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index b79ea5d0c1..924fe4639b 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "staged-archives" "nodeRole" "control-plane" "capacity" "20Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml index 77f0324264..d96f2a62e1 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "staged-archives" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml index 18cd4a68b7..b091cc1634 100644 --- a/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "tmp" "nodeRole" "control-plane" "capacity" "10Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml index 51b7bc0d60..f1f500e60c 100644 --- a/tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "tmp" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/database-data-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml index 1515a853c8..3bd6ea2b9a 100644 --- a/tools/deployment/package-helm/templates/database-data-pv.yaml +++ b/tools/deployment/package-helm/templates/database-data-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "database" + "component_category" "database" "name" "data" "nodeRole" "control-plane" "capacity" "20Gi" diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml index ea454a8676..794215cf3a 100644 --- a/tools/deployment/package-helm/templates/database-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "database" + "component_category" "database" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index b5e22ebff2..1cac183243 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -76,17 +76,17 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "garbage-collector" + "component_category" "garbage-collector" "name" "logs" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "archives" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "streams" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index d986b3cf7a..e03597028c 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "garbage-collector" + "component_category" "garbage-collector" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml index f84222d9b8..45e39243b6 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "garbage-collector" + "component_category" "garbage-collector" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 13863f2f60..5f5a7d939d 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -84,7 +84,7 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "mcp-server" + "component_category" "mcp-server" "name" "logs" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index 5af6d1c6ed..9c53c9c7f1 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -1,7 +1,7 @@ {{- if .Values.clpConfig.mcp_server }} {{- include "clp.createLocalPv" (dict "root" . - "component" "mcp-server" + "component_category" "mcp-server" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml index 61219f973f..7ab19f231d 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml @@ -1,7 +1,7 @@ {{- if .Values.clpConfig.mcp_server }} {{- include "clp.createPvc" (dict "root" . - "component" "mcp-server" + "component_category" "mcp-server" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 94d79a28d3..76fc1c60fb 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -85,7 +85,7 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "query-scheduler" + "component_category" "query-scheduler" "name" "logs" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index 346b250c2c..de7633da55 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "query-scheduler" + "component_category" "query-scheduler" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml index 559d3518ff..fa56f7af7e 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "query-scheduler" + "component_category" "query-scheduler" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 5329f3b745..729f7becc8 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -75,22 +75,22 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "query-worker" + "component_category" "query-worker" "name" "logs" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "query-worker" + "component_category" "query-worker" "name" "staged-streams" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "archives" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "streams" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index 65f9ff3eed..4f602a8c74 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "query-worker" + "component_category" "query-worker" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml index eb2f41e916..84d478b40a 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "query-worker" + "component_category" "query-worker" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index 02b95378a6..157c203f65 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "query-worker" + "component_category" "query-worker" "name" "staged-streams" "nodeRole" "control-plane" "capacity" "20Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml index 242fe1ff4a..cf9bb28eaa 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "query-worker" + "component_category" "query-worker" "name" "staged-streams" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml index a24808b632..c9315630f5 100644 --- a/tools/deployment/package-helm/templates/queue-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "queue" + "component_category" "queue" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml index d5be52b6b1..56efc9d19c 100644 --- a/tools/deployment/package-helm/templates/redis-data-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "redis" + "component_category" "redis" "name" "data" "nodeRole" "control-plane" "capacity" "20Gi" diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml index 755484bacd..7f03c8cdad 100644 --- a/tools/deployment/package-helm/templates/redis-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "redis" + "component_category" "redis" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index f6ee050a35..a9af48f0be 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -66,7 +66,7 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "reducer" + "component_category" "reducer" "name" "logs" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index 1d12c20d16..0aab54233a 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "reducer" + "component_category" "reducer" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml index 78dbd64288..40a3affaea 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "reducer" + "component_category" "reducer" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml index a4a32223b5..8410734b2e 100644 --- a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "results-cache" + "component_category" "results-cache" "name" "data" "nodeRole" "control-plane" "capacity" "20Gi" diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index 9498d0ad65..e76aaf90af 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "results-cache" + "component_category" "results-cache" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml index 9373c85fcf..d35eb98dbb 100644 --- a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "archives" "nodeRole" "control-plane" "capacity" "50Gi" diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml index fb53ebd225..1cbc9c3f4d 100644 --- a/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "archives" "capacity" "50Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml index d96a7be523..bba9a8d5b5 100644 --- a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "streams" "nodeRole" "control-plane" "capacity" "20Gi" diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml index 4f4e6ea9a0..3477876d73 100644 --- a/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "streams" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 4f0415556f..627a44abed 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -87,7 +87,7 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "streams" ) | nindent 8 }} - name: "client-settings" From a0cbb98dee06b7bea818244570df0f6a0c30bd13 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 14 Dec 2025 18:58:08 -0500 Subject: [PATCH 112/155] refactor(helm): Standardize indentation and list formatting in templates for consistent readability --- .../package-helm/templates/_helpers.tpl | 40 ++++++------- .../templates/api-server-deployment.yaml | 30 +++++----- .../compression-scheduler-deployment.yaml | 60 +++++++++---------- .../compression-worker-deployment.yaml | 50 ++++++++-------- .../templates/db-table-creator-job.yaml | 2 +- .../garbage-collector-deployment.yaml | 50 ++++++++-------- .../templates/mcp-server-deployment.yaml | 30 +++++----- .../templates/query-scheduler-deployment.yaml | 46 +++++++------- .../templates/query-worker-deployment.yaml | 50 ++++++++-------- .../templates/reducer-deployment.yaml | 30 +++++----- .../results-cache-indices-creator-job.yaml | 10 ++-- .../templates/webui-deployment.yaml | 30 +++++----- 12 files changed, 214 insertions(+), 214 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 90c444f565..0c5832bcb2 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -174,9 +174,9 @@ Creates a volume definition that references a PersistentVolumeClaim. @return {string} YAML-formatted volume definition */}} {{- define "clp.pvcVolume" -}} -- name: {{ printf "%s-%s" .component_category .name | quote }} - persistentVolumeClaim: - claimName: {{ include "clp.fullname" .root }}-{{ .component_category }}-{{ .name }} +name: {{ printf "%s-%s" .component_category .name | quote }} +persistentVolumeClaim: + claimName: {{ include "clp.fullname" .root }}-{{ .component_category }}-{{ .name }} {{- end }} {{/* @@ -189,8 +189,8 @@ Creates the BROKER_URL env var for Celery workers. {{- $user := .Values.credentials.queue.username -}} {{- $pass := .Values.credentials.queue.password -}} {{- $host := printf "%s-queue" (include "clp.fullname" .) -}} -- name: "BROKER_URL" - value: {{ printf "amqp://%s:%s@%s:5672" $user $pass $host | quote }} +name: "BROKER_URL" +value: {{ printf "amqp://%s:%s@%s:5672" $user $pass $host | quote }} {{- end }} {{/* @@ -203,8 +203,8 @@ Creates the RESULT_BACKEND env var for Celery workers. {{- define "clp.celeryResultBackendEnvVar" -}} {{- $pass := .root.Values.credentials.redis.password -}} {{- $host := printf "%s-redis" (include "clp.fullname" .root) -}} -- name: "RESULT_BACKEND" - value: {{ printf "redis://default:%s@%s:6379/%d" $pass $host (int .database) | quote }} +name: "RESULT_BACKEND" +value: {{ printf "redis://default:%s@%s:6379/%d" $pass $host (int .database) | quote }} {{- end }} {{/* @@ -217,17 +217,17 @@ Creates an initContainer that waits for a Kubernetes resource to be ready. @return {string} YAML-formatted initContainer definition */}} {{- define "clp.waitFor" -}} -- name: "wait-for-{{ .name }}" - image: "bitnami/kubectl:latest" - command: [ - "kubectl", "wait", - {{- if eq .type "service" }} - "--for=condition=ready", - "pod", "--selector", "app.kubernetes.io/component={{ .name }}", - {{- else if eq .type "job" }} - "--for=condition=complete", - "job/{{ include "clp.fullname" .root }}-{{ .name }}", - {{- end }} - "--timeout=300s" - ] +name: "wait-for-{{ .name }}" +image: "bitnami/kubectl:latest" +command: [ + "kubectl", "wait", + {{- if eq .type "service" }} + "--for=condition=ready", + "pod", "--selector", "app.kubernetes.io/component={{ .name }}", + {{- else if eq .type "job" }} + "--for=condition=complete", + "job/{{ include "clp.fullname" .root }}-{{ .name }}", + {{- end }} + "--timeout=300s" +] {{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 8756623df3..f40eec24f4 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -24,16 +24,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "db-table-creator" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "results-cache-indices-creator" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 10 }} containers: - name: "api-server" image: "{{ include "clp.image.ref" . }}" @@ -78,11 +78,11 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} httpGet: *api-server-health-check volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "api-server" - "name" "logs" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "api-server" + "name" "logs" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index fee1ba1d85..bfff811db9 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -24,31 +24,31 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "db-table-creator" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "service" - "name" "queue" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "service" - "name" "redis" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "queue" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "redis" + ) | nindent 10 }} containers: - name: "compression-scheduler" image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} - {{- include "clp.celeryResultBackendEnvVar" (dict - "root" . - "database" .Values.clpConfig.redis.compression_backend_database - ) | nindent 12 }} + - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 14 }} + - {{- include "clp.celeryResultBackendEnvVar" (dict + "root" . + "database" .Values.clpConfig.redis.compression_backend_database + ) | nindent 14 }} - name: "CLP_DB_PASS" valueFrom: secretKeyRef: @@ -83,16 +83,16 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "compression-scheduler" - "name" "logs" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "compression-scheduler" - "name" "user-logs" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-scheduler" + "name" "logs" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-scheduler" + "name" "user-logs" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 8a4e31a972..f8f8d1b5cd 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -24,18 +24,18 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} + - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 10 }} + - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 10 }} containers: - name: "compression-worker" image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} - {{- include "clp.celeryResultBackendEnvVar" (dict + - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 14 }} + - {{- include "clp.celeryResultBackendEnvVar" (dict "root" . "database" .Values.clpConfig.redis.compression_backend_database - ) | nindent 12 }} + ) | nindent 14 }} - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yaml" - name: "CLP_HOME" @@ -76,26 +76,26 @@ spec: "-n", "compression-worker" ] volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "compression-worker" - "name" "logs" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "compression-worker" - "name" "staged-archives" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "compression-worker" - "name" "tmp" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "archives" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-worker" + "name" "logs" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-worker" + "name" "staged-archives" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-worker" + "name" "tmp" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "archives" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index 8772c56fd8..f5e5d0698d 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -20,7 +20,7 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "database") | nindent 8 }} + - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "database") | nindent 10 }} containers: - name: "db-table-creator" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 1cac183243..301bec93f7 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -24,16 +24,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "db-table-creator" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "results-cache-indices-creator" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 10 }} containers: - name: "garbage-collector" image: "{{ include "clp.image.ref" . }}" @@ -74,21 +74,21 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "garbage-collector" - "name" "logs" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "archives" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "streams" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "garbage-collector" + "name" "logs" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "archives" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "streams" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 5f5a7d939d..c66ab63747 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -25,16 +25,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "db-table-creator" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "results-cache-indices-creator" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 10 }} containers: - name: "mcp-server" image: "{{ include "clp.image.ref" . }}" @@ -82,11 +82,11 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} httpGet: *mcp-server-health-check volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "mcp-server" - "name" "logs" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "mcp-server" + "name" "logs" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 76fc1c60fb..ba31a85aa6 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -26,31 +26,31 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "db-table-creator" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "service" - "name" "queue" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "service" - "name" "redis" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "queue" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "redis" + ) | nindent 10 }} containers: - name: "query-scheduler" image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} - {{- include "clp.celeryResultBackendEnvVar" (dict + - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 14 }} + - {{- include "clp.celeryResultBackendEnvVar" (dict "root" . "database" .Values.clpConfig.redis.query_backend_database - ) | nindent 12 }} + ) | nindent 14 }} - name: "CLP_DB_PASS" valueFrom: secretKeyRef: @@ -83,11 +83,11 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "query-scheduler" - "name" "logs" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "query-scheduler" + "name" "logs" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 729f7becc8..93a58bbca2 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -24,18 +24,18 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} + - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 10 }} + - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 10 }} containers: - name: "query-worker" image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} - {{- include "clp.celeryResultBackendEnvVar" (dict + - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 14 }} + - {{- include "clp.celeryResultBackendEnvVar" (dict "root" . "database" .Values.clpConfig.redis.query_backend_database - ) | nindent 12 }} + ) | nindent 14 }} - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yaml" - name: "CLP_HOME" @@ -73,26 +73,26 @@ spec: "-n", "query-worker" ] volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "query-worker" - "name" "logs" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "query-worker" - "name" "staged-streams" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "archives" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "streams" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "query-worker" + "name" "logs" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "query-worker" + "name" "staged-streams" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "archives" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "streams" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index a9af48f0be..97fc4d55df 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -26,16 +26,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "service" - "name" "query-scheduler" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "results-cache-indices-creator" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "query-scheduler" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 10 }} containers: - name: "reducer" image: "{{ include "clp.image.ref" . }}" @@ -64,11 +64,11 @@ spec: "--upsert-interval", "{{ .Values.clpConfig.reducer.upsert_interval }}" ] volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "reducer" - "name" "logs" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "reducer" + "name" "logs" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index 0dcf9ee646..170707562c 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -20,11 +20,11 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "service" - "name" "results-cache" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "results-cache" + ) | nindent 10 }} containers: - name: "results-cache-indices-creator" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 627a44abed..86198f55df 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -24,16 +24,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "db-table-creator" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "results-cache-indices-creator" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 10 }} containers: - name: "webui" image: "{{ include "clp.image.ref" . }}" @@ -85,11 +85,11 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: *webui-health-check volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "streams" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "streams" + ) | nindent 10 }} - name: "client-settings" configMap: name: {{ include "clp.fullname" . }}-config From 8beb8e073985245ffe65776f96c664bea961ac97 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 11:51:31 -0500 Subject: [PATCH 113/155] feat(helm): Add S3 support for storage configurations and AWS authentication options. --- .../templates/api-server-deployment.yaml | 20 +++++ .../api-server-staged-streams-pv.yaml | 11 +++ .../api-server-staged-streams-pvc.yaml | 9 +++ .../compression-scheduler-deployment.yaml | 15 ++++ .../compression-worker-deployment.yaml | 19 +++++ .../package-helm/templates/configmap.yaml | 77 +++++++++++++++++++ .../garbage-collector-deployment.yaml | 19 +++++ .../templates/query-worker-deployment.yaml | 20 +++++ .../templates/webui-deployment.yaml | 22 ++++++ 9 files changed, 212 insertions(+) create mode 100644 tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml create mode 100644 tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index f40eec24f4..e54d126f51 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -63,6 +63,13 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + - name: "api-server-staged-streams" + mountPath: "/var/data/staged-streams" + {{- else }} + - name: "shared-data-streams" + mountPath: "/var/data/streams" + {{- end }} command: [ "/opt/clp/bin/api_server", "--host", "0.0.0.0", @@ -83,6 +90,19 @@ spec: "component_category" "api-server" "name" "logs" ) | nindent 10 }} + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "api-server" + "name" "staged-streams" + ) | nindent 10 }} + {{- else }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "streams" + ) | nindent 10 }} + {{- end }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml new file mode 100644 index 0000000000..6187dd822a --- /dev/null +++ b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml @@ -0,0 +1,11 @@ +{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} +{{- include "clp.createLocalPv" (dict + "root" . + "component_category" "api-server" + "name" "staged-streams" + "nodeRole" "control-plane" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml new file mode 100644 index 0000000000..79ef5617c8 --- /dev/null +++ b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml @@ -0,0 +1,9 @@ +{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} +{{- include "clp.createPvc" (dict + "root" . + "component_category" "api-server" + "name" "staged-streams" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index bfff811db9..91d76adbd9 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -74,9 +74,16 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - name: "logs-input" mountPath: "/mnt/logs" readOnly: true + {{- end }} command: [ "python3", "-u", "-m", "job_orchestration.scheduler.compress.compression_scheduler", @@ -96,7 +103,15 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: "{{ .Values.clpConfig.aws_config_directory }}" + type: "Directory" + {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" type: "Directory" + {{- end }} diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index f8f8d1b5cd..508f0388ae 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -59,11 +59,20 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - name: "logs-input" mountPath: "/mnt/logs" readOnly: true + {{- end }} + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: "shared-data-archives" mountPath: "/var/data/archives" + {{- end }} command: [ "python3", "-u", "/opt/clp/lib/python3/site-packages/bin/celery", @@ -91,15 +100,25 @@ spec: "component_category" "compression-worker" "name" "tmp" ) | nindent 10 }} + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "archives" ) | nindent 10 }} + {{- end }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" type: "Directory" + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: "{{ .Values.clpConfig.aws_config_directory }}" + type: "Directory" + {{- end }} diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index d595a36acd..27276e9669 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -22,8 +22,28 @@ data: compression_level: {{ .Values.clpConfig.archive_output.compression_level }} retention_period: {{ .Values.clpConfig.archive_output.retention_period }} storage: + {{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} + type: "s3" + staging_directory: "/var/data/staged-archives" + s3_config: + endpoint_url: {{ .Values.clpConfig.archive_output.storage.s3_config.endpoint_url | default "null" }} + region_code: {{ .Values.clpConfig.archive_output.storage.s3_config.region_code | quote }} + bucket: {{ .Values.clpConfig.archive_output.storage.s3_config.bucket | quote }} + key_prefix: {{ .Values.clpConfig.archive_output.storage.s3_config.key_prefix | quote }} + aws_authentication: + type: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.type | quote }} + {{- if eq .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.type "credentials" }} + credentials: + access_key_id: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} + secret_access_key: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + {{- end }} + {{- if eq .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.type "profile" }} + profile: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.profile | quote }} + {{- end }} + {{- else }} directory: "/var/data/archives" type: "fs" + {{- end }} target_archive_size: {{ .Values.clpConfig.archive_output.target_archive_size | int }} target_dictionaries_size: {{ .Values.clpConfig.archive_output.target_dictionaries_size | int }} @@ -55,8 +75,22 @@ data: search_result: {{ .Values.clpConfig.garbage_collector.sweep_interval.search_result }} logs_directory: "/var/log" logs_input: + {{- if eq .Values.clpConfig.logs_input.type "s3" }} + type: "s3" + aws_authentication: + type: {{ .Values.clpConfig.logs_input.aws_authentication.type | quote }} + {{- if eq .Values.clpConfig.logs_input.aws_authentication.type "credentials" }} + credentials: + access_key_id: {{ .Values.clpConfig.logs_input.aws_authentication.credentials.access_key_id | quote }} + secret_access_key: {{ .Values.clpConfig.logs_input.aws_authentication.credentials.secret_access_key | quote }} + {{- end }} + {{- if eq .Values.clpConfig.logs_input.aws_authentication.type "profile" }} + profile: {{ .Values.clpConfig.logs_input.aws_authentication.profile | quote }} + {{- end }} + {{- else }} directory: "/mnt/logs" type: "fs" + {{- end }} mcp_server: {{ .Values.clpConfig.mcp_server }} package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} @@ -92,8 +126,28 @@ data: stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} stream_output: storage: + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + type: "s3" + staging_directory: "/var/data/staged-streams" + s3_config: + endpoint_url: {{ .Values.clpConfig.stream_output.storage.s3_config.endpoint_url | default "null" }} + region_code: {{ .Values.clpConfig.stream_output.storage.s3_config.region_code | quote }} + bucket: {{ .Values.clpConfig.stream_output.storage.s3_config.bucket | quote }} + key_prefix: {{ .Values.clpConfig.stream_output.storage.s3_config.key_prefix | quote }} + aws_authentication: + type: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type | quote }} + {{- if eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "credentials" }} + credentials: + access_key_id: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} + secret_access_key: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "profile" }} + profile: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.profile | quote }} + {{- end }} + {{- else }} directory: "/var/data/streams" type: "fs" + {{- end }} target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }} tmp_directory: "/var/tmp" webui: @@ -136,6 +190,11 @@ data: "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, "LogsInputType": {{ .Values.clpConfig.logs_input.type | quote }}, + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + "LogsInputRootDir": "/mnt/logs", + {{- else }} + "LogsInputRootDir": null, + {{- end }} "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, "SqlDbClpArchivesTableName": "clp_archives", @@ -160,12 +219,30 @@ data: "MongoDbStreamFilesCollectionName": "stream-files", "ClientDir": "/opt/clp/var/www/webui/client", "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer", + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + "LogsInputRootDir": "/mnt/logs", + {{- else }} + "LogsInputRootDir": null, + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + "StreamFilesDir": null, + "StreamTargetUncompressedSize": + {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }}, + "StreamFilesS3Region": {{ .Values.clpConfig.stream_output.storage.s3_config.region_code | quote }}, + "StreamFilesS3PathPrefix": "{{ .Values.clpConfig.stream_output.storage.s3_config.bucket }}/{{ .Values.clpConfig.stream_output.storage.s3_config.key_prefix }}", + {{- if eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "profile" }} + "StreamFilesS3Profile": {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.profile | quote }}, + {{- else }} + "StreamFilesS3Profile": null, + {{- end }} + {{- else }} "StreamFilesDir": "/var/data/streams", "StreamTargetUncompressedSize": {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }}, "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, + {{- end }} "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level }}, "ArchiveOutputTargetArchiveSize": {{ .Values.clpConfig.archive_output.target_archive_size | int }}, diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 301bec93f7..08f5b7bab3 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -64,10 +64,19 @@ spec: readOnly: true - name: "garbage-collector-logs" mountPath: "/var/log/garbage_collector" + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: "shared-data-archives" mountPath: "/var/data/archives" + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: "shared-data-streams" mountPath: "/var/data/streams" + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} command: [ "python3", "-u", "-m", "job_orchestration.garbage_collector.garbage_collector", @@ -79,16 +88,26 @@ spec: "component_category" "garbage-collector" "name" "logs" ) | nindent 10 }} + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "archives" ) | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "streams" ) | nindent 10 }} + {{- end }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: "{{ .Values.clpConfig.aws_config_directory }}" + type: "Directory" + {{- end }} diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 93a58bbca2..4d54971e75 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -57,10 +57,20 @@ spec: mountPath: "/var/log/query_worker" - name: "query-worker-staged-streams" mountPath: "/var/data/staged-streams" + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: "shared-data-archives" mountPath: "/var/data/archives" + readOnly: true + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: "shared-data-streams" mountPath: "/var/data/streams" + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} command: [ "python3", "-u", "/opt/clp/lib/python3/site-packages/bin/celery", @@ -83,16 +93,26 @@ spec: "component_category" "query-worker" "name" "staged-streams" ) | nindent 10 }} + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "archives" ) | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "streams" ) | nindent 10 }} + {{- end }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: "{{ .Values.clpConfig.aws_config_directory }}" + type: "Directory" + {{- end }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 86198f55df..acc2ac51b7 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -39,6 +39,12 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: + {{- if and (eq .Values.clpConfig.stream_output.storage.type "s3") (eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "credentials") }} + - name: "AWS_ACCESS_KEY_ID" + value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} + - name: "AWS_SECRET_ACCESS_KEY" + value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + {{- end }} - name: "CLP_DB_USER" valueFrom: secretKeyRef: @@ -71,8 +77,16 @@ spec: mountPath: "/opt/clp/var/www/webui/server/dist/settings.json" subPath: "webui-server-settings.json" readOnly: true + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: "shared-data-streams" mountPath: "/var/data/streams" + readOnly: true + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} command: [ "/opt/clp/bin/node-22", "/opt/clp/var/www/webui/server/dist/src/main.js" @@ -85,14 +99,22 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: *webui-health-check volumes: + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "streams" ) | nindent 10 }} + {{- end }} - name: "client-settings" configMap: name: {{ include "clp.fullname" . }}-config - name: "server-settings" configMap: name: {{ include "clp.fullname" . }}-config + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: "{{ .Values.clpConfig.aws_config_directory }}" + type: "Directory" + {{- end }} From 41f0348265c187d0360644ac3b8d624dea83044e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:15:55 -0500 Subject: [PATCH 114/155] refactor(helm): Reorganize volumeMounts and environment variables for clarity and alignment across templates --- .../templates/api-server-deployment.yaml | 25 +++++++++----- .../compression-scheduler-deployment.yaml | 6 ++-- .../compression-worker-deployment.yaml | 27 ++++++++------- .../garbage-collector-deployment.yaml | 33 ++++++++++++------- .../templates/mcp-server-deployment.yaml | 13 +++++--- .../templates/webui-deployment.yaml | 24 +++++++------- 6 files changed, 76 insertions(+), 52 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index e54d126f51..2b46eb5e66 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -39,16 +39,16 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "CLP_DB_USER" + - name: "CLP_DB_PASS" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "username" - - name: "CLP_DB_PASS" + key: "password" + - name: "CLP_DB_USER" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "username" - name: "CLP_LOGS_DIR" value: "/var/log/api_server" - name: "RUST_LOG" @@ -57,17 +57,26 @@ spec: - name: "api-server" containerPort: 3001 volumeMounts: - - name: "api-server-logs" - mountPath: "/var/log/api_server" - name: "config" mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true + - name: {{ include "clp.volumeName" (dict + "component_category" "api-server" + "name" "logs" + ) | quote }} + mountPath: "/var/log/api_server" {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - - name: "api-server-staged-streams" + - name: {{ include "clp.volumeName" (dict + "component_category" "api-server" + "name" "staged-streams" + ) | quote }} mountPath: "/var/data/staged-streams" {{- else }} - - name: "shared-data-streams" + - name: {{ include "clp.volumeName" (dict + "component_category" "shared-data" + "name" "streams" + ) | quote }} mountPath: "/var/data/streams" {{- end }} command: [ diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 06bc69c31f..97f9c484b8 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -66,9 +66,6 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - {{- if eq .Values.clpConfig.logs_input.type "fs" }} - - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} - {{- end }} - name: "config" mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" @@ -88,6 +85,9 @@ spec: mountPath: "/opt/clp/.aws" readOnly: true {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} + {{- end }} command: [ "python3", "-u", "-m", "job_orchestration.scheduler.compress.compression_scheduler", diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index aa3fdd0174..b48eebf08b 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -45,23 +45,30 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - {{- if eq .Values.clpConfig.logs_input.type "fs" }} - - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} - {{- end }} + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "compression-worker" "name" "logs" ) | quote }} mountPath: "/var/log/compression_worker" + - name: {{ include "clp.volumeName" (dict + "component_category" "compression-worker" + "name" "staged-archives" + ) | quote }} + mountPath: "/var/data/staged-archives" - name: {{ include "clp.volumeName" (dict "component_category" "compression-worker" "name" "tmp" ) | quote }} mountPath: "/var/tmp" - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" readOnly: true + {{- end }} {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -69,12 +76,8 @@ spec: ) | quote }} mountPath: "/var/data/archives" {{- end }} - - name: "compression-worker-staged-archives" - mountPath: "/var/data/staged-archives" - {{- if .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - mountPath: "/opt/clp/.aws" - readOnly: true + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} {{- end }} command: [ "python3", "-u", diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 08f5b7bab3..ab2a3e770e 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -39,16 +39,16 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "CLP_DB_USER" + - name: "CLP_DB_PASS" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "username" - - name: "CLP_DB_PASS" + key: "password" + - name: "CLP_DB_USER" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "username" - name: "CLP_HOME" value: "/opt/clp" - name: "CLP_LOGGING_LEVEL" @@ -62,21 +62,30 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true - - name: "garbage-collector-logs" + - name: {{ include "clp.volumeName" (dict + "component_category" "garbage-collector" + "name" "logs" + ) | quote }} mountPath: "/var/log/garbage_collector" + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - - name: "shared-data-archives" + - name: {{ include "clp.volumeName" (dict + "component_category" "shared-data" + "name" "archives" + ) | quote }} mountPath: "/var/data/archives" {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - - name: "shared-data-streams" + - name: {{ include "clp.volumeName" (dict + "component_category" "shared-data" + "name" "streams" + ) | quote }} mountPath: "/var/data/streams" {{- end }} - {{- if .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - mountPath: "/opt/clp/.aws" - readOnly: true - {{- end }} command: [ "python3", "-u", "-m", "job_orchestration.garbage_collector.garbage_collector", diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index c66ab63747..1238602ba3 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -40,16 +40,16 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "CLP_DB_USER" + - name: "CLP_DB_PASS" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "username" - - name: "CLP_DB_PASS" + key: "password" + - name: "CLP_DB_USER" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "username" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.mcp_server.logging_level | quote }} - name: "CLP_LOGS_DIR" @@ -64,7 +64,10 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true - - name: "mcp-server-logs" + - name: {{ include "clp.volumeName" (dict + "component_category" "mcp-server" + "name" "logs" + ) | quote }} mountPath: "/var/log/mcp_server" command: [ "python3", "-u", diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 5a1a6d50b6..b418874783 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -39,12 +39,6 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - {{- if and (eq .Values.clpConfig.stream_output.storage.type "s3") (eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "credentials") }} - - name: "AWS_ACCESS_KEY_ID" - value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} - - name: "AWS_SECRET_ACCESS_KEY" - value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} - {{- end }} - name: "CLP_DB_PASS" valueFrom: secretKeyRef: @@ -65,11 +59,16 @@ spec: value: "4000" - name: "RATE_LIMIT" value: {{ .Values.clpConfig.webui.rate_limit | quote }} + {{- if and (eq .Values.clpConfig.stream_output.storage.type "s3") (eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "credentials") }} + - name: "AWS_ACCESS_KEY_ID" + value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} + - name: "AWS_SECRET_ACCESS_KEY" + value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + {{- end }} ports: - name: "webui" containerPort: 4000 volumeMounts: - - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} - name: "client-settings" mountPath: "/opt/clp/var/www/webui/client/settings.json" subPath: "webui-client-settings.json" @@ -78,6 +77,12 @@ spec: mountPath: "/opt/clp/var/www/webui/server/dist/settings.json" subPath: "webui-server-settings.json" readOnly: true + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} + {{- include "clp.logsInputVolumeMount" . | nindent 12 }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -85,11 +90,6 @@ spec: ) | quote }} mountPath: "/var/data/streams" {{- end }} - {{- if .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - mountPath: "/opt/clp/.aws" - readOnly: true - {{- end }} command: [ "/opt/clp/bin/node-22", "/opt/clp/var/www/webui/server/dist/src/main.js" From 3363229cf2c22170d24ba8ac5797d2f17272f2f1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:37:50 -0500 Subject: [PATCH 115/155] feat(helm): Add log_ingestor configuration for S3 log input support --- .../package-helm/templates/configmap.yaml | 9 +++++++++ tools/deployment/package-helm/values.yaml | 13 +++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index d1f3b7a52d..b54e31a4a8 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -91,6 +91,15 @@ data: type: "fs" directory: "/mnt/logs" {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "s3" }} + log_ingestor: + buffer_flush_threshold: {{ .Values.clpConfig.log_ingestor.buffer_flush_threshold }} + buffer_flush_timeout: {{ .Values.clpConfig.log_ingestor.buffer_flush_timeout }} + channel_capacity: {{ .Values.clpConfig.log_ingestor.channel_capacity }} + host: "localhost" + logging_level: {{ .Values.clpConfig.log_ingestor.logging_level | quote }} + port: 3002 + {{- end }} package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index ae1029ff51..3320175ebd 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -139,6 +139,19 @@ clpConfig: initial_backoff_ms: 100 max_backoff_ms: 5000 + # log-ingestor config. Currently, the config is applicable only if `logs_input.type` is "s3". + log_ingestor: + port: 30302 + # The timeout (in seconds) after which the log buffer is flushed for compression if no new + # input arrives. + buffer_flush_timeout: 300 + # The log buffer size (in bytes) that triggers a flush for compression. + buffer_flush_threshold: 268435456 # 256 MiB + # The capacity of the internal channel used for communication between an ingestion job and the + # log buffer. + channel_capacity: 10 + logging_level: "INFO" + # Presto client config presto: null # port: 30889 From 17e2fc235d488d6dc3ea17b0b1bfc3ebb539ca28 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:39:17 -0500 Subject: [PATCH 116/155] feat(helm): Add `type` configuration to compression_scheduler for task execution mode selection --- tools/deployment/package-helm/templates/configmap.yaml | 1 + tools/deployment/package-helm/values.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index b54e31a4a8..f2aee9a1d0 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -57,6 +57,7 @@ data: logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} max_concurrent_tasks_per_job: {{ .Values.clpConfig.compression_scheduler.max_concurrent_tasks_per_job }} + type: {{ .Values.clpConfig.compression_scheduler.type | quote }} compression_worker: logging_level: {{ .Values.clpConfig.compression_worker.logging_level | quote }} data_directory: "/var/data" diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 3320175ebd..a65c38df6a 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -45,6 +45,7 @@ clpConfig: jobs_poll_delay: 0.1 # seconds logging_level: "INFO" max_concurrent_tasks_per_job: 0 # A value of 0 disables the limit + type: "celery" # "celery" or "spider" query_scheduler: port: 7000 From 2300cc5689b26fc72c656c547be8032e4b0b06f4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:41:53 -0500 Subject: [PATCH 117/155] feat(helm): Add log-ingestor deployment, service, and volume configurations for S3 log input handling --- .../templates/log-ingestor-deployment.yaml | 88 +++++++++++++++++++ .../templates/log-ingestor-logs-pv.yaml | 11 +++ .../templates/log-ingestor-logs-pvc.yaml | 9 ++ .../templates/log-ingestor-service.yaml | 18 ++++ 4 files changed, 126 insertions(+) create mode 100644 tools/deployment/package-helm/templates/log-ingestor-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/log-ingestor-logs-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/log-ingestor-service.yaml diff --git a/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml b/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml new file mode 100644 index 0000000000..4c4946a892 --- /dev/null +++ b/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml @@ -0,0 +1,88 @@ +{{- if eq .Values.clpConfig.logs_input.type "s3" }} +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp.fullname" . }}-log-ingestor + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "log-ingestor" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "log-ingestor" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "log-ingestor" + spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher + terminationGracePeriodSeconds: 60 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + containers: + - name: "log-ingestor" + image: "{{ include "clp.image.ref" . }}" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" + env: + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "password" + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "username" + - name: "CLP_LOGS_DIR" + value: "/var/log/log_ingestor" + - name: "RUST_LOG" + value: {{ .Values.clpConfig.log_ingestor.logging_level | quote }} + ports: + - name: "log-ingestor" + containerPort: 3002 + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true + - name: {{ include "clp.volumeName" (dict + "component_category" "log-ingestor" + "name" "logs" + ) | quote }} + mountPath: "/var/log/log_ingestor" + command: [ + "/opt/clp/bin/log-ingestor", + "--config", "/etc/clp-config.yaml", + "--host", "0.0.0.0", + "--port", "3002" + ] + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + httpGet: &log-ingestor-health-check + path: "/health" + port: 3002 + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + httpGet: *log-ingestor-health-check + volumes: + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "log-ingestor" + "name" "logs" + ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config +{{- end }} diff --git a/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml b/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml new file mode 100644 index 0000000000..1abb5d03c4 --- /dev/null +++ b/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml @@ -0,0 +1,11 @@ +{{- if eq .Values.clpConfig.logs_input.type "s3" }} +{{- include "clp.createLocalPv" (dict + "root" . + "component_category" "log-ingestor" + "name" "logs" + "nodeRole" "control-plane" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/log_ingestor" .Values.clpConfig.logs_directory) +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/log-ingestor-logs-pvc.yaml b/tools/deployment/package-helm/templates/log-ingestor-logs-pvc.yaml new file mode 100644 index 0000000000..62fc3c9740 --- /dev/null +++ b/tools/deployment/package-helm/templates/log-ingestor-logs-pvc.yaml @@ -0,0 +1,9 @@ +{{- if eq .Values.clpConfig.logs_input.type "s3" }} +{{- include "clp.createPvc" (dict + "root" . + "component_category" "log-ingestor" + "name" "logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/log-ingestor-service.yaml b/tools/deployment/package-helm/templates/log-ingestor-service.yaml new file mode 100644 index 0000000000..f79a324e61 --- /dev/null +++ b/tools/deployment/package-helm/templates/log-ingestor-service.yaml @@ -0,0 +1,18 @@ +{{- if eq .Values.clpConfig.logs_input.type "s3" }} +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-log-ingestor + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "log-ingestor" +spec: + type: "NodePort" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "log-ingestor" + ports: + - port: 3002 + targetPort: "log-ingestor" + nodePort: {{ .Values.clpConfig.log_ingestor.port }} +{{- end }} From 0e34c789f17f628b27e7b4a72e96b668728d9908 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:42:04 -0500 Subject: [PATCH 118/155] refactor(helm): Replace `database.name` with `database.names.clp` --- tools/deployment/package-helm/templates/configmap.yaml | 5 +++-- .../package-helm/templates/database-statefulset.yaml | 2 +- tools/deployment/package-helm/values.yaml | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index f2aee9a1d0..55a8652f5b 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -65,7 +65,8 @@ data: auto_commit: false compress: true host: {{ include "clp.fullname" . }}-database - name: {{ .Values.clpConfig.database.name | quote }} + names: + clp: {{ .Values.clpConfig.database.names.clp | quote }} port: 3306 ssl_cert: null type: {{ .Values.clpConfig.database.type | quote }} @@ -217,7 +218,7 @@ data: { "SqlDbHost": "{{ include "clp.fullname" . }}-database", "SqlDbPort": 3306, - "SqlDbName": {{ .Values.clpConfig.database.name | quote }}, + "SqlDbName": {{ .Values.clpConfig.database.names.clp | quote }}, "SqlDbQueryJobsTableName": "query_jobs", "SqlDbCompressionJobsTableName": "compression_jobs", "MongoDbHost": "{{ include "clp.fullname" . }}-results-cache", diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 8e67cee934..4c510c57c8 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -33,7 +33,7 @@ spec: imagePullPolicy: "Always" env: - name: "MYSQL_DATABASE" - value: {{ .Values.clpConfig.database.name | quote }} + value: {{ .Values.clpConfig.database.names.clp | quote }} - name: "MYSQL_USER" valueFrom: secretKeyRef: diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index a65c38df6a..a974e5ee56 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -39,7 +39,8 @@ clpConfig: database: type: "mariadb" # "mariadb" or "mysql" port: 30306 - name: "clp-db" + names: + clp: "clp-db" compression_scheduler: jobs_poll_delay: 0.1 # seconds From 50906eadc4bf6c533fd3f5449cca744c88f3c6d8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:45:28 -0500 Subject: [PATCH 119/155] fix(helm): Update S3 profile condition to include storage type validation --- tools/deployment/package-helm/templates/configmap.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 55a8652f5b..9099c55d4e 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -244,7 +244,8 @@ data: "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, {{- end }} - {{- if eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "profile" }} + {{- if and (eq .Values.clpConfig.stream_output.storage.type "s3") + (eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "profile") }} "StreamFilesS3Profile": {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.profile | quote }}, {{- else }} "StreamFilesS3Profile": null, From 748c176532fc084ebc8a9f0b6f35e63953f0bc0b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:47:26 -0500 Subject: [PATCH 120/155] feat(helm): Add mcp_server configuration support in configmap template --- tools/deployment/package-helm/templates/configmap.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 9099c55d4e..b533112967 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -102,6 +102,14 @@ data: logging_level: {{ .Values.clpConfig.log_ingestor.logging_level | quote }} port: 3002 {{- end }} + {{- if .Values.clpConfig.mcp_server }} + mcp_server: + host: "localhost" + logging_level: {{ .Values.clpConfig.mcp_server.logging_level | quote }} + port: 8000 + {{- else }} + mcp_server: null + {{- end }} package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} From b807445f50ce24d9366eaa1054efa732437afde7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:55:20 -0500 Subject: [PATCH 121/155] chore(helm): Update api-server RUST_LOG environment variable to use "INFO" logging level --- .../package-helm/templates/api-server-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 2b46eb5e66..a1850c0dec 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -52,7 +52,7 @@ spec: - name: "CLP_LOGS_DIR" value: "/var/log/api_server" - name: "RUST_LOG" - value: "DEBUG" + value: "INFO" ports: - name: "api-server" containerPort: 3001 From 25c90036f1d9fea3781c739a75e63c6c0df471fb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:56:06 -0500 Subject: [PATCH 122/155] feat(helm): Add readiness and liveness probes to query-scheduler deployment --- .../templates/query-scheduler-deployment.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index ca7149dd75..b12b6fe5dc 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -85,6 +85,14 @@ spec: "-m", "job_orchestration.scheduler.query.query_scheduler", "--config", "/etc/clp-config.yaml" ] + readinessProbe: + tcpSocket: + port: 7000 + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + livenessProbe: + tcpSocket: + port: 7000 + {{- include "clp.livenessProbeTimings" . | nindent 12 }} volumes: - {{- include "clp.pvcVolume" (dict "root" . From e3a34e92bd02d2971cc048055b5ed779a657c731 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:57:58 -0500 Subject: [PATCH 123/155] fix(helm): Correct indentation for logsInputVolumeMount in webui-deployment template --- tools/deployment/package-helm/templates/webui-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index b418874783..eb2fa25c89 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -82,7 +82,7 @@ spec: mountPath: "/opt/clp/.aws" readOnly: true {{- end }} - {{- include "clp.logsInputVolumeMount" . | nindent 12 }} + - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" From dc878b9dc699f50733ca46d116cafeead53766d4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:59:02 -0500 Subject: [PATCH 124/155] feat(helm): Add S3 storage type support for query-worker staged streams configuration --- .../package-helm/templates/query-worker-deployment.yaml | 9 +++++++++ .../templates/query-worker-staged-streams-pv.yaml | 2 ++ .../templates/query-worker-staged-streams-pvc.yaml | 2 ++ 3 files changed, 13 insertions(+) diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 5c3491aac5..fcbb976519 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -68,6 +68,13 @@ spec: ) | quote }} mountPath: "/var/data/streams" {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + - name: {{ include "clp.volumeName" (dict + "component_category" "query-worker" + "name" "staged-streams" + ) | quote }} + mountPath: "/var/data/staged-streams" + {{- end }} {{- if .Values.clpConfig.aws_config_directory }} - name: "aws-config" mountPath: "/opt/clp/.aws" @@ -90,11 +97,13 @@ spec: "component_category" "query-worker" "name" "logs" ) | nindent 10 }} + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "query-worker" "name" "staged-streams" ) | nindent 10 }} + {{- end }} {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index 157c203f65..e443b1ace9 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "query-worker" @@ -7,3 +8,4 @@ "accessModes" (list "ReadWriteMany") "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml index cf9bb28eaa..c433a8b2ea 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} {{- include "clp.createPvc" (dict "root" . "component_category" "query-worker" @@ -5,3 +6,4 @@ "capacity" "20Gi" "accessModes" (list "ReadWriteMany") ) }} +{{- end }} From bdcfb5e89ad752a46cfcf5cdd94682d875006abe Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 20:03:05 -0500 Subject: [PATCH 125/155] feat(helm): Add Presto configuration support in values.yaml and configmap template --- tools/deployment/package-helm/templates/configmap.yaml | 5 +++++ tools/deployment/package-helm/values.yaml | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index b533112967..747f96df08 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -271,6 +271,11 @@ data: {{ .Values.clpConfig.archive_output.target_segment_size | int }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, + {{- if .Values.clpConfig.presto }} + "PrestoHost": {{ .Values.clpConfig.presto.host | quote }}, + "PrestoPort": {{ .Values.clpConfig.presto.port }} + {{- else }} "PrestoHost": null, "PrestoPort": null + {{- end }} } diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index a974e5ee56..603ea06b51 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -156,7 +156,8 @@ clpConfig: # Presto client config presto: null - # port: 30889 + # host: "localhost" + # port: 8080 # Location where other data (besides archives) are stored. It will be created if # it doesn't exist. From 6ad5757949f6bb93ba6f9c5d77b9ee7b9b6c37c7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 20:05:15 -0500 Subject: [PATCH 126/155] feat(helm): Add conditional checks for api-server and garbage-collector configurations across templates --- .../package-helm/templates/api-server-deployment.yaml | 2 ++ .../deployment/package-helm/templates/api-server-logs-pv.yaml | 2 ++ .../deployment/package-helm/templates/api-server-logs-pvc.yaml | 2 ++ .../deployment/package-helm/templates/api-server-service.yaml | 2 ++ .../package-helm/templates/api-server-staged-streams-pv.yaml | 3 ++- .../package-helm/templates/api-server-staged-streams-pvc.yaml | 3 ++- .../package-helm/templates/garbage-collector-deployment.yaml | 3 +++ .../package-helm/templates/garbage-collector-logs-pv.yaml | 3 +++ .../package-helm/templates/garbage-collector-logs-pvc.yaml | 3 +++ 9 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index a1850c0dec..40286036fc 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -1,3 +1,4 @@ +{{- if .Values.clpConfig.api_server }} apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -115,3 +116,4 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config +{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index 8d51f753fc..89fd30dda8 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1,3 +1,4 @@ +{{- if .Values.clpConfig.api_server }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "api-server" @@ -7,3 +8,4 @@ "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/api_server" .Values.clpConfig.logs_directory) ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml index d290ac58cf..d9429b6dad 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml @@ -1,3 +1,4 @@ +{{- if .Values.clpConfig.api_server }} {{- include "clp.createPvc" (dict "root" . "component_category" "api-server" @@ -5,3 +6,4 @@ "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-service.yaml b/tools/deployment/package-helm/templates/api-server-service.yaml index 500b5cd6b6..0aed0e7efa 100644 --- a/tools/deployment/package-helm/templates/api-server-service.yaml +++ b/tools/deployment/package-helm/templates/api-server-service.yaml @@ -1,3 +1,4 @@ +{{- if .Values.clpConfig.api_server }} apiVersion: "v1" kind: "Service" metadata: @@ -14,3 +15,4 @@ spec: - port: 3001 targetPort: "api-server" nodePort: {{ .Values.clpConfig.api_server.port }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml index 6187dd822a..0d191046d2 100644 --- a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml @@ -1,4 +1,5 @@ -{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} +{{- if and .Values.clpConfig.api_server + (eq .Values.clpConfig.stream_output.storage.type "s3") }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "api-server" diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml index 79ef5617c8..0979cd49aa 100644 --- a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml @@ -1,4 +1,5 @@ -{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} +{{- if and .Values.clpConfig.api_server + (eq .Values.clpConfig.stream_output.storage.type "s3") }} {{- include "clp.createPvc" (dict "root" . "component_category" "api-server" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index ab2a3e770e..155a99c9f5 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -1,3 +1,5 @@ +{{- if or .Values.clpConfig.archive_output.retention_period + .Values.clpConfig.results_cache.retention_period }} apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -120,3 +122,4 @@ spec: path: "{{ .Values.clpConfig.aws_config_directory }}" type: "Directory" {{- end }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index e03597028c..47c5417a64 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,3 +1,5 @@ +{{- if or .Values.clpConfig.archive_output.retention_period + .Values.clpConfig.results_cache.retention_period }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "garbage-collector" @@ -7,3 +9,4 @@ "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/garbage_collector" .Values.clpConfig.logs_directory) ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml index 45e39243b6..af6f5a0392 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml @@ -1,3 +1,5 @@ +{{- if or .Values.clpConfig.archive_output.retention_period + .Values.clpConfig.results_cache.retention_period }} {{- include "clp.createPvc" (dict "root" . "component_category" "garbage-collector" @@ -5,3 +7,4 @@ "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} +{{- end }} From 77743e722247d0abf03faede9088b3a01431549c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 20:20:40 -0500 Subject: [PATCH 127/155] fix(helm): Remove unused `port` field from query_scheduler configuration in values.yaml --- tools/deployment/package-helm/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 603ea06b51..7a674ea5d8 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -49,7 +49,6 @@ clpConfig: type: "celery" # "celery" or "spider" query_scheduler: - port: 7000 jobs_poll_delay: 0.1 # seconds num_archives_to_search_per_sub_job: 16 logging_level: "INFO" From 4e0cd58dd4e26e824b52b66af4493c3898fb2dca Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 20:22:28 -0500 Subject: [PATCH 128/155] feat(helm): Add support for container port 30302 in query-scheduler deployment --- tools/deployment/package-helm/test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index e9f43cda77..417f533f21 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -91,6 +91,9 @@ cat < Date: Thu, 18 Dec 2025 20:25:30 -0500 Subject: [PATCH 129/155] fix(helm): Add missing log_ingestor directory to test script --- tools/deployment/package-helm/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 417f533f21..d8daaf3bec 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -59,7 +59,7 @@ mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ "$CLP_HOME/var/data/"{archives,streams} \ "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ - "$CLP_HOME/var/log/"{garbage_collector,api_server,mcp_server} \ + "$CLP_HOME/var/log/"{garbage_collector,api_server,log_ingestor,mcp_server} \ "$CLP_HOME/var/tmp" \ "$CLP_HOME/samples" From ffcd41297a11aac821e4f383787a6d406e118cb6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 20:38:34 -0500 Subject: [PATCH 130/155] fix(helm): Simplify S3 credentials configuration in webui-deployment template --- .../package-helm/templates/webui-deployment.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index eb2fa25c89..ec6f7adfda 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -59,11 +59,13 @@ spec: value: "4000" - name: "RATE_LIMIT" value: {{ .Values.clpConfig.webui.rate_limit | quote }} - {{- if and (eq .Values.clpConfig.stream_output.storage.type "s3") (eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "credentials") }} + {{- with .Values.clpConfig.stream_output.storage }} + {{- if and (eq .type "s3") (eq .s3_config.aws_authentication.type "credentials") }} - name: "AWS_ACCESS_KEY_ID" - value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} + value: {{ .s3_config.aws_authentication.credentials.access_key_id | quote }} - name: "AWS_SECRET_ACCESS_KEY" - value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + value: {{ .s3_config.aws_authentication.credentials.secret_access_key | quote }} + {{- end }} {{- end }} ports: - name: "webui" From aee79bcae2539d01dffdfd6524822725bb59c02d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:08:10 -0500 Subject: [PATCH 131/155] fix(helm): Correct placement of readiness and liveness probes in query-scheduler deployment template --- .../package-helm/templates/query-scheduler-deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index b12b6fe5dc..5ebc901a89 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -86,13 +86,13 @@ spec: "--config", "/etc/clp-config.yaml" ] readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} tcpSocket: port: 7000 - {{- include "clp.readinessProbeTimings" . | nindent 12 }} livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: port: 7000 - {{- include "clp.livenessProbeTimings" . | nindent 12 }} volumes: - {{- include "clp.pvcVolume" (dict "root" . From a9cdeddc89bd079cfde406223093f3f7a0aa1a05 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:18:09 -0500 Subject: [PATCH 132/155] fix(helm): Refactor configmap template to improve conditional handling and simplify values mapping --- .../package-helm/templates/configmap.yaml | 164 ++++++++++-------- 1 file changed, 89 insertions(+), 75 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 747f96df08..038f5c7b4f 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -6,51 +6,53 @@ metadata: {{- include "clp.labels" . | nindent 4 }} data: clp-config.yaml: | - {{- if eq .Values.clpConfig.package.storage_engine "clp-s" }} + {{- with .Values.clpConfig.api_server }} api_server: - default_max_num_query_results: {{ - .Values.clpConfig.api_server.default_max_num_query_results }} + default_max_num_query_results: {{ .default_max_num_query_results }} host: "localhost" port: 3001 query_job_polling: - initial_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.initial_backoff_ms }} - max_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.max_backoff_ms }} + initial_backoff_ms: {{ .query_job_polling.initial_backoff_ms }} + max_backoff_ms: {{ .query_job_polling.max_backoff_ms }} {{- else }} api_server: null {{- end }} + {{- with .Values.clpConfig.archive_output }} archive_output: - compression_level: {{ .Values.clpConfig.archive_output.compression_level }} - retention_period: {{ .Values.clpConfig.archive_output.retention_period }} + compression_level: {{ .compression_level }} + retention_period: {{ .retention_period }} storage: - {{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} + {{- if eq .storage.type "fs" }} + type: "fs" + directory: "/var/data/archives" + {{- else }} type: "s3" staging_directory: "/var/data/staged-archives" + {{- with .storage.s3_config }} s3_config: - endpoint_url: {{ .Values.clpConfig.archive_output.storage.s3_config.endpoint_url | default "null" }} - region_code: {{ .Values.clpConfig.archive_output.storage.s3_config.region_code | quote }} - bucket: {{ .Values.clpConfig.archive_output.storage.s3_config.bucket | quote }} - key_prefix: {{ .Values.clpConfig.archive_output.storage.s3_config.key_prefix | quote }} + endpoint_url: {{ .endpoint_url | default "null" }} + region_code: {{ .region_code | quote }} + bucket: {{ .bucket | quote }} + key_prefix: {{ .key_prefix | quote }} + {{- with .aws_authentication }} aws_authentication: - type: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.type | quote }} - {{- if eq .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.type "credentials" }} + type: {{ .type | quote }} + {{- if eq .type "credentials" }} credentials: - access_key_id: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} - secret_access_key: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + access_key_id: {{ .credentials.access_key_id | quote }} + secret_access_key: {{ .credentials.secret_access_key | quote }} {{- end }} - {{- if eq .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.type "profile" }} - profile: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.profile | quote }} + {{- if eq .type "profile" }} + profile: {{ .profile | quote }} {{- end }} - {{- else }} - type: "fs" - directory: "/var/data/archives" - {{- end }} - target_archive_size: {{ .Values.clpConfig.archive_output.target_archive_size | int }} - target_dictionaries_size: {{ .Values.clpConfig.archive_output.target_dictionaries_size - | int }} - target_encoded_file_size: {{ .Values.clpConfig.archive_output.target_encoded_file_size - | int }} - target_segment_size: {{ .Values.clpConfig.archive_output.target_segment_size - | int }} + {{- end }}{{/* with .aws_authentication */}} + {{- end }}{{/* with .storage.s3_config */}} + {{- end }}{{/* if eq .storage.type "fs" */}} + target_archive_size: {{ .target_archive_size | int }} + target_dictionaries_size: {{ .target_dictionaries_size | int }} + target_encoded_file_size: {{ .target_encoded_file_size | int }} + target_segment_size: {{ .target_segment_size | int }} + {{- end }}{{/* with .Values.clpConfig.archive_output */}} aws_config_directory: {{ .Values.clpConfig.aws_config_directory }} compression_scheduler: jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} @@ -76,36 +78,40 @@ data: archive: {{ .Values.clpConfig.garbage_collector.sweep_interval.archive }} search_result: {{ .Values.clpConfig.garbage_collector.sweep_interval.search_result }} logs_directory: "/var/log" + {{- with .Values.clpConfig.logs_input }} logs_input: - {{- if eq .Values.clpConfig.logs_input.type "s3" }} + {{- if eq .type "fs" }} + type: "fs" + directory: "/mnt/logs" + {{- else }} type: "s3" + {{- with .aws_authentication }} aws_authentication: - type: {{ .Values.clpConfig.logs_input.aws_authentication.type | quote }} - {{- if eq .Values.clpConfig.logs_input.aws_authentication.type "credentials" }} + type: {{ .type | quote }} + {{- if eq .type "credentials" }} credentials: - access_key_id: {{ .Values.clpConfig.logs_input.aws_authentication.credentials.access_key_id | quote }} - secret_access_key: {{ .Values.clpConfig.logs_input.aws_authentication.credentials.secret_access_key | quote }} + access_key_id: {{ .credentials.access_key_id | quote }} + secret_access_key: {{ .credentials.secret_access_key | quote }} {{- end }} - {{- if eq .Values.clpConfig.logs_input.aws_authentication.type "profile" }} - profile: {{ .Values.clpConfig.logs_input.aws_authentication.profile | quote }} + {{- if eq .type "profile" }} + profile: {{ .profile | quote }} {{- end }} - {{- else }} - type: "fs" - directory: "/mnt/logs" - {{- end }} - {{- if eq .Values.clpConfig.logs_input.type "s3" }} + {{- end }}{{/* with .aws_authentication */}} + {{- end }}{{/* if eq .type "fs" */}} + {{- end }}{{/* with .Values.clpConfig.logs_input */}} + {{- with .Values.clpConfig.log_ingestor }} log_ingestor: - buffer_flush_threshold: {{ .Values.clpConfig.log_ingestor.buffer_flush_threshold }} - buffer_flush_timeout: {{ .Values.clpConfig.log_ingestor.buffer_flush_timeout }} - channel_capacity: {{ .Values.clpConfig.log_ingestor.channel_capacity }} + buffer_flush_threshold: {{ .buffer_flush_threshold }} + buffer_flush_timeout: {{ .buffer_flush_timeout }} + channel_capacity: {{ .channel_capacity }} host: "localhost" - logging_level: {{ .Values.clpConfig.log_ingestor.logging_level | quote }} + logging_level: {{ .logging_level | quote }} port: 3002 {{- end }} - {{- if .Values.clpConfig.mcp_server }} + {{- with .Values.clpConfig.mcp_server }} mcp_server: host: "localhost" - logging_level: {{ .Values.clpConfig.mcp_server.logging_level | quote }} + logging_level: {{ .logging_level | quote }} port: 8000 {{- else }} mcp_server: null @@ -142,31 +148,37 @@ data: port: 27017 retention_period: {{ .Values.clpConfig.results_cache.retention_period }} stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} + {{- with .Values.clpConfig.stream_output }} stream_output: storage: - {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + {{- if eq .storage.type "fs" }} + type: "fs" + directory: "/var/data/streams" + {{- else }} type: "s3" staging_directory: "/var/data/staged-streams" + {{- with .storage.s3_config }} s3_config: - endpoint_url: {{ .Values.clpConfig.stream_output.storage.s3_config.endpoint_url | default "null" }} - region_code: {{ .Values.clpConfig.stream_output.storage.s3_config.region_code | quote }} - bucket: {{ .Values.clpConfig.stream_output.storage.s3_config.bucket | quote }} - key_prefix: {{ .Values.clpConfig.stream_output.storage.s3_config.key_prefix | quote }} + endpoint_url: {{ .endpoint_url | default "null" }} + region_code: {{ .region_code | quote }} + bucket: {{ .bucket | quote }} + key_prefix: {{ .key_prefix | quote }} + {{- with .aws_authentication }} aws_authentication: - type: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type | quote }} - {{- if eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "credentials" }} + type: {{ .type | quote }} + {{- if eq .type "credentials" }} credentials: - access_key_id: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} - secret_access_key: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + access_key_id: {{ .credentials.access_key_id | quote }} + secret_access_key: {{ .credentials.secret_access_key | quote }} {{- end }} - {{- if eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "profile" }} - profile: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.profile | quote }} + {{- if eq .type "profile" }} + profile: {{ .profile | quote }} {{- end }} - {{- else }} - directory: "/var/data/streams" - type: "fs" - {{- end }} - target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }} + {{- end }}{{/* with .aws_authentication */}} + {{- end }}{{/* with .storage.s3_config */}} + {{- end }}{{/* if eq .storage.type "fs" */}} + target_uncompressed_size: {{ .target_uncompressed_size | int }} + {{- end }}{{/* with .Values.clpConfig.stream_output */}} tmp_directory: "/var/tmp" webui: host: "localhost" @@ -243,21 +255,23 @@ data: {{- else }} "LogsInputRootDir": null, {{- end }} - {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - "StreamFilesDir": null, - "StreamFilesS3Region": {{ .Values.clpConfig.stream_output.storage.s3_config.region_code | quote }}, - "StreamFilesS3PathPrefix": "{{ .Values.clpConfig.stream_output.storage.s3_config.bucket }}/{{ .Values.clpConfig.stream_output.storage.s3_config.key_prefix }}", - {{- else }} + {{- with .Values.clpConfig.stream_output.storage }} + {{- if eq .type "fs" }} "StreamFilesDir": "/var/data/streams", "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, - {{- end }} - {{- if and (eq .Values.clpConfig.stream_output.storage.type "s3") - (eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "profile") }} - "StreamFilesS3Profile": {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.profile | quote }}, + "StreamFilesS3Profile": null, + {{- else }} + "StreamFilesDir": null, + "StreamFilesS3Region": {{ .s3_config.region_code | quote }}, + "StreamFilesS3PathPrefix": "{{ .s3_config.bucket }}/{{ .s3_config.key_prefix }}", + {{- if eq .s3_config.aws_authentication.type "profile" }} + "StreamFilesS3Profile": {{ .s3_config.aws_authentication.profile | quote }}, {{- else }} "StreamFilesS3Profile": null, {{- end }} + {{- end }}{{/* if eq .type "fs" */}} + {{- end }}{{/* with .Values.clpConfig.stream_output.storage */}} "StreamTargetUncompressedSize": {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }}, "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level }}, @@ -271,9 +285,9 @@ data: {{ .Values.clpConfig.archive_output.target_segment_size | int }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, - {{- if .Values.clpConfig.presto }} - "PrestoHost": {{ .Values.clpConfig.presto.host | quote }}, - "PrestoPort": {{ .Values.clpConfig.presto.port }} + {{- with .Values.clpConfig.presto }} + "PrestoHost": {{ .host | quote }}, + "PrestoPort": {{ .port }} {{- else }} "PrestoHost": null, "PrestoPort": null From 3a5968a163e51cc0724a5217bfaa9dcf7db43fca Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:20:38 -0500 Subject: [PATCH 133/155] fix(helm): Use `with` for aws_config_directory in webui-deployment template to improve readability --- tools/deployment/package-helm/templates/webui-deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index ec6f7adfda..44f588eee3 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -118,9 +118,9 @@ spec: - name: "server-settings" configMap: name: {{ include "clp.fullname" . }}-config - {{- if .Values.clpConfig.aws_config_directory }} + {{- with .Values.clpConfig.aws_config_directory }} - name: "aws-config" hostPath: - path: "{{ .Values.clpConfig.aws_config_directory }}" + path: {{ . | quote }} type: "Directory" {{- end }} From 753d40288530b8d2bb97a8b24d64b2bfb3200fad Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:23:09 -0500 Subject: [PATCH 134/155] fix(helm): Replace `if` with `with` for aws_config_directory in deployment templates for consistency and cleaner syntax --- .../templates/compression-scheduler-deployment.yaml | 4 ++-- .../package-helm/templates/compression-worker-deployment.yaml | 4 ++-- .../package-helm/templates/garbage-collector-deployment.yaml | 4 ++-- .../package-helm/templates/query-worker-deployment.yaml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 97f9c484b8..86d897d0ac 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -108,10 +108,10 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - {{- if .Values.clpConfig.aws_config_directory }} + {{- with .Values.clpConfig.aws_config_directory }} - name: "aws-config" hostPath: - path: "{{ .Values.clpConfig.aws_config_directory }}" + path: {{ . | quote }} type: "Directory" {{- end }} {{- if eq .Values.clpConfig.logs_input.type "fs" }} diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index b48eebf08b..03ec9a290c 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -119,9 +119,9 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - {{- if .Values.clpConfig.aws_config_directory }} + {{- with .Values.clpConfig.aws_config_directory }} - name: "aws-config" hostPath: - path: "{{ .Values.clpConfig.aws_config_directory }}" + path: {{ . | quote }} type: "Directory" {{- end }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 155a99c9f5..29cde9ca1d 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -116,10 +116,10 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - {{- if .Values.clpConfig.aws_config_directory }} + {{- with .Values.clpConfig.aws_config_directory }} - name: "aws-config" hostPath: - path: "{{ .Values.clpConfig.aws_config_directory }}" + path: {{ . | quote }} type: "Directory" {{- end }} {{- end }} diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index fcbb976519..14d8fc301f 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -121,9 +121,9 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - {{- if .Values.clpConfig.aws_config_directory }} + {{- with .Values.clpConfig.aws_config_directory }} - name: "aws-config" hostPath: - path: "{{ .Values.clpConfig.aws_config_directory }}" + path: {{ . | quote }} type: "Directory" {{- end }} From 896cb315421ea723390263c9eef4781007cbb6be Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:38:30 -0500 Subject: [PATCH 135/155] fix(helm): Update accessModes to ReadWriteOnce in PVC and PV templates for staged-streams and staged-archives --- .../package-helm/templates/api-server-staged-streams-pv.yaml | 2 +- .../package-helm/templates/api-server-staged-streams-pvc.yaml | 2 +- .../templates/compression-worker-staged-archives-pv.yaml | 2 +- .../templates/compression-worker-staged-archives-pvc.yaml | 2 +- .../package-helm/templates/query-worker-staged-streams-pv.yaml | 2 +- .../package-helm/templates/query-worker-staged-streams-pvc.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml index 0d191046d2..40c81da7a4 100644 --- a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml @@ -6,7 +6,7 @@ "name" "staged-streams" "nodeRole" "control-plane" "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") + "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) ) }} {{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml index 0979cd49aa..e695d9dcee 100644 --- a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml @@ -5,6 +5,6 @@ "component_category" "api-server" "name" "staged-streams" "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") + "accessModes" (list "ReadWriteOnce") ) }} {{- end }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index 924fe4639b..eb69332671 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -4,6 +4,6 @@ "name" "staged-archives" "nodeRole" "control-plane" "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") + "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/staged-archives" .Values.clpConfig.data_directory) ) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml index d96f2a62e1..5f6f3a346a 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -3,5 +3,5 @@ "component_category" "compression-worker" "name" "staged-archives" "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") + "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index e443b1ace9..c3f14f6b02 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -5,7 +5,7 @@ "name" "staged-streams" "nodeRole" "control-plane" "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") + "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) ) }} {{- end }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml index c433a8b2ea..e909a57b21 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -4,6 +4,6 @@ "component_category" "query-worker" "name" "staged-streams" "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") + "accessModes" (list "ReadWriteOnce") ) }} {{- end }} From 5c81b0fc206b3f8fdaa87191613846ce8de7d84c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:49:31 -0500 Subject: [PATCH 136/155] fix(helm): Add missing quotes for host fields in configmap template to ensure proper YAML formatting --- .../deployment/package-helm/templates/configmap.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 038f5c7b4f..026726e4a6 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -66,7 +66,7 @@ data: database: auto_commit: false compress: true - host: {{ include "clp.fullname" . }}-database + host: "{{ include "clp.fullname" . }}-database" names: clp: {{ .Values.clpConfig.database.names.clp | quote }} port: 3306 @@ -121,7 +121,7 @@ data: storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} presto: {{ .Values.clpConfig.presto }} query_scheduler: - host: {{ include "clp.fullname" . }}-query-scheduler + host: "{{ include "clp.fullname" . }}-query-scheduler" jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} num_archives_to_search_per_sub_job: {{ @@ -130,21 +130,21 @@ data: query_worker: logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }} queue: - host: {{ include "clp.fullname" . }}-queue + host: "{{ include "clp.fullname" . }}-queue" port: 5672 redis: compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} - host: {{ include "clp.fullname" . }}-redis + host: "{{ include "clp.fullname" . }}-redis" port: 6379 query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} reducer: base_port: 14009 - host: {{ include "clp.fullname" . }}-reducer + host: "{{ include "clp.fullname" . }}-reducer" logging_level: {{ .Values.clpConfig.reducer.logging_level | quote }} upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval }} results_cache: db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} - host: {{ include "clp.fullname" . }}-results-cache + host: "{{ include "clp.fullname" . }}-results-cache" port: 27017 retention_period: {{ .Values.clpConfig.results_cache.retention_period }} stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} From f9a507de187fb007d18eceb21ab8912146cfeac4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:50:37 -0500 Subject: [PATCH 137/155] fix(helm): Remove S3-specific PVC/PV and volume mounts for staged-streams in API server deployment --- .../templates/api-server-deployment.yaml | 16 ++-------------- .../templates/api-server-staged-streams-pv.yaml | 12 ------------ .../templates/api-server-staged-streams-pvc.yaml | 10 ---------- 3 files changed, 2 insertions(+), 36 deletions(-) delete mode 100644 tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml delete mode 100644 tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 40286036fc..58add1c665 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -67,13 +67,7 @@ spec: "name" "logs" ) | quote }} mountPath: "/var/log/api_server" - {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - - name: {{ include "clp.volumeName" (dict - "component_category" "api-server" - "name" "staged-streams" - ) | quote }} - mountPath: "/var/data/staged-streams" - {{- else }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" "name" "streams" @@ -100,13 +94,7 @@ spec: "component_category" "api-server" "name" "logs" ) | nindent 10 }} - {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "api-server" - "name" "staged-streams" - ) | nindent 10 }} - {{- else }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml deleted file mode 100644 index 40c81da7a4..0000000000 --- a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if and .Values.clpConfig.api_server - (eq .Values.clpConfig.stream_output.storage.type "s3") }} -{{- include "clp.createLocalPv" (dict - "root" . - "component_category" "api-server" - "name" "staged-streams" - "nodeRole" "control-plane" - "capacity" "20Gi" - "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) -) }} -{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml deleted file mode 100644 index e695d9dcee..0000000000 --- a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{- if and .Values.clpConfig.api_server - (eq .Values.clpConfig.stream_output.storage.type "s3") }} -{{- include "clp.createPvc" (dict - "root" . - "component_category" "api-server" - "name" "staged-streams" - "capacity" "20Gi" - "accessModes" (list "ReadWriteOnce") -) }} -{{- end }} From ee248a1b508942f8d5b7cb63d8506b35fed8e4b4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:53:56 -0500 Subject: [PATCH 138/155] fix(helm): Add `| quote` and handle null case for aws_config_directory in configmap template --- tools/deployment/package-helm/templates/configmap.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 026726e4a6..831f6cd7f0 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -53,7 +53,11 @@ data: target_encoded_file_size: {{ .target_encoded_file_size | int }} target_segment_size: {{ .target_segment_size | int }} {{- end }}{{/* with .Values.clpConfig.archive_output */}} - aws_config_directory: {{ .Values.clpConfig.aws_config_directory }} + {{- with .Values.clpConfig.aws_config_directory }} + aws_config_directory: {{ . | quote }} + {{- else }} + aws_config_directory: null + {{- end }} compression_scheduler: jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} From be70bd34cb02499e26f7fab054911844a02dac2a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:01:23 -0500 Subject: [PATCH 139/155] fix(helm): Add conditional for logsInputVolumeMount in webui-deployment template --- tools/deployment/package-helm/templates/webui-deployment.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 44f588eee3..a10e393f6c 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -84,7 +84,9 @@ spec: mountPath: "/opt/clp/.aws" readOnly: true {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} + {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -104,7 +106,9 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: *webui-health-check volumes: + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . From c346734be1652e2efb1dfb4bc5d0a8fc59057608 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:01:27 -0500 Subject: [PATCH 140/155] fix(helm): Add conditional for logsInputVolume in compression-scheduler deployment template --- .../templates/compression-scheduler-deployment.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 86d897d0ac..f49fa6c3b9 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -94,7 +94,9 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "compression-scheduler" @@ -114,9 +116,3 @@ spec: path: {{ . | quote }} type: "Directory" {{- end }} - {{- if eq .Values.clpConfig.logs_input.type "fs" }} - - name: "logs-input" - hostPath: - path: "{{ .Values.clpConfig.logs_input.directory }}" - type: "Directory" - {{- end }} From 80251dcb451777ce3f68476dccacb9a03bd4645c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:03:58 -0500 Subject: [PATCH 141/155] fix(helm): Correct placement of logsInputVolume and pvcVolume conditionals in deployment templates --- .../compression-scheduler-deployment.yaml | 6 +++--- .../templates/webui-deployment.yaml | 20 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index f49fa6c3b9..5a93bb721c 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -94,9 +94,6 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: - {{- if eq .Values.clpConfig.logs_input.type "fs" }} - - {{- include "clp.logsInputVolume" . | nindent 10 }} - {{- end }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "compression-scheduler" @@ -116,3 +113,6 @@ spec: path: {{ . | quote }} type: "Directory" {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index a10e393f6c..80e5376a58 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -106,16 +106,6 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: *webui-health-check volumes: - {{- if eq .Values.clpConfig.logs_input.type "fs" }} - - {{- include "clp.logsInputVolume" . | nindent 10 }} - {{- end }} - {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "streams" - ) | nindent 10 }} - {{- end }} - name: "client-settings" configMap: name: {{ include "clp.fullname" . }}-config @@ -128,3 +118,13 @@ spec: path: {{ . | quote }} type: "Directory" {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "streams" + ) | nindent 10 }} + {{- end }} From 82dc4d7972b3b704e58a166fdfc09b5df2da7403 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:17:50 -0500 Subject: [PATCH 142/155] fix(helm): Add int casting for numerical values in configmap template to ensure proper YAML formatting --- .../package-helm/templates/configmap.yaml | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 831f6cd7f0..b4e104ae6f 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -8,19 +8,23 @@ data: clp-config.yaml: | {{- with .Values.clpConfig.api_server }} api_server: - default_max_num_query_results: {{ .default_max_num_query_results }} + default_max_num_query_results: {{ .default_max_num_query_results | int }} host: "localhost" port: 3001 query_job_polling: - initial_backoff_ms: {{ .query_job_polling.initial_backoff_ms }} - max_backoff_ms: {{ .query_job_polling.max_backoff_ms }} + initial_backoff_ms: {{ .query_job_polling.initial_backoff_ms | int }} + max_backoff_ms: {{ .query_job_polling.max_backoff_ms | int }} {{- else }} api_server: null {{- end }} {{- with .Values.clpConfig.archive_output }} archive_output: - compression_level: {{ .compression_level }} - retention_period: {{ .retention_period }} + compression_level: {{ .compression_level | int }} + {{ with .retention_period }} + retention_period: {{ . | int }} + {{ else }} + retention_period: null + {{ end }} storage: {{- if eq .storage.type "fs" }} type: "fs" @@ -62,7 +66,7 @@ data: jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} max_concurrent_tasks_per_job: {{ - .Values.clpConfig.compression_scheduler.max_concurrent_tasks_per_job }} + .Values.clpConfig.compression_scheduler.max_concurrent_tasks_per_job | int }} type: {{ .Values.clpConfig.compression_scheduler.type | quote }} compression_worker: logging_level: {{ .Values.clpConfig.compression_worker.logging_level | quote }} @@ -79,8 +83,8 @@ data: garbage_collector: logging_level: {{ .Values.clpConfig.garbage_collector.logging_level | quote }} sweep_interval: - archive: {{ .Values.clpConfig.garbage_collector.sweep_interval.archive }} - search_result: {{ .Values.clpConfig.garbage_collector.sweep_interval.search_result }} + archive: {{ .Values.clpConfig.garbage_collector.sweep_interval.archive | int }} + search_result: {{ .Values.clpConfig.garbage_collector.sweep_interval.search_result | int }} logs_directory: "/var/log" {{- with .Values.clpConfig.logs_input }} logs_input: @@ -105,9 +109,9 @@ data: {{- end }}{{/* with .Values.clpConfig.logs_input */}} {{- with .Values.clpConfig.log_ingestor }} log_ingestor: - buffer_flush_threshold: {{ .buffer_flush_threshold }} - buffer_flush_timeout: {{ .buffer_flush_timeout }} - channel_capacity: {{ .channel_capacity }} + buffer_flush_threshold: {{ .buffer_flush_threshold | int }} + buffer_flush_timeout: {{ .buffer_flush_timeout | int }} + channel_capacity: {{ .channel_capacity | int }} host: "localhost" logging_level: {{ .logging_level | quote }} port: 3002 @@ -129,7 +133,7 @@ data: jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} num_archives_to_search_per_sub_job: {{ - .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job }} + .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job | int }} port: 7000 query_worker: logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }} @@ -137,20 +141,24 @@ data: host: "{{ include "clp.fullname" . }}-queue" port: 5672 redis: - compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} + compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database | int }} host: "{{ include "clp.fullname" . }}-redis" port: 6379 - query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} + query_backend_database: {{ .Values.clpConfig.redis.query_backend_database | int }} reducer: base_port: 14009 host: "{{ include "clp.fullname" . }}-reducer" logging_level: {{ .Values.clpConfig.reducer.logging_level | quote }} - upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval }} + upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval | int }} results_cache: db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} host: "{{ include "clp.fullname" . }}-results-cache" port: 27017 - retention_period: {{ .Values.clpConfig.results_cache.retention_period }} + {{ with .Values.clpConfig.results_cache.retention_period }} + retention_period: {{ . | int }} + {{ else }} + retention_period: null + {{ end }} stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} {{- with .Values.clpConfig.stream_output }} stream_output: @@ -187,7 +195,7 @@ data: webui: host: "localhost" port: 4000 - rate_limit: {{ .Values.clpConfig.webui.rate_limit }} + rate_limit: {{ .Values.clpConfig.webui.rate_limit | int }} results_metadata_collection_name: {{ .Values.clpConfig.webui.results_metadata_collection_name | quote }} @@ -278,7 +286,7 @@ data: {{- end }}{{/* with .Values.clpConfig.stream_output.storage */}} "StreamTargetUncompressedSize": {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }}, - "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level }}, + "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level | int }}, "ArchiveOutputTargetArchiveSize": {{ .Values.clpConfig.archive_output.target_archive_size | int }}, "ArchiveOutputTargetDictionariesSize": @@ -291,7 +299,7 @@ data: "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, {{- with .Values.clpConfig.presto }} "PrestoHost": {{ .host | quote }}, - "PrestoPort": {{ .port }} + "PrestoPort": {{ .port | int }} {{- else }} "PrestoHost": null, "PrestoPort": null From 870944375d13ded6e03d2aea9ae6ee7e3c371385 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:25:39 -0500 Subject: [PATCH 143/155] fix(helm): Adjust volume mounts and conditionals for staged archives and streams in worker deployments --- .../compression-worker-deployment.yaml | 52 +++++++++-------- ...compression-worker-staged-archives-pv.yaml | 2 + ...ompression-worker-staged-archives-pvc.yaml | 2 + .../templates/query-worker-deployment.yaml | 58 +++++++++---------- tools/deployment/package-helm/test.sh | 2 +- 5 files changed, 62 insertions(+), 54 deletions(-) diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 03ec9a290c..d8d3308353 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -45,30 +45,20 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "compression-worker" "name" "logs" ) | quote }} mountPath: "/var/log/compression_worker" - - name: {{ include "clp.volumeName" (dict - "component_category" "compression-worker" - "name" "staged-archives" - ) | quote }} - mountPath: "/var/data/staged-archives" - name: {{ include "clp.volumeName" (dict "component_category" "compression-worker" "name" "tmp" ) | quote }} mountPath: "/var/tmp" - {{- if .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - mountPath: "/opt/clp/.aws" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - {{- end }} {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -76,9 +66,21 @@ spec: ) | quote }} mountPath: "/var/data/archives" {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} {{- if eq .Values.clpConfig.logs_input.type "fs" }} - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} {{- end }} + {{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} + - name: {{ include "clp.volumeName" (dict + "component_category" "compression-worker" + "name" "staged-archives" + ) | quote }} + mountPath: "/var/data/staged-archives" + {{- end }} command: [ "python3", "-u", "/opt/clp/lib/python3/site-packages/bin/celery", @@ -91,24 +93,19 @@ spec: "-n", "compression-worker" ] volumes: - {{- if eq .Values.clpConfig.logs_input.type "fs" }} - - {{- include "clp.logsInputVolume" . | nindent 10 }} - {{- end }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "compression-worker" "name" "logs" ) | nindent 10 }} - - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "compression-worker" - "name" "staged-archives" - ) | nindent 10 }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "compression-worker" "name" "tmp" ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . @@ -116,12 +113,19 @@ spec: "name" "archives" ) | nindent 10 }} {{- end }} - - name: "config" - configMap: - name: {{ include "clp.fullname" . }}-config {{- with .Values.clpConfig.aws_config_directory }} - name: "aws-config" hostPath: path: {{ . | quote }} type: "Directory" {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-worker" + "name" "staged-archives" + ) | nindent 10 }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index eb69332671..c00226b797 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "compression-worker" @@ -7,3 +8,4 @@ "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/staged-archives" .Values.clpConfig.data_directory) ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml index 5f6f3a346a..b8a9367236 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} {{- include "clp.createPvc" (dict "root" . "component_category" "compression-worker" @@ -5,3 +6,4 @@ "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 14d8fc301f..cdac3c390e 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -45,15 +45,15 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "query-worker" "name" "logs" ) | quote }} mountPath: "/var/log/query_worker" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -61,12 +61,10 @@ spec: ) | quote }} mountPath: "/var/data/archives" {{- end }} - {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - - name: {{ include "clp.volumeName" (dict - "component_category" "shared-data" - "name" "streams" - ) | quote }} - mountPath: "/var/data/streams" + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - name: {{ include "clp.volumeName" (dict @@ -75,10 +73,12 @@ spec: ) | quote }} mountPath: "/var/data/staged-streams" {{- end }} - {{- if .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - mountPath: "/opt/clp/.aws" - readOnly: true + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} + - name: {{ include "clp.volumeName" (dict + "component_category" "shared-data" + "name" "streams" + ) | quote }} + mountPath: "/var/data/streams" {{- end }} command: [ "python3", "-u", @@ -97,18 +97,27 @@ spec: "component_category" "query-worker" "name" "logs" ) | nindent 10 }} - {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . - "component_category" "query-worker" - "name" "staged-streams" + "component_category" "shared-data" + "name" "archives" ) | nindent 10 }} {{- end }} - {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} + {{- with .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: {{ . | quote }} + type: "Directory" + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - {{- include "clp.pvcVolume" (dict "root" . - "component_category" "shared-data" - "name" "archives" + "component_category" "query-worker" + "name" "staged-streams" ) | nindent 10 }} {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} @@ -118,12 +127,3 @@ spec: "name" "streams" ) | nindent 10 }} {{- end }} - - name: "config" - configMap: - name: {{ include "clp.fullname" . }}-config - {{- with .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - hostPath: - path: {{ . | quote }} - type: "Directory" - {{- end }} diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index d8daaf3bec..69b8139904 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -56,7 +56,7 @@ wait_for_pods() { kind delete cluster --name clp-test rm -rf "$CLP_HOME" mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ - "$CLP_HOME/var/data/"{archives,streams} \ + "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \ "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ "$CLP_HOME/var/log/"{garbage_collector,api_server,log_ingestor,mcp_server} \ From 7443e4e3cefb4315e05bca5518dd5432dd418515 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:28:46 -0500 Subject: [PATCH 144/155] fix(helm): Correct placement of config volume mounts across deployment templates for consistency --- .../templates/api-server-deployment.yaml | 14 ++++---- .../compression-scheduler-deployment.yaml | 8 ++--- .../garbage-collector-deployment.yaml | 34 +++++++++---------- .../templates/log-ingestor-deployment.yaml | 8 ++--- .../templates/mcp-server-deployment.yaml | 8 ++--- .../templates/query-scheduler-deployment.yaml | 8 ++--- .../templates/reducer-deployment.yaml | 8 ++--- 7 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 58add1c665..26da76486f 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -58,15 +58,15 @@ spec: - name: "api-server" containerPort: 3001 volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "api-server" "name" "logs" ) | quote }} mountPath: "/var/log/api_server" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -94,6 +94,9 @@ spec: "component_category" "api-server" "name" "logs" ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . @@ -101,7 +104,4 @@ spec: "name" "streams" ) | nindent 10 }} {{- end }} - - name: "config" - configMap: - name: {{ include "clp.fullname" . }}-config {{- end }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 5a93bb721c..37a86ca019 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -66,10 +66,6 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "compression-scheduler" "name" "logs" @@ -80,6 +76,10 @@ spec: "name" "user-logs" ) | quote }} mountPath: "/var/log/user" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true {{- if .Values.clpConfig.aws_config_directory }} - name: "aws-config" mountPath: "/opt/clp/.aws" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 29cde9ca1d..f50d0e4e57 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -60,20 +60,15 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "garbage-collector" "name" "logs" ) | quote }} mountPath: "/var/log/garbage_collector" - {{- if .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - mountPath: "/opt/clp/.aws" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - {{- end }} {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -81,6 +76,11 @@ spec: ) | quote }} mountPath: "/var/data/archives" {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -99,6 +99,9 @@ spec: "component_category" "garbage-collector" "name" "logs" ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . @@ -106,6 +109,12 @@ spec: "name" "archives" ) | nindent 10 }} {{- end }} + {{- with .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: {{ . | quote }} + type: "Directory" + {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . @@ -113,13 +122,4 @@ spec: "name" "streams" ) | nindent 10 }} {{- end }} - - name: "config" - configMap: - name: {{ include "clp.fullname" . }}-config - {{- with .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - hostPath: - path: {{ . | quote }} - type: "Directory" - {{- end }} {{- end }} diff --git a/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml b/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml index 4c4946a892..bf753c7636 100644 --- a/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml +++ b/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml @@ -53,15 +53,15 @@ spec: - name: "log-ingestor" containerPort: 3002 volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "log-ingestor" "name" "logs" ) | quote }} mountPath: "/var/log/log_ingestor" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true command: [ "/opt/clp/bin/log-ingestor", "--config", "/etc/clp-config.yaml", diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 1238602ba3..09244b1b6b 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -60,15 +60,15 @@ spec: - name: "mcp-server" containerPort: 8000 volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "mcp-server" "name" "logs" ) | quote }} mountPath: "/var/log/mcp_server" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true command: [ "python3", "-u", "-m", "clp_mcp_server.clp_mcp_server", diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 5ebc901a89..b42c6546da 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -71,15 +71,15 @@ spec: - name: "query-scheduler" containerPort: 7000 volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "query-scheduler" "name" "logs" ) | quote }} mountPath: "/var/log/query_scheduler" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true command: [ "python3", "-u", "-m", "job_orchestration.scheduler.query.query_scheduler", diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index 09954063b5..51970785f7 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -50,15 +50,15 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "reducer" "name" "logs" ) | quote }} mountPath: "/var/log/reducer" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true command: [ "python3", "-u", "-m", "job_orchestration.reducer.reducer", From b32613b621e07a4f69e1e798687765ad38f03d6b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:32:26 -0500 Subject: [PATCH 145/155] chore(helm): Bump chart version to 0.1.2-dev.8 --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 1da0d1598e..d26da895a4 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.1.2-dev.7" +version: "0.1.2-dev.8" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.7.1-dev" From 3b411a7a1eb2635c31dfc1f7ea2c16553a4526e9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Dec 2025 15:43:27 -0500 Subject: [PATCH 146/155] fix(helm): Handle null case for log_ingestor and update default clpConfig values. --- .../package-helm/templates/configmap.yaml | 2 ++ tools/deployment/package-helm/values.yaml | 23 ++++--------------- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index b4e104ae6f..881dfbb972 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -115,6 +115,8 @@ data: host: "localhost" logging_level: {{ .logging_level | quote }} port: 3002 + {{- else }} + log_ingestor: null {{- end }} {{- with .Values.clpConfig.mcp_server }} mcp_server: diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 7a674ea5d8..5fbee3177c 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -22,8 +22,8 @@ workerConcurrency: 8 clpConfig: package: - storage_engine: "clp-s" - query_engine: "clp-s" + storage_engine: "clp" + query_engine: "clp" # Location (e.g., directory) containing any logs you wish to compress. Must be reachable by all # workers. @@ -133,25 +133,10 @@ clpConfig: search_result: 30 # API server config - api_server: - port: 30301 - default_max_num_query_results: 1000 - query_job_polling: - initial_backoff_ms: 100 - max_backoff_ms: 5000 + api_server: null # log-ingestor config. Currently, the config is applicable only if `logs_input.type` is "s3". - log_ingestor: - port: 30302 - # The timeout (in seconds) after which the log buffer is flushed for compression if no new - # input arrives. - buffer_flush_timeout: 300 - # The log buffer size (in bytes) that triggers a flush for compression. - buffer_flush_threshold: 268435456 # 256 MiB - # The capacity of the internal channel used for communication between an ingestion job and the - # log buffer. - channel_capacity: 10 - logging_level: "INFO" + log_ingestor: null # Presto client config presto: null From 1f82b194ade18d2c670ea9d1df52ce30bc574f01 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 23 Dec 2025 21:07:16 -0500 Subject: [PATCH 147/155] docs(helm): Update deployment guides for Docker Compose and add ones for Kubernetes, enhance configuration details, and improve orchestration documentation --- .../design-deployment-orchestration.md | 367 ++++++------ ...md => guides-docker-compose-deployment.md} | 48 +- .../src/user-docs/guides-external-database.md | 2 +- docs/src/user-docs/guides-k8s-deployment.md | 543 ++++++++++++++++++ docs/src/user-docs/guides-overview.md | 76 ++- docs/src/user-docs/index.md | 26 +- docs/src/user-docs/quick-start/clp-json.md | 239 +++++++- docs/src/user-docs/quick-start/clp-text.md | 218 ++++++- docs/src/user-docs/quick-start/index.md | 70 ++- 9 files changed, 1340 insertions(+), 249 deletions(-) rename docs/src/user-docs/{guides-multi-host.md => guides-docker-compose-deployment.md} (90%) create mode 100644 docs/src/user-docs/guides-k8s-deployment.md diff --git a/docs/src/dev-docs/design-deployment-orchestration.md b/docs/src/dev-docs/design-deployment-orchestration.md index 80bbe0c675..eea5facd0f 100755 --- a/docs/src/dev-docs/design-deployment-orchestration.md +++ b/docs/src/dev-docs/design-deployment-orchestration.md @@ -1,8 +1,9 @@ # Deployment orchestration -The CLP package is composed of several components that are currently designed to be deployed in a -set of containers that are orchestrated using a framework like [Docker Compose][docker-compose]. -This document explains the architecture of that orchestration and any associated nuances. +The CLP package is composed of several components that are designed to be deployed in a set of +containers that are orchestrated using a framework like [Docker Compose][docker-compose] or +[Kubernetes][kubernetes] (via [Helm][helm]). This document explains the architecture of that +orchestration and any associated nuances. ## Architecture @@ -21,120 +22,120 @@ one-time initialization jobs and their functions. :::{mermaid} %%{ - init: { - "theme": "base", - "themeVariables": { - "primaryColor": "#0066cc", - "primaryTextColor": "#fff", - "primaryBorderColor": "transparent", - "lineColor": "#007fff", - "secondaryColor": "#007fff", - "tertiaryColor": "#fff" - } - } +init: { +"theme": "base", +"themeVariables": { +"primaryColor": "#0066cc", +"primaryTextColor": "#fff", +"primaryBorderColor": "transparent", +"lineColor": "#007fff", +"secondaryColor": "#007fff", +"tertiaryColor": "#fff" +} +} }%% graph LR - %% Services - database["database (MySQL)"] - queue["queue (RabbitMQ)"] - redis["redis (Redis)"] - results_cache["results-cache (MongoDB)"] - compression_scheduler["compression-scheduler"] - query_scheduler["query-scheduler"] - spider_scheduler["spider-scheduler"] - compression_worker["compression-worker"] - spider_compression_worker["spider-compression-worker"] - query_worker["query-worker"] - reducer["reducer"] - api_server["api-server"] - garbage_collector["garbage-collector"] - webui["webui"] - mcp_server["mcp-server"] - log_ingestor["log-ingestor"] - - %% One-time jobs - db_table_creator["db-table-creator"] - results_cache_indices_creator["results-cache-indices-creator"] - - %% Dependencies - %% Link 0-1: Database --> Database initialization jobs - database -->|healthy| db_table_creator - results_cache -->|healthy| results_cache_indices_creator - linkStyle 0,1 stroke:#ffa500 - - %% Link 2-5: Celery dependencies --> Schedulers - queue -->|healthy| compression_scheduler - redis -->|healthy| compression_scheduler - queue -->|healthy| query_scheduler - redis -->|healthy| query_scheduler - linkStyle 2,3,4,5 stroke:#ff0000 - - %% Link 6: Schedulers --> Workers - query_scheduler -->|healthy| reducer - linkStyle 6 stroke:#800080 - - %% Link 7-15: Database initialization job --> Services - db_table_creator -->|completed_successfully| api_server - db_table_creator -->|completed_successfully| compression_scheduler - db_table_creator -->|completed_successfully| garbage_collector - db_table_creator -->|completed_successfully| log_ingestor - db_table_creator -->|completed_successfully| mcp_server - db_table_creator -->|completed_successfully| query_scheduler - db_table_creator -->|completed_successfully| spider_compression_worker - db_table_creator -->|completed_successfully| spider_scheduler - db_table_creator -->|completed_successfully| webui - linkStyle 7,8,9,10,11,12,13,14,15 stroke:#0000ff - - %% Link 16-20: Results cache initialization job --> Services - results_cache_indices_creator -->|completed_successfully| api_server - results_cache_indices_creator -->|completed_successfully| garbage_collector - results_cache_indices_creator -->|completed_successfully| mcp_server - results_cache_indices_creator -->|completed_successfully| reducer - results_cache_indices_creator -->|completed_successfully| webui - linkStyle 16,17,18,19,20 stroke:#008000 - - subgraph Databases - database - results_cache - subgraph celery_dependencies[Celery Dependencies] - queue - redis - end - end - - subgraph Initialization jobs - db_table_creator - results_cache_indices_creator - end - - subgraph Schedulers - compression_scheduler - query_scheduler - spider_scheduler - end - - subgraph Workers - compression_worker - spider_compression_worker - query_worker - reducer - end - - subgraph Management & UI - api_server - log_ingestor - garbage_collector - webui - end - - subgraph AI - mcp_server - end - - %% Subgraph styles - style celery_dependencies fill:#ffffe0 - style spider_compression_worker fill:#008080 - style spider_scheduler fill:#008080 +%% Services +database["database (MySQL)"] +queue["queue (RabbitMQ)"] +redis["redis (Redis)"] +results_cache["results-cache (MongoDB)"] +compression_scheduler["compression-scheduler"] +query_scheduler["query-scheduler"] +spider_scheduler["spider-scheduler"] +compression_worker["compression-worker"] +spider_compression_worker["spider-compression-worker"] +query_worker["query-worker"] +reducer["reducer"] +api_server["api-server"] +garbage_collector["garbage-collector"] +webui["webui"] +mcp_server["mcp-server"] +log_ingestor["log-ingestor"] + +%% One-time jobs +db_table_creator["db-table-creator"] +results_cache_indices_creator["results-cache-indices-creator"] + +%% Dependencies +%% Link 0-1: Database --> Database initialization jobs +database -->|healthy| db_table_creator +results_cache -->|healthy| results_cache_indices_creator +linkStyle 0,1 stroke:#ffa500 + +%% Link 2-5: Celery dependencies --> Schedulers +queue -->|healthy| compression_scheduler +redis -->|healthy| compression_scheduler +queue -->|healthy| query_scheduler +redis -->|healthy| query_scheduler +linkStyle 2,3,4,5 stroke:#ff0000 + +%% Link 6: Schedulers --> Workers +query_scheduler -->|healthy| reducer +linkStyle 6 stroke:#800080 + +%% Link 7-15: Database initialization job --> Services +db_table_creator -->|completed_successfully| api_server +db_table_creator -->|completed_successfully| compression_scheduler +db_table_creator -->|completed_successfully| garbage_collector +db_table_creator -->|completed_successfully| log_ingestor +db_table_creator -->|completed_successfully| mcp_server +db_table_creator -->|completed_successfully| query_scheduler +db_table_creator -->|completed_successfully| spider_compression_worker +db_table_creator -->|completed_successfully| spider_scheduler +db_table_creator -->|completed_successfully| webui +linkStyle 7,8,9,10,11,12,13,14,15 stroke:#0000ff + +%% Link 16-20: Results cache initialization job --> Services +results_cache_indices_creator -->|completed_successfully| api_server +results_cache_indices_creator -->|completed_successfully| garbage_collector +results_cache_indices_creator -->|completed_successfully| mcp_server +results_cache_indices_creator -->|completed_successfully| reducer +results_cache_indices_creator -->|completed_successfully| webui +linkStyle 16,17,18,19,20 stroke:#008000 + +subgraph Databases +database +results_cache +subgraph celery_dependencies[Celery Dependencies] +queue +redis +end +end + +subgraph Initialization jobs +db_table_creator +results_cache_indices_creator +end + +subgraph Schedulers +compression_scheduler +query_scheduler +spider_scheduler +end + +subgraph Workers +compression_worker +spider_compression_worker +query_worker +reducer +end + +subgraph Management & UI +api_server +log_ingestor +garbage_collector +webui +end + +subgraph AI +mcp_server +end + +%% Subgraph styles +style celery_dependencies fill:#ffffe0 +style spider_compression_worker fill:#008080 +style spider_scheduler fill:#008080 +++ **Figure 1**: Orchestration architecture of the services in the CLP package. @@ -188,102 +189,96 @@ graph LR **Table 2**: One-time initialization jobs in the CLP package. :::: -## Code structure +## Orchestration methods -The orchestration code is split up into: +CLP supports two orchestration methods: Docker Compose for single-host or manual multi-host +deployments, and Helm for Kubernetes deployments. Both methods share the same configuration +interface (`clp-config.yaml` and `credentials.yaml`) and support the same deployment types. -* `BaseController` that defines: - * common logic for preparing the environment variables, configuration files, and directories - necessary for each service. - * abstract methods that orchestrator-specific derived classes must implement in order to - orchestrate a deployment. -* `Controller` that implements (and/or overrides) any of the methods in - `BaseController` (`` is a placeholder for the specific orchestrator for which the - class is being implemented). +### Configuration -## Docker Compose orchestration +Each service requires configuration values passed through config files, environment variables, +and/or command line arguments. Since services run in containers, some values must be adapted for the +orchestration environment—specifically, host paths must be converted to container paths, and +hostnames/ports must use service discovery mechanisms. -This section explains how we use Docker Compose to orchestrate the CLP package and is broken into -the following subsections: +The orchestration controller (e.g., `DockerComposeController`) reads `etc/clp-config.yaml` and +`etc/credentials.yaml`, then generates: +* A container-specific CLP config file with adapted paths and service names +* Runtime configuration (environment variables or ConfigMaps) +* Required directories (e.g., data output directories) -* [Setting up the Docker Compose project's environment](#setting-up-the-environment) -* [Starting and stoping the Docker Compose project](#starting-and-stopping-the-project) -* [Deployment types](#deployment-types) -* [Implementation details](#implementation-details) -* [Troubleshooting](#troubleshooting) +For Docker Compose, this generates `var/log/.clp-config.yaml` and `.env`. For Kubernetes, the Helm +chart generates a ConfigMap and Secrets from `values.yaml`. -### Setting up the environment - -Several services require configuration values to be passed in through the CLP package's config file, -environment variables, and/or command line arguments. Since the services are running in containers, -some of these configuration values need to be modified for the orchestration environment. -Specifically: +:::{note} +A `KubernetesController` is also planned that will read `clp-config.yaml` and `credentials.yaml` +like `DockerComposeController`, then set up the Helm release accordingly. This will unify the +configuration experience across both orchestration methods. +::: -1. Paths on the host must be converted to appropriate paths in the container. -2. Component hostnames must be converted to service names, and component ports must be converted to the component's default ports. - * This ensures that in the Docker Compose configuration, services can communicate over fixed, predictable hostnames and ports rather than relying on configurable variables. +### Secrets -To achieve this, before starting the deployment, `DockerComposeController.start` generates: +Sensitive credentials (database passwords, API keys) are stored in `etc/credentials.yaml` and +require special handling to avoid exposure. -* a CLP configuration file (`/var/log/.clp-config.yaml` on the host) specific to the - Docker Compose project environment. -* an environment variable file (`/.env`) for any other configuration values. -* any necessary directories (e.g., data output directories). +* **Docker Compose**: Credentials are written to `.env` and passed as environment variables +* **Kubernetes**: Credentials are stored in Kubernetes Secrets -The Docker Compose project then passes those environment variables to the relevant services, either -as environment variables or command line arguments, as necessary. +### Dependencies -### Starting and stopping the project +As shown in [Figure 1](#figure-1), services have complex interdependencies. Both orchestrators +ensure services start only after their dependencies are healthy. -To start and stop the project, `DockerComposeController` simply invokes `docker compose up` or -`docker compose down` as appropriate. However, to allow multiple CLP packages to be run on the same -host, we explicitly specify a project name for the project, where the name is based on the package's -instance ID. +* **Docker Compose**: Uses `depends_on` with `condition: service_healthy` and container healthchecks +* **Kubernetes**: Uses init containers (via the `clp.waitFor` helper) and readiness/liveness probes -### Deployment Types +### Storage -CLP supports four deployment types determined by the `compression_scheduler.type` and -`package.query_engine` configuration setting. +Services require persistent storage for logs, data, archives, and streams. -| Deployment Type | Compression Scheduler | Query Engine | Docker Compose File | -|-----------------|-----------------------|------------------------------|------------------------------------| -| Base | Celery | [Presto][presto-integration] | `docker-compose-base.yaml` | -| Full | Celery | Native | `docker-compose.yaml` | -| Spider Base | Spider | [Presto][presto-integration] | `docker-compose-spider-base.yaml` | -| Spider Full | Spider | Native | `docker-compose-spider.yaml` | +* **Docker Compose**: Uses bind mounts for host directories and named volumes for database data. + Conditional mounts use variable interpolation to mount empty tmpfs when not needed. +* **Kubernetes**: Uses PersistentVolumeClaims per component, with shared PVCs (`ReadWriteMany`) for + archives and streams. Uses `local-storage` StorageClass by default. -### Implementation details +### Deployment types -One notable implementation detail is in how we handle mounts that are only necessary under certain -configurations. For instance, the input logs mount is only necessary when the `logs_input.type` is -`fs`. If `logs_input.type` is `s3`, we shouldn't mount some random directory from the user's -host filesystem into the container. However, Docker doesn't provide a mechanism to perform -conditional mounts. Instead, we use Docker's variable interpolation to conditionally mount an empty -tmpfs mount into the container. This strategy is used wherever we need a conditional mount. +CLP supports multiple deployment configurations based on the compression scheduler and query engine. -### Troubleshooting +| Deployment Type | Compression Scheduler | Query Engine | +|-----------------|-----------------------|------------------------------| +| Base | Celery | [Presto][presto-integration] | +| Full | Celery | Native | +| Spider Base | Spider | [Presto][presto-integration] | +| Spider Full | Spider | Native | -If you encounter issues with the Docker Compose deployment, first determine the instance ID for your -deployment by checking the content of `/var/log/instance-id`. Then run one of the -commands below as necessary. +:::{note} +Spider support is not yet available for Helm. +::: -1. Check service status: +Docker Compose selects the appropriate compose file (e.g., `docker-compose.yaml` for Full, +`docker-compose-spider.yaml` for Spider Full) and uses `deploy.replicas` with environment +variables (e.g., `CLP_MCP_SERVER_ENABLED`) to toggle optional services. Helm uses conditional +templating to include/exclude resources. - ```bash - docker compose --project-name clp-package- ps - ``` +## Troubleshooting -2. View service logs: +When issues arise, use the appropriate commands for your orchestration method: - ```bash - docker compose --project-name clp-package- logs - ``` +* [Docker Compose debugging][docker-compose-debugging] +* [Kubernetes Helm debugging][kubernetes-debugging] -3. Validate configuration: +## User guides - ```bash - docker compose config - ``` +* [Kubernetes deployment][kubernetes-guide]: Deploying CLP with Helm +* [Multi-host deployment][multi-host]: Manual Docker Compose across multiple hosts [docker-compose]: https://docs.docker.com/compose/ +[docker-compose-debugging]: ../user-docs/guides-docker-compose-deployment.md#monitoring-and-debugging +[helm]: https://helm.sh/ +[kubernetes]: https://kubernetes.io/ +[kubernetes-debugging]: ../user-docs/guides-k8s-deployment.md#monitoring-and-debugging +[kubernetes-guide]: ../user-docs/guides-k8s-deployment.md +[multi-host]: ../user-docs/guides-docker-compose-deployment.md [presto-integration]: ../user-docs/guides-using-presto.md diff --git a/docs/src/user-docs/guides-multi-host.md b/docs/src/user-docs/guides-docker-compose-deployment.md similarity index 90% rename from docs/src/user-docs/guides-multi-host.md rename to docs/src/user-docs/guides-docker-compose-deployment.md index e9e5ecd436..3f8be96752 100755 --- a/docs/src/user-docs/guides-multi-host.md +++ b/docs/src/user-docs/guides-docker-compose-deployment.md @@ -1,12 +1,28 @@ -# Multi-host deployment +# Docker Compose deployment + +This guide explains how to deploy CLP using Docker Compose. Docker Compose provides a +straightforward way to orchestrate CLP's services, suitable for both development and production +environments. + +## Deployment options + +Docker Compose can be used for: + +* **Single-host deployment**: Run all CLP services on a single machine. This is the simplest setup, + covered in the [quick-start guides](quick-start/index.md). +* **Multi-host deployment**: Distribute CLP services across multiple machines for higher throughput + and scalability. This is covered in detail below. + +--- + +## Multi-host deployment A multi-host deployment allows you to run CLP across a distributed set of hosts. -:::{warning} -The instructions below provide a temporary solution for multi-host deployment and may change as we -actively work to improve ease of deployment. The present solution uses *manual* Docker Compose -orchestration; however, Kubernetes Helm support will be available in a future release, which will -simplify multi-host deployments significantly. +:::{note} +The instructions below use *manual* Docker Compose orchestration, which is more lightweight and +provides fine-grained control over service placement, but requires more configuration than +Helm-based deployments. ::: ## Requirements @@ -305,9 +321,11 @@ sbin/stop-clp.sh This will stop all CLP services managed by Docker Compose on the current host. -## Monitoring services +## Monitoring and debugging -To check the status of services on a host: +First, determine your instance ID from `/var/log/instance-id`. + +To check the status of services: ```bash docker compose --project-name clp-package- ps @@ -319,6 +337,20 @@ To view logs for a specific service: docker compose --project-name clp-package- logs -f ``` +To execute commands in a running container: + +```bash +docker compose --project-name clp-package- exec /bin/bash +``` + +To validate your Docker Compose configuration: + +```bash +docker compose config +``` + +--- + ## Setting up SeaweedFS The instructions below are for running a simple SeaweedFS cluster on a set of hosts. For other use diff --git a/docs/src/user-docs/guides-external-database.md b/docs/src/user-docs/guides-external-database.md index e4d5a8b670..3c435771e3 100644 --- a/docs/src/user-docs/guides-external-database.md +++ b/docs/src/user-docs/guides-external-database.md @@ -208,4 +208,4 @@ initialization jobs (`db-table-creator` and `results-cache-indices-creator`). [docker-compose-orchestration]: ../dev-docs/design-deployment-orchestration.md#docker-compose-orchestration [mongodb-install]: https://www.mongodb.com/docs/manual/installation/ [mongodb-security]: https://docs.mongodb.com/manual/security/ -[multi-host-guide]: guides-multi-host.md#starting-clp +[multi-host-guide]: guides-docker-compose-deployment.md#starting-clp diff --git a/docs/src/user-docs/guides-k8s-deployment.md b/docs/src/user-docs/guides-k8s-deployment.md new file mode 100644 index 0000000000..211f360d0c --- /dev/null +++ b/docs/src/user-docs/guides-k8s-deployment.md @@ -0,0 +1,543 @@ +# Kubernetes deployment + +This guide explains how to deploy CLP on Kubernetes using [Helm]. This provides an alternative to +Docker Compose and enables deployment on Kubernetes clusters ranging from local development setups +to production environments. + +:::{note} +For a detailed overview of CLP's services and their dependencies, see the [deployment orchestration +design doc][design-orchestration]. +::: + +--- + +## Requirements + +The following tools are required to deploy CLP on Kubernetes: + +* [kubectl] >= 1.30 +* [Helm] >= 4.0 +* A Kubernetes cluster (see [Setting up a cluster](#setting-up-a-cluster) below) +* When not using S3 storage, a shared filesystem accessible by all worker pods (e.g., NFS, + [SeaweedFS]) or local storage for single-node deployments + +--- + +## Setting up a cluster + +You can deploy CLP on either a local development cluster or a production Kubernetes cluster. + +### Option 1: Local development with kind + +[kind] (Kubernetes in Docker) is ideal for testing and development. It runs a Kubernetes cluster +inside Docker containers on your local machine. + +For single-host kind deployments, see the [quick-start guides][quick-start], which cover +creating a kind cluster and installing the Helm chart. + +### Option 2: Production Kubernetes cluster + +For production deployments, you can use any Kubernetes distribution: + +* Managed Kubernetes services: [Amazon EKS][eks], [Google GKE][gke], [Azure AKS][aks] +* Self-hosted: [kubeadm], [k3s], [RKE2] + +#### Setting up a cluster with kubeadm + +[kubeadm] is the official Kubernetes tool for bootstrapping clusters. Follow the +[official kubeadm installation guide][kubeadm] to install the prerequisites, container runtime, +and kubeadm on all nodes. + +1. **Initialize the control plane** (on the control-plane node only): + + ```bash + sudo kubeadm init --pod-network-cidr=10.244.0.0/16 + ``` + + :::{tip} + Save the `kubeadm join` command printed at the end of the output. You'll need it to join worker + nodes later. + ::: + + :::{note} + The `--pod-network-cidr` specifies the IP range for pods. If `10.244.0.0/16` conflicts with your + network, use a different private range as [RFC 1918][rfc-1918] specifies (e.g., `192.168.0.0/16`, + `172.16.0.0/16`, or `10.200.0.0/16`). + ::: + + To set up `kubectl` for your user: + + ```bash + mkdir -p "$HOME/.kube" + sudo cp -i /etc/kubernetes/admin.conf "$HOME/.kube/config" + sudo chown "$(id -u):$(id -g)" "$HOME/.kube/config" + ``` + +2. **Install a CNI plugin** (on the control-plane node): + + A CNI plugin is required for pod-to-pod networking. The following installs [Cilium], a + high-performance CNI that uses eBPF: + + ```bash + helm repo add cilium https://helm.cilium.io/ + helm repo update + helm install cilium cilium/cilium --namespace kube-system \ + --set ipam.operator.clusterPoolIPv4PodCIDRList=10.244.0.0/16 + ``` + + :::{note} + The `clusterPoolIPv4PodCIDRList` must match the `--pod-network-cidr` used in `kubeadm init`. + ::: + +3. **Join worker nodes** (on each worker node): + + Run `kubeadm join` with the token and hash you saved from step 1: + + ```bash + sudo kubeadm join :6443 \ + --token \ + --discovery-token-ca-cert-hash sha256: + ``` + + If you lost the command, regenerate it on the control-plane node with + `kubeadm token create --print-join-command`. + +4. **Label worker nodes** (on the control-plane node): + + CLP's Helm chart uses node affinity to schedule PersistentVolumes. Worker nodes must be labeled + so that storage can be properly allocated: + + ```bash + # List nodes to get their names + kubectl get nodes + + # Label each worker node + kubectl label node node-role.kubernetes.io/worker= + + # Verify labels + kubectl get nodes --show-labels + ``` + +5. **Allow scheduling on control-plane**: + + By default, kubeadm taints the control-plane node to prevent workloads from running on it. You + must remove this taint so CLP pods can be scheduled: + + ```bash + kubectl taint nodes --selector node-role.kubernetes.io/control-plane= node-role.kubernetes.io/control-plane- + ``` + + :::{note} + The CLP Helm chart currently requires some pods (database, queue, redis) to run on the + control-plane node. We plan to address this limitation in a future release. + ::: + +--- + +## Installing the Helm chart + +Once your cluster is ready, you can install CLP using the Helm chart. + +### Getting the chart + +The CLP Helm chart is located in the repository at `tools/deployment/package-helm/`. + +```bash +# Clone the repository (if you haven't already) +git clone https://github.com/y-scope/clp.git +cd clp/tools/deployment/package-helm +``` + +#### Production cluster requirements (optional) + +The following configurations are optional but recommended for production deployments. You can skip +this section for testing or development. + +1. **Storage for CLP Package services' data and logs** (optional, for centralized debugging): + + The Helm chart creates static PersistentVolumes using local host paths by default, so no + StorageClass configuration is required for basic deployments. For easier debugging, you can + configure a centralized storage backend for the following directories: + + * `data_directory` - where CLP stores runtime data + * `logs_directory` - where CLP services write logs + * `tmp_directory` - where temporary files are stored + + :::{note} + We aim to improve the logging infrastructure so mapping log volumes will not be required in the + future. See [issue #1760][logging-infra-issue] for details. + ::: + +2. **Shared storage for workers** (required for multi-node clusters using filesystem storage): + + :::{tip} + [S3 storage][s3-storage] is **strongly recommended** for multi-node clusters as it does not + require shared local storage between workers. If you use S3 storage, you can skip this section. + ::: + + For multi-node clusters using filesystem storage, the following directories **must** be + accessible from all worker nodes at the same paths. Without shared storage, compressed logs + created by one worker cannot be searched by other workers. + + * `archive_output.storage.directory` - where compressed archives are stored + * `stream_output.storage.directory` - where stream files are stored + * `logs_input.directory` - where input logs are read from + + Set up NFS, SeaweedFS, or another shared filesystem to provide this access. See the + [multi-host deployment guide][multi-host-guide] for SeaweedFS setup instructions. + +3. **External databases** (recommended for production): + * See the [external database setup guide][external-db-guide] for using external + MariaDB/MySQL and MongoDB databases + +### Basic installation + +Create the required directories on all worker nodes: + +```bash +export CLP_HOME="/tmp/clp" + +mkdir -p "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \ + "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ + "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ + "$CLP_HOME/var/tmp" +``` + +Then on the **control-plane node**, generate credentials and install CLP: + +```bash +export CLP_HOME="/tmp/clp" + +mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ + "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \ + "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ + "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ + "$CLP_HOME/var/log/"{garbage_collector,api_server,log_ingestor,mcp_server} \ + "$CLP_HOME/var/tmp" + +# Credentials (change these for production) +export CLP_DB_PASS="pass" +export CLP_DB_ROOT_PASS="root-pass" +export CLP_QUEUE_PASS="pass" +export CLP_REDIS_PASS="pass" + +# Worker replicas (increase for multi-node clusters) +export CLP_COMPRESSION_WORKER_REPLICAS=1 +export CLP_QUERY_WORKER_REPLICAS=1 + +helm install clp . \ + --set clpConfig.data_directory="$CLP_HOME/var/data" \ + --set clpConfig.logs_directory="$CLP_HOME/var/log" \ + --set clpConfig.tmp_directory="$CLP_HOME/var/tmp" \ + --set clpConfig.archive_output.storage.directory="$CLP_HOME/var/data/archives" \ + --set clpConfig.stream_output.storage.directory="$CLP_HOME/var/data/streams" \ + --set credentials.database.password="$CLP_DB_PASS" \ + --set credentials.database.root_password="$CLP_DB_ROOT_PASS" \ + --set credentials.queue.password="$CLP_QUEUE_PASS" \ + --set credentials.redis.password="$CLP_REDIS_PASS" \ + --set replicas.compressionWorker="$CLP_COMPRESSION_WORKER_REPLICAS" \ + --set replicas.queryWorker="$CLP_QUERY_WORKER_REPLICAS" +``` + +### Installation with custom values + +For highly customized deployments, create a values file instead of using many `--set` flags: + +```{code-block} yaml +:caption: custom-values.yaml + +# Use a custom image. For local images, import to each node's container runtime first. +image: + clpPackage: + repository: "clp-package" + pullPolicy: "Never" # Use "Never" for local images, "IfNotPresent" for remote + tag: "latest" + +# Adjust worker concurrency +workerConcurrency: 16 + +# Configure CLP settings +clpConfig: + # Use clp-text, instead of clp-json (default) + package: + storage_engine: "clp" # Use "clp-s" for clp-json, "clp" for clp-text + query_engine: "clp" # Use "clp-s" for clp-json, "clp" for clp-text, "presto" for Presto + + # Configure archive output + archive_output: + target_archive_size: 536870912 # 512 MB + compression_level: 6 + + # Enable MCP server + mcp_server: + port: 30800 + logging_level: "INFO" + + # Configure data retention (in minutes) + archive_output: + retention_period: 10080 # 7 days + results_cache: + retention_period: 120 # 2 hours + +# Override credentials (use secrets in production!) +credentials: + database: + username: "clp-user" + password: "your-db-password" + root_username: "root" + root_password: "your-db-root-password" + queue: + username: "clp-user" + password: "your-queue-password" + redis: + password: "your-redis-password" +``` + +Install with custom values: + +```bash +helm install clp . -f custom-values.yaml +``` + +::::{tip} +To preview the generated Kubernetes manifests before installing, use `helm template`: + +```bash +helm template clp . -f custom-values.yaml +``` +:::: + +### Common configuration options + +The following table lists commonly used Helm values. For a complete list, see `values.yaml` in the +chart directory. + +| Parameter | Description | Default | +|----------------------------------------------|----------------------------------------------|-----------------------------------| +| `image.clpPackage.repository` | CLP package image repository | `ghcr.io/y-scope/clp/clp-package` | +| `image.clpPackage.tag` | Image tag | `main` | +| `workerConcurrency` | Number of worker processes | `8` | +| `allowHostAccessForSbinScripts` | Expose database/cache for sbin scripts | `true` | +| `clpConfig.package.storage_engine` | Storage engine (`clp-s` or `clp`) | `clp-s` | +| `clpConfig.package.query_engine` | Query engine (`clp-s`, `clp`, or `presto`) | `clp-s` | +| `clpConfig.webui.port` | Web UI NodePort | `30000` | +| `clpConfig.api_server.port` | API server NodePort | `30301` | +| `clpConfig.database.port` | Database NodePort | `30306` | +| `clpConfig.results_cache.port` | Results cache (MongoDB) NodePort | `30017` | +| `clpConfig.mcp_server.port` | MCP server NodePort | `30800` | +| `clpConfig.logs_input.directory` | Directory containing logs to compress | `/` | +| `clpConfig.data_directory` | Directory for data storage | `/tmp/clp/var/data` | +| `clpConfig.logs_directory` | Directory for log files | `/tmp/clp/var/log` | +| `clpConfig.tmp_directory` | Directory for temporary files | `/tmp/clp/var/tmp` | +| `clpConfig.archive_output.storage.directory` | Directory for compressed archives | `/tmp/clp/var/data/archives` | +| `clpConfig.stream_output.storage.directory` | Directory for stream files | `/tmp/clp/var/data/streams` | +| `clpConfig.archive_output.retention_period` | Archive retention (minutes, null to disable) | `null` | +| `clpConfig.results_cache.retention_period` | Search results retention (minutes) | `60` | + +--- + +## Verifying the deployment + +After installing the Helm chart, verify that all components are running correctly. + +### Check pod status + +Wait for all pods to be ready: + +```bash +# Watch pod status +kubectl get pods -w + +# Wait for all pods to be ready +kubectl wait pods --all --for=condition=Ready --timeout=300s +``` + +Expected output shows all pods in `Running` state: + +``` +NAME READY STATUS RESTARTS AGE +clp-api-server-... 1/1 Running 0 2m +clp-compression-scheduler-... 1/1 Running 0 2m +clp-compression-worker-... 1/1 Running 0 2m +clp-database-0 1/1 Running 0 2m +clp-garbage-collector-... 1/1 Running 0 2m +clp-query-scheduler-... 1/1 Running 0 2m +clp-query-worker-... 1/1 Running 0 2m +clp-queue-0 1/1 Running 0 2m +clp-reducer-... 1/1 Running 0 2m +clp-redis-0 1/1 Running 0 2m +clp-results-cache-0 1/1 Running 0 2m +clp-webui-... 1/1 Running 0 2m +``` + +### Check initialization jobs + +CLP runs initialization jobs on first deployment: + +```bash +# Check job completion +kubectl get jobs + +# Expected output: +# NAME COMPLETIONS DURATION AGE +# clp-db-table-creator 1/1 5s 2m +# clp-results-cache-indices-creator 1/1 3s 2m +``` + +### Access the Web UI + +Once all pods are ready, access the CLP Web UI: `http://:30000` (the value of `clpConfig.webui.port`) + +--- + +## Using CLP + +With CLP deployed on Kubernetes, you can compress and search logs using the same workflows as +Docker Compose deployments. Refer to the quick-start guide for your chosen flavor: + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: quick-start/clp-json +Using clp-json +^^^ +How to compress and search JSON logs. +::: + +:::{grid-item-card} +:link: quick-start/clp-text +Using clp-text +^^^ +How to compress and search unstructured text logs. +::: +:::: + +:::{note} +By default (`allowHostAccessForSbinScripts: true`), the database and results cache are exposed on +NodePorts, allowing you to use `sbin/` scripts from the CLP package. Download a +[release][clp-releases] matching the chart's `appVersion`, then configure `etc/clp-config.yaml`: + +```yaml +database: + port: 30306 # Match `clpConfig.database.port` in Helm values +results_cache: + port: 30017 # Match `clpConfig.results_cache.port` in Helm values +``` + +Alternatively, use the Web UI ([clp-json][webui-clp-json]|[clp-text][webui-clp-text]) to compress +logs and search interactively, or the [API server][api-server] to submit queries and view results +programmatically. +::: + +--- + +## Monitoring and debugging + +To check the status of pods: + +```bash +kubectl get pods +``` + +To view logs for a specific pod: + +```bash +kubectl logs -f +``` + +To execute commands in a pod: + +```bash +kubectl exec -it -- /bin/bash +``` + +To debug Helm chart issues: + +```bash +helm install clp . --dry-run --debug +``` + +--- + +## Managing releases + +This section covers how to manage your CLP Helm release. + +:::{note} +Upgrade and rollback are not yet supported. We plan to add support as we finalize the migration +mechanism. +::: + +### Uninstall CLP + +```bash +helm uninstall clp +``` + +:::{warning} +Uninstalling the Helm release will delete all CLP pods and services. However, PersistentVolumes +with `Retain` policy will preserve your data. To completely remove all data, delete the PVs and +the data directories manually. +::: + +--- + +## Cleaning up + +To tear down a kubeadm cluster: + +1. **Uninstall Cilium** (on the control-plane): + + ```bash + helm uninstall cilium --namespace kube-system + ``` + +2. **Reset each node** (run on all worker nodes first, then the control-plane): + + ```bash + sudo kubeadm reset -f + sudo rm -rf /etc/cni/net.d/* + sudo umount /var/run/cilium/cgroupv2/ + sudo rm -rf /var/run/cilium + ``` + +3. **Clean up kubeconfig** (on the control-plane): + + ```bash + rm -rf ~/.kube + ``` + +--- + +## Related guides + +* [Docker Compose deployment][docker-compose-deployment] - Docker Compose orchestration for single or multi-host setups +* [External database setup][external-db-guide] - Using external MariaDB and MongoDB +* [Using object storage][s3-storage] - Configuring S3 storage +* [Configuring retention periods][retention-guide] - Setting up data retention policies + +[aks]: https://azure.microsoft.com/en-us/products/kubernetes-service +[api-server]: guides-using-the-api-server.md +[Cilium]: https://cilium.io/ +[clp-releases]: https://github.com/y-scope/clp/releases +[design-orchestration]: ../dev-docs/design-deployment-orchestration.md +[docker-compose-deployment]: guides-docker-compose-deployment.md +[eks]: https://aws.amazon.com/eks/ +[external-db-guide]: guides-external-database.md +[gke]: https://cloud.google.com/kubernetes-engine +[Helm]: https://helm.sh/ +[k3s]: https://k3s.io/ +[kind]: https://kind.sigs.k8s.io/ +[kubeadm]: https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/ +[kubectl]: https://kubernetes.io/docs/tasks/tools/ +[logging-infra-issue]: https://github.com/y-scope/clp/issues/1760 +[multi-host-guide]: guides-docker-compose-deployment.md +[quick-start]: quick-start/index.md +[retention-guide]: guides-retention.md +[rfc-1918]: https://datatracker.ietf.org/doc/html/rfc1918#section-3 +[RKE2]: https://docs.rke2.io/ +[s3-storage]: guides-using-object-storage/index +[SeaweedFS]: https://github.com/seaweedfs/seaweedfs +[webui-clp-json]: quick-start/clp-json.md#searching-from-the-ui +[webui-clp-text]: quick-start/clp-text.md#searching-from-the-ui diff --git a/docs/src/user-docs/guides-overview.md b/docs/src/user-docs/guides-overview.md index 8b9186768b..90c6c1ed10 100644 --- a/docs/src/user-docs/guides-overview.md +++ b/docs/src/user-docs/guides-overview.md @@ -1,6 +1,41 @@ # Overview -The guides below describe how to use CLP in different use cases. +These guides cover deploying, configuring, and integrating CLP for various use cases. + +--- + +## Deployment + +Guides for deploying CLP in production environments. + +:::{tip} +For single-host deployments, see the [quick-start guide](quick-start/index), which includes tabs +for both Docker Compose and Kubernetes (kind) orchestration. +::: + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: guides-docker-compose-deployment +Docker Compose deployment +^^^ +Deploy CLP using Docker Compose for single or multi-host setups. +::: + +:::{grid-item-card} +:link: guides-k8s-deployment +Kubernetes deployment +^^^ +Deploy CLP on a Kubernetes cluster using Helm. +::: +:::: + +--- + +## Input & storage + +Guides for configuring data sources and storage backends. ::::{grid} 1 1 2 2 :gutter: 2 @@ -9,41 +44,58 @@ The guides below describe how to use CLP in different use cases. :link: guides-using-object-storage/index Using object storage ^^^ -Using CLP to ingest logs from object storage and store archives on object storage. +Ingest logs from and store archives on S3-compatible object storage. ::: :::{grid-item-card} :link: guides-external-database External database setup ^^^ -Guide for setting up external databases for CLP package components. +Use external MariaDB/MySQL and MongoDB databases. ::: :::{grid-item-card} :link: guides-retention Configuring retention periods ^^^ -How to configure retention periods for CLP archives and search results. +Configure retention periods for archives and search results. ::: +:::: + +--- + +## Package services + +Guides for using services included in the CLP package. + +::::{grid} 1 1 2 2 +:gutter: 2 :::{grid-item-card} -:link: guides-multi-host -Multi-host deployment +:link: guides-using-the-api-server +Using the API server ^^^ -How to deploy CLP across multiple hosts. +Submit queries, view results, and manage jobs programmatically. ::: :::{grid-item-card} -:link: guides-using-the-api-server -Using the API server +:link: guides-mcp-server/index +MCP server ^^^ -How to use the API server to interact with CLP. +Integrate CLP with AI assistants using the Model Context Protocol. ::: :::{grid-item-card} :link: guides-using-presto -Using Presto with CLP +Using Presto +^^^ +Use Presto for distributed SQL queries on compressed logs. +::: + +:::{grid-item-card} +:link: guides-using-spider +Using Spider ^^^ -How to use Presto to query compressed logs in CLP. +Use Spider for compression and query job task distribution. ::: :::: diff --git a/docs/src/user-docs/index.md b/docs/src/user-docs/index.md index 715dcefe55..2dd116bc42 100644 --- a/docs/src/user-docs/index.md +++ b/docs/src/user-docs/index.md @@ -57,15 +57,33 @@ quick-start/clp-text :::{toctree} :hidden: :caption: Guides -:glob: guides-overview -guides-mcp-server/index +::: + +:::{toctree} +:hidden: +:caption: Deployment + +guides-docker-compose-deployment +guides-k8s-deployment +::: + +:::{toctree} +:hidden: +:caption: Input & storage + guides-using-object-storage/index -guides-using-the-api-server guides-external-database -guides-multi-host guides-retention +::: + +:::{toctree} +:hidden: +:caption: Package services + +guides-using-the-api-server +guides-mcp-server/index guides-using-presto guides-using-spider ::: diff --git a/docs/src/user-docs/quick-start/clp-json.md b/docs/src/user-docs/quick-start/clp-json.md index 385736e339..c93e7cce90 100644 --- a/docs/src/user-docs/quick-start/clp-json.md +++ b/docs/src/user-docs/quick-start/clp-json.md @@ -11,26 +11,146 @@ text logs, refer to [this section below](#compressing-unstructured-text-logs). ## Starting CLP +::::{tab-set} +:::{tab-item} Docker Compose +:sync: docker + To start CLP, run: ```bash sbin/start-clp.sh ``` -:::{tip} +```{tip} To validate configuration and prepare directories without launching services, add the `--setup-only` flag (e.g., `sbin/start-clp.sh --setup-only`). -::: +``` -:::{note} +```{note} If CLP fails to start (e.g., due to a port conflict), try adjusting the settings in `etc/clp-config.yaml` and then run the start command again. +``` + +For more details on Docker Compose deployment, see the [Docker Compose deployment guide][docker-compose-deployment]. ::: +:::{tab-item} Kubernetes (kind) +:sync: kind + +First, create a kind cluster: + +```bash +# Data and logs directory for the CLP Package +export CLP_HOME="$HOME/clp" + +# Host port mappings +export CLP_WEBUI_PORT=30000 +export CLP_RESULTS_CACHE_PORT=30017 +export CLP_API_SERVER_PORT=30301 +export CLP_DATABASE_PORT=30306 +export CLP_MCP_SERVER_PORT=30800 + +# Credentials (generate random or use your own) +export CLP_DB_PASS=$(openssl rand -hex 16) +export CLP_DB_ROOT_PASS=$(openssl rand -hex 16) +export CLP_QUEUE_PASS=$(openssl rand -hex 16) +export CLP_REDIS_PASS=$(openssl rand -hex 16) + +# Create required directories +mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ + "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \ + "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ + "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ + "$CLP_HOME/var/log/"{garbage_collector,api_server,log_ingestor,mcp_server} \ + "$CLP_HOME/var/tmp" + +# Create the kind cluster +cat <' [ ...] * `` are paths to JSON log files or directories containing such files. * Each JSON log file should contain each log event as a - [separate JSON object](./index.md#clp-json), i.e., *not* as an array. + [separate JSON object][clp-json-format], i.e., *not* as an array. The compression script will output the compression ratio of each dataset you compress, or you can use the UI to view overall statistics. @@ -61,10 +181,29 @@ config option in `etc/clp-config.yaml` (`archive_output.storage.directory` defau :::{tip} To compress logs from object storage, see -[Using object storage](../guides-using-object-storage/index). +[Using object storage][object-storage]. ::: -## Compressing unstructured text logs +### Compressing unstructured text logs + +::::{tab-set} +:::{tab-item} Docker Compose +:sync: docker + +No additional configuration is required. +::: + +:::{tab-item} Kubernetes (kind) +:sync: kind + +Configure `etc/clp-config.yaml` to connect to the kind-deployed database: + +```yaml +database: + port: 30306 +``` +::: +:::: clp-json supports compressing unstructured text logs by converting them into JSON. To enable this conversion, run the compression script with the `--unstructured` flag: @@ -103,7 +242,7 @@ When the `--unstructured` flag is used, clp-json will always use `"timestamp"` a ### Sample logs -For some sample logs, check out the [open-source datasets](../resources-datasets). +For some sample logs, check out the [open-source datasets][datasets]. --- @@ -155,13 +294,16 @@ as well as a kv-pair with key `"msg"` and a value that matches the wildcard quer `"*write concern*"`. A complete reference for clp-json's query syntax is available on the -[syntax reference page](../reference-json-search-syntax). +[syntax reference page][json-search-syntax]. ### Searching from the UI -To search your compressed logs from CLP's UI, open [http://localhost:4000](http://localhost:4000) in -your browser (if you changed `webui.host` or `webui.port` in `etc/clp-config.yaml`, use the new -values). +To search your compressed logs from CLP's UI, open [http://localhost:4000](http://localhost:4000) +(Docker Compose) or [http://localhost:30000](http://localhost:30000) (Kubernetes) in your browser. + +:::{note} +If you changed `webui.host` or `webui.port` in the configuration, use the new values. +::: [Figure 3](#figure-3) shows the search page after running a query. @@ -177,7 +319,7 @@ values). The numbered circles in [Figure 3](#figure-3) correspond to the following elements: 1. **The query input box**. The format of your query should conform to CLP's - [JSON search syntax](../reference-json-search-syntax.md). + [JSON search syntax][json-search-syntax]. 2. **The query case-sensitivity toggle**. When turned on, CLP will search for log events that match the case of your query. 3. **The time range selector**. CLP will search for log events that are in the specified time range. @@ -196,11 +338,51 @@ The numbered circles in [Figure 3](#figure-3) correspond to the following elemen :::{note} By default, the UI will only return 1,000 of the latest search results. To perform searches which -return more results, use the [command line](#searching-from-the-command-line). +return more results, use the [command line](#searching-from-the-command-line) or +[API server](#searching-via-the-api-server). ::: +### Searching via the API server + +To search via the API server: + +```bash +curl -X POST "http://localhost:30301/query/submit" \ + -H "Content-Type: application/json" \ + -d '{ + "query_string": "", + "max_num_results": 1000, + "timestamp_begin": null, + "timestamp_end": null, + "case_sensitive": false + }' +``` + +For more details on the API, see [Using the API server][api-server]. + ### Searching from the command line +::::{tab-set} +:::{tab-item} Docker Compose +:sync: docker + +No additional configuration is required. +::: + +:::{tab-item} Kubernetes (kind) +:sync: kind + +Configure `etc/clp-config.yaml` to connect to the kind-deployed services: + +```yaml +database: + port: 30306 +results_cache: + port: 30017 +``` +::: +:::: + To search your compressed logs from the command line, run: ```bash @@ -224,8 +406,39 @@ searches are case-**sensitive** on the command line. ## Stopping CLP +::::{tab-set} +:::{tab-item} Docker Compose +:sync: docker + If you need to stop CLP, run: ```bash sbin/stop-clp.sh ``` +::: + +:::{tab-item} Kubernetes (kind) +:sync: kind + +To stop CLP, uninstall the Helm release: + +```bash +helm uninstall clp +``` + +To also delete the kind cluster: + +```bash +kind delete cluster --name clp +``` +::: +:::: + +[api-server]: ../guides-using-the-api-server.md +[clp-json-format]: ./index.md#clp-json +[clp-releases]: https://github.com/y-scope/clp/releases +[datasets]: ../resources-datasets +[docker-compose-deployment]: ../guides-docker-compose-deployment.md +[json-search-syntax]: ../reference-json-search-syntax.md +[k8s-deployment]: ../guides-k8s-deployment.md +[object-storage]: ../guides-using-object-storage/index diff --git a/docs/src/user-docs/quick-start/clp-text.md b/docs/src/user-docs/quick-start/clp-text.md index f1f637c66c..b21dcadc9f 100644 --- a/docs/src/user-docs/quick-start/clp-text.md +++ b/docs/src/user-docs/quick-start/clp-text.md @@ -13,23 +13,27 @@ query individual fields. This limitation will be addressed in a future version o ## Starting CLP +::::{tab-set} +:::{tab-item} Docker Compose +:sync: docker + To start CLP, run: ```bash sbin/start-clp.sh ``` -:::{tip} +```{tip} To validate configuration and prepare directories without launching services, add the `--setup-only` flag (e.g., `sbin/start-clp.sh --setup-only`). -::: +``` -:::{note} +```{note} If CLP fails to start (e.g., due to a port conflict), try adjusting the settings in `etc/clp-config.yaml` and then run the start command again. -::: +``` -:::{warning} +````{warning} **Do not comment out or remove the `package` block in `etc/clp-config.yaml`**; otherwise, the storage and query engines will default to `clp-s`, which is optimized for JSON logs rather than unstructured text logs. @@ -41,12 +45,130 @@ package: storage_engine: "clp" query_engine: "clp" ``` +```` + +For more details on Docker Compose deployment, see the [Docker Compose deployment guide][docker-compose-deployment]. +::: + +:::{tab-item} Kubernetes (kind) +:sync: kind + +First, create a kind cluster: + +```bash +# Data and logs directory for the CLP Package +export CLP_HOME="$HOME/clp" + +# Host port mappings +export CLP_WEBUI_PORT=30000 +export CLP_RESULTS_CACHE_PORT=30017 +export CLP_API_SERVER_PORT=30301 +export CLP_DATABASE_PORT=30306 +export CLP_MCP_SERVER_PORT=30800 + +# Credentials (generate random or use your own) +export CLP_DB_PASS=$(openssl rand -hex 16) +export CLP_DB_ROOT_PASS=$(openssl rand -hex 16) +export CLP_QUEUE_PASS=$(openssl rand -hex 16) +export CLP_REDIS_PASS=$(openssl rand -hex 16) + +# Create required directories +mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ + "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \ + "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ + "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ + "$CLP_HOME/var/log/"{garbage_collector,api_server,log_ingestor,mcp_server} \ + "$CLP_HOME/var/tmp" + +# Create the kind cluster +cat <", + "max_num_results": 1000, + "timestamp_begin": null, + "timestamp_end": null, + "case_sensitive": false + }' +``` + +For more details on the API, see [Using the API server][api-server]. + ### Searching from the command line +::::{tab-set} +:::{tab-item} Docker Compose +:sync: docker + +No additional configuration is required. +::: + +:::{tab-item} Kubernetes (kind) +:sync: kind + +Configure `etc/clp-config.yaml` to connect to the kind-deployed services: + +```yaml +database: + port: 30306 +results_cache: + port: 30017 +``` +::: +:::: + To search your compressed logs from the command line, run: ```bash @@ -181,8 +346,37 @@ searches are case-**sensitive** on the command line. ## Stopping CLP +::::{tab-set} +:::{tab-item} Docker Compose +:sync: docker + If you need to stop CLP, run: ```bash sbin/stop-clp.sh ``` +::: + +:::{tab-item} Kubernetes (kind) +:sync: kind + +To stop CLP, uninstall the Helm release: + +```bash +helm uninstall clp +``` + +To also delete the kind cluster: + +```bash +kind delete cluster --name clp +``` +::: +:::: + +[api-server]: ../guides-using-the-api-server.md +[clp-releases]: https://github.com/y-scope/clp/releases +[datasets]: ../resources-datasets +[docker-compose-deployment]: ../guides-docker-compose-deployment.md +[k8s-deployment]: ../guides-k8s-deployment.md +[text-search-syntax]: ../reference-text-search-syntax.md diff --git a/docs/src/user-docs/quick-start/index.md b/docs/src/user-docs/quick-start/index.md index dc165f1c6e..c1d2350773 100644 --- a/docs/src/user-docs/quick-start/index.md +++ b/docs/src/user-docs/quick-start/index.md @@ -4,35 +4,72 @@ This guide describes the following: * [CLP's system requirements](#system-requirements) * [How to choose a CLP flavor](#choosing-a-flavor) -* [How to use CLP](#using-clp). +* [How to use CLP](#using-clp) --- ## System Requirements -To run a CLP release, you'll need: +This quick start guide covers **single-host** deployment using Docker Compose or Kubernetes with +Helm. For deployments that scale across multiple machines for higher throughput, see: -* [Docker](#docker) +* [Docker Compose deployment][docker-compose-deployment] for advanced Docker Compose configurations +* [Kubernetes deployment][k8s-deployment] for production Kubernetes clusters + +Choose the requirements below based on your preferred orchestration method. + +::::{tab-set} +:::{tab-item} Docker Compose +:sync: docker + +* [Docker][Docker] * `containerd.io` >= 1.7.18 * `docker-ce` >= 27.0.3 * `docker-ce-cli` >= 27.0.3 * `docker-compose-plugin` >= 2.28.1 -### Docker - -To check whether Docker is installed on your system, run: +To check whether the required tools are installed on your system, run: ```bash -docker version +containerd --version +docker version --format '{{.Server.Version}}' +docker compose version --short ``` -If Docker isn't installed, follow [these instructions][Docker] to install it. - -NOTE: - +```{note} * If you're not running as root, ensure Docker can be run [without superuser privileges][docker-non-root]. * If you're using Docker Desktop, ensure version 4.34 or higher is installed. +``` + +::: + +:::{tab-item} Kubernetes (kind) +:sync: kind + +[kind] (Kubernetes in Docker) runs a Kubernetes cluster inside Docker containers, making it ideal +for local Kubernetes testing and development. + +* [Docker][Docker] (required for kind) + * `containerd.io` >= 1.7.18 + * `docker-ce` >= 27.0.3 + * `docker-ce-cli` >= 27.0.3 +* [kubectl][kubectl] >= 1.30 +* [Helm][Helm] >= 4.0 +* [kind][kind] >= 0.23 + +To check whether the tools are installed on your system, run: + +```bash +containerd --version +docker version --format '{{.Server.Version}}' +kubectl version --client --output=yaml | grep gitVersion +helm version --short +kind version +``` + +::: +:::: --- @@ -73,7 +110,7 @@ The log file above contains two log events represented by two JSON objects print other. Whitespace is ignored, so the log events could also appear with no newlines and indentation. If you're using JSON logs, download and extract the `clp-json` release from the -[Releases][clp-releases] page, then proceed to the [clp-json quick-start](./clp-json.md) guide. +[Releases][clp-releases] page, then proceed to the [clp-json quick-start][clp-json] guide. ### clp-text @@ -101,7 +138,7 @@ The log file above contains two log events, both beginning with a timestamp. The line, while the second contains multiple lines. If you're using unstructured text logs, download and extract the `clp-text` release from the -[Releases][clp-releases] page, then proceed to the [clp-text quick-start](./clp-text.md) guide. +[Releases][clp-releases] page, then proceed to the [clp-text quick-start][clp-text] guide. --- @@ -128,6 +165,13 @@ How to compress and search unstructured text logs. ::: :::: +[clp-json]: ./clp-json.md [clp-releases]: https://github.com/y-scope/clp/releases +[clp-text]: ./clp-text.md [Docker]: https://docs.docker.com/engine/install/ +[docker-compose-deployment]: ../guides-docker-compose-deployment.md [docker-non-root]: https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user +[Helm]: https://helm.sh/docs/intro/install/ +[k8s-deployment]: ../guides-k8s-deployment.md +[kind]: https://kind.sigs.k8s.io/docs/user/quick-start/#installation +[kubectl]: https://kubernetes.io/docs/tasks/tools/ From ee75a28df9c31052d867ac2186ad29cc3400c47d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 23 Dec 2025 21:07:38 -0500 Subject: [PATCH 148/155] feat(helm): Add node affinity and topology spread constraints for multinode worker deployments --- .../package-helm/templates/_helpers.tpl | 32 +++++++++++++++++++ .../compression-worker-deployment.yaml | 9 +++++- .../templates/compression-worker-logs-pv.yaml | 2 +- .../templates/compression-worker-tmp-pv.yaml | 2 +- .../templates/query-worker-deployment.yaml | 9 +++++- .../templates/query-worker-logs-pv.yaml | 2 +- .../templates/shared-data-archives-pv.yaml | 2 +- .../templates/shared-data-streams-pv.yaml | 2 +- tools/deployment/package-helm/values.yaml | 4 +++ 9 files changed, 57 insertions(+), 7 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 882c8a3b17..a023ac2189 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -247,6 +247,38 @@ hostPath: type: "Directory" {{- end }} +{{/* +Creates a nodeAffinity that prefers scheduling on worker nodes. + +@return {string} YAML-formatted nodeAffinity definition +*/}} +{{- define "clp.preferWorkerNodeAffinity" -}} +nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/worker" + operator: "Exists" +{{- end }} + +{{/* +Creates a topologySpreadConstraint to ensure at most one pod per node. + +@param {object} root Root template context +@param {string} component The component name (e.g., "compression-worker", "query-worker") +@return {string} YAML-formatted topologySpreadConstraint definition +*/}} +{{- define "clp.uniquePodPerNodeConstraint" -}} +maxSkew: 1 +topologyKey: "kubernetes.io/hostname" +whenUnsatisfiable: "DoNotSchedule" +labelSelector: + matchLabels: + {{- include "clp.selectorLabels" .root | nindent 4 }} + app.kubernetes.io/component: {{ .component | quote }} +{{- end }} + {{/* Creates an initContainer that waits for a Kubernetes resource to be ready. diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index d8d3308353..2897ada94b 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-worker" spec: - replicas: 1 + replicas: {{ .Values.replicas.compressionWorker }} selector: matchLabels: {{- include "clp.selectorLabels" . | nindent 6 }} @@ -17,6 +17,13 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "compression-worker" spec: + affinity: + {{- include "clp.preferWorkerNodeAffinity" . | nindent 8 }} + topologySpreadConstraints: + - {{- include "clp.uniquePodPerNodeConstraint" (dict + "root" . + "component" "compression-worker" + ) | nindent 10 }} terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index 4b6d55466b..02955db86e 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -2,7 +2,7 @@ "root" . "component_category" "compression-worker" "name" "logs" - "nodeRole" "control-plane" + "nodeRole" "worker" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression_worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml index 43e46c2503..e73a08add5 100644 --- a/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml @@ -2,7 +2,7 @@ "root" . "component_category" "compression-worker" "name" "tmp" - "nodeRole" "control-plane" + "nodeRole" "worker" "capacity" "10Gi" "accessModes" (list "ReadWriteOnce") "hostPath" .Values.clpConfig.tmp_directory diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index cdac3c390e..19151a259c 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-worker" spec: - replicas: 1 + replicas: {{ .Values.replicas.queryWorker }} selector: matchLabels: {{- include "clp.selectorLabels" . | nindent 6 }} @@ -17,6 +17,13 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "query-worker" spec: + affinity: + {{- include "clp.preferWorkerNodeAffinity" . | nindent 8 }} + topologySpreadConstraints: + - {{- include "clp.uniquePodPerNodeConstraint" (dict + "root" . + "component" "query-worker" + ) | nindent 10 }} terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index 4f602a8c74..f7a04c5061 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -2,7 +2,7 @@ "root" . "component_category" "query-worker" "name" "logs" - "nodeRole" "control-plane" + "nodeRole" "worker" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query_worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml index 3e17545925..06850b9867 100644 --- a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml @@ -2,7 +2,7 @@ "root" . "component_category" "shared-data" "name" "archives" - "nodeRole" "control-plane" + "nodeRole" "worker" "capacity" "50Gi" "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.archive_output.storage.directory diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml index 8cc2cd8019..50c6f35de8 100644 --- a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml @@ -2,7 +2,7 @@ "root" . "component_category" "shared-data" "name" "streams" - "nodeRole" "control-plane" + "nodeRole" "worker" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.stream_output.storage.directory diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 5fbee3177c..a803ae795d 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -20,6 +20,10 @@ image: workerConcurrency: 8 +replicas: + compressionWorker: 1 + queryWorker: 1 + clpConfig: package: storage_engine: "clp" From 7469e5cf1fff88c584362c3653f845c126623a2c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 26 Dec 2025 01:26:58 -0500 Subject: [PATCH 149/155] fix(helm): Update default storage and query engine values in `values.yaml` --- tools/deployment/package-helm/values.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index a803ae795d..e3692f692f 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -18,6 +18,7 @@ image: pullPolicy: "Always" tag: "main" +# Number of concurrent processes per worker pod. workerConcurrency: 8 replicas: @@ -26,8 +27,8 @@ replicas: clpConfig: package: - storage_engine: "clp" - query_engine: "clp" + storage_engine: "clp-s" + query_engine: "clp-s" # Location (e.g., directory) containing any logs you wish to compress. Must be reachable by all # workers. From 0b8a6951abad8868bbdba9be83e809e8068cfdcc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 26 Dec 2025 02:18:33 -0500 Subject: [PATCH 150/155] docs: Improve orchestration docs and fix broken links in deployment guides --- .../design-deployment-orchestration.md | 228 +++++++++--------- .../src/user-docs/guides-external-database.md | 2 +- 2 files changed, 115 insertions(+), 115 deletions(-) diff --git a/docs/src/dev-docs/design-deployment-orchestration.md b/docs/src/dev-docs/design-deployment-orchestration.md index eea5facd0f..51efa05788 100755 --- a/docs/src/dev-docs/design-deployment-orchestration.md +++ b/docs/src/dev-docs/design-deployment-orchestration.md @@ -22,120 +22,120 @@ one-time initialization jobs and their functions. :::{mermaid} %%{ -init: { -"theme": "base", -"themeVariables": { -"primaryColor": "#0066cc", -"primaryTextColor": "#fff", -"primaryBorderColor": "transparent", -"lineColor": "#007fff", -"secondaryColor": "#007fff", -"tertiaryColor": "#fff" -} -} + init: { + "theme": "base", + "themeVariables": { + "primaryColor": "#0066cc", + "primaryTextColor": "#fff", + "primaryBorderColor": "transparent", + "lineColor": "#007fff", + "secondaryColor": "#007fff", + "tertiaryColor": "#fff" + } + } }%% graph LR -%% Services -database["database (MySQL)"] -queue["queue (RabbitMQ)"] -redis["redis (Redis)"] -results_cache["results-cache (MongoDB)"] -compression_scheduler["compression-scheduler"] -query_scheduler["query-scheduler"] -spider_scheduler["spider-scheduler"] -compression_worker["compression-worker"] -spider_compression_worker["spider-compression-worker"] -query_worker["query-worker"] -reducer["reducer"] -api_server["api-server"] -garbage_collector["garbage-collector"] -webui["webui"] -mcp_server["mcp-server"] -log_ingestor["log-ingestor"] - -%% One-time jobs -db_table_creator["db-table-creator"] -results_cache_indices_creator["results-cache-indices-creator"] - -%% Dependencies -%% Link 0-1: Database --> Database initialization jobs -database -->|healthy| db_table_creator -results_cache -->|healthy| results_cache_indices_creator -linkStyle 0,1 stroke:#ffa500 - -%% Link 2-5: Celery dependencies --> Schedulers -queue -->|healthy| compression_scheduler -redis -->|healthy| compression_scheduler -queue -->|healthy| query_scheduler -redis -->|healthy| query_scheduler -linkStyle 2,3,4,5 stroke:#ff0000 - -%% Link 6: Schedulers --> Workers -query_scheduler -->|healthy| reducer -linkStyle 6 stroke:#800080 - -%% Link 7-15: Database initialization job --> Services -db_table_creator -->|completed_successfully| api_server -db_table_creator -->|completed_successfully| compression_scheduler -db_table_creator -->|completed_successfully| garbage_collector -db_table_creator -->|completed_successfully| log_ingestor -db_table_creator -->|completed_successfully| mcp_server -db_table_creator -->|completed_successfully| query_scheduler -db_table_creator -->|completed_successfully| spider_compression_worker -db_table_creator -->|completed_successfully| spider_scheduler -db_table_creator -->|completed_successfully| webui -linkStyle 7,8,9,10,11,12,13,14,15 stroke:#0000ff - -%% Link 16-20: Results cache initialization job --> Services -results_cache_indices_creator -->|completed_successfully| api_server -results_cache_indices_creator -->|completed_successfully| garbage_collector -results_cache_indices_creator -->|completed_successfully| mcp_server -results_cache_indices_creator -->|completed_successfully| reducer -results_cache_indices_creator -->|completed_successfully| webui -linkStyle 16,17,18,19,20 stroke:#008000 - -subgraph Databases -database -results_cache -subgraph celery_dependencies[Celery Dependencies] -queue -redis -end -end - -subgraph Initialization jobs -db_table_creator -results_cache_indices_creator -end - -subgraph Schedulers -compression_scheduler -query_scheduler -spider_scheduler -end - -subgraph Workers -compression_worker -spider_compression_worker -query_worker -reducer -end - -subgraph Management & UI -api_server -log_ingestor -garbage_collector -webui -end - -subgraph AI -mcp_server -end - -%% Subgraph styles -style celery_dependencies fill:#ffffe0 -style spider_compression_worker fill:#008080 -style spider_scheduler fill:#008080 + %% Services + database["database (MySQL)"] + queue["queue (RabbitMQ)"] + redis["redis (Redis)"] + results_cache["results-cache (MongoDB)"] + compression_scheduler["compression-scheduler"] + query_scheduler["query-scheduler"] + spider_scheduler["spider-scheduler"] + compression_worker["compression-worker"] + spider_compression_worker["spider-compression-worker"] + query_worker["query-worker"] + reducer["reducer"] + api_server["api-server"] + garbage_collector["garbage-collector"] + webui["webui"] + mcp_server["mcp-server"] + log_ingestor["log-ingestor"] + + %% One-time jobs + db_table_creator["db-table-creator"] + results_cache_indices_creator["results-cache-indices-creator"] + + %% Dependencies + %% Link 0-1: Database --> Database initialization jobs + database -->|healthy| db_table_creator + results_cache -->|healthy| results_cache_indices_creator + linkStyle 0,1 stroke:#ffa500 + + %% Link 2-5: Celery dependencies --> Schedulers + queue -->|healthy| compression_scheduler + redis -->|healthy| compression_scheduler + queue -->|healthy| query_scheduler + redis -->|healthy| query_scheduler + linkStyle 2,3,4,5 stroke:#ff0000 + + %% Link 6: Schedulers --> Workers + query_scheduler -->|healthy| reducer + linkStyle 6 stroke:#800080 + + %% Link 7-15: Database initialization job --> Services + db_table_creator -->|completed_successfully| api_server + db_table_creator -->|completed_successfully| compression_scheduler + db_table_creator -->|completed_successfully| garbage_collector + db_table_creator -->|completed_successfully| log_ingestor + db_table_creator -->|completed_successfully| mcp_server + db_table_creator -->|completed_successfully| query_scheduler + db_table_creator -->|completed_successfully| spider_compression_worker + db_table_creator -->|completed_successfully| spider_scheduler + db_table_creator -->|completed_successfully| webui + linkStyle 7,8,9,10,11,12,13,14,15 stroke:#0000ff + + %% Link 16-20: Results cache initialization job --> Services + results_cache_indices_creator -->|completed_successfully| api_server + results_cache_indices_creator -->|completed_successfully| garbage_collector + results_cache_indices_creator -->|completed_successfully| mcp_server + results_cache_indices_creator -->|completed_successfully| reducer + results_cache_indices_creator -->|completed_successfully| webui + linkStyle 16,17,18,19,20 stroke:#008000 + + subgraph Databases + database + results_cache + subgraph celery_dependencies[Celery Dependencies] + queue + redis + end + end + + subgraph Initialization jobs + db_table_creator + results_cache_indices_creator + end + + subgraph Schedulers + compression_scheduler + query_scheduler + spider_scheduler + end + + subgraph Workers + compression_worker + spider_compression_worker + query_worker + reducer + end + + subgraph Management & UI + api_server + log_ingestor + garbage_collector + webui + end + + subgraph AI + mcp_server + end + + %% Subgraph styles + style celery_dependencies fill:#ffffe0 + style spider_compression_worker fill:#008080 + style spider_scheduler fill:#008080 +++ **Figure 1**: Orchestration architecture of the services in the CLP package. @@ -272,7 +272,7 @@ When issues arise, use the appropriate commands for your orchestration method: ## User guides * [Kubernetes deployment][kubernetes-guide]: Deploying CLP with Helm -* [Multi-host deployment][multi-host]: Manual Docker Compose across multiple hosts +* [Multi-host deployment][docker-compose-multi-host]: Manual Docker Compose across multiple hosts [docker-compose]: https://docs.docker.com/compose/ [docker-compose-debugging]: ../user-docs/guides-docker-compose-deployment.md#monitoring-and-debugging @@ -280,5 +280,5 @@ When issues arise, use the appropriate commands for your orchestration method: [kubernetes]: https://kubernetes.io/ [kubernetes-debugging]: ../user-docs/guides-k8s-deployment.md#monitoring-and-debugging [kubernetes-guide]: ../user-docs/guides-k8s-deployment.md -[multi-host]: ../user-docs/guides-docker-compose-deployment.md +[docker-compose-multi-host]: ../user-docs/guides-docker-compose-deployment.md#multi-host-deployment [presto-integration]: ../user-docs/guides-using-presto.md diff --git a/docs/src/user-docs/guides-external-database.md b/docs/src/user-docs/guides-external-database.md index 3c435771e3..dcfb27b896 100644 --- a/docs/src/user-docs/guides-external-database.md +++ b/docs/src/user-docs/guides-external-database.md @@ -205,7 +205,7 @@ initialization jobs (`db-table-creator` and `results-cache-indices-creator`). [aws-rds]: https://aws.amazon.com/rds/ [azure-databases]: https://azure.microsoft.com/en-us/products/category/databases -[docker-compose-orchestration]: ../dev-docs/design-deployment-orchestration.md#docker-compose-orchestration +[docker-compose-orchestration]: ../user-docs/guides-docker-compose-deployment.md [mongodb-install]: https://www.mongodb.com/docs/manual/installation/ [mongodb-security]: https://docs.mongodb.com/manual/security/ [multi-host-guide]: guides-docker-compose-deployment.md#starting-clp From eb55b31ecd9b3f726fda4f3ae4259fbe0026bc38 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 26 Dec 2025 02:19:41 -0500 Subject: [PATCH 151/155] feat(helm): Proper multi-node support: Refactor storage provisioning templates, add distributed mode support, and enhance worker scheduling options. --- docs/src/user-docs/guides-k8s-deployment.md | 99 ++++++++----- tools/deployment/package-helm/.test-common.sh | 109 ++++++++++++++ .../package-helm/templates/_helpers.tpl | 87 ++++++----- .../templates/api-server-logs-pv.yaml | 3 +- .../compression-scheduler-logs-pv.yaml | 3 +- .../compression-scheduler-user-logs-pv.yaml | 3 +- .../compression-worker-deployment.yaml | 13 +- .../templates/compression-worker-logs-pv.yaml | 3 +- ...compression-worker-staged-archives-pv.yaml | 3 +- .../templates/compression-worker-tmp-pv.yaml | 3 +- .../templates/database-data-pv.yaml | 3 +- .../templates/database-logs-pv.yaml | 3 +- .../templates/garbage-collector-logs-pv.yaml | 3 +- .../templates/log-ingestor-logs-pv.yaml | 3 +- .../templates/mcp-server-logs-pv.yaml | 3 +- .../templates/query-scheduler-logs-pv.yaml | 3 +- .../templates/query-worker-deployment.yaml | 13 +- .../templates/query-worker-logs-pv.yaml | 3 +- .../query-worker-staged-streams-pv.yaml | 3 +- .../package-helm/templates/queue-logs-pv.yaml | 3 +- .../package-helm/templates/redis-data-pv.yaml | 3 +- .../package-helm/templates/redis-logs-pv.yaml | 3 +- .../templates/reducer-logs-pv.yaml | 3 +- .../templates/results-cache-data-pv.yaml | 3 +- .../templates/results-cache-logs-pv.yaml | 3 +- .../templates/shared-data-archives-pv.yaml | 3 +- .../templates/shared-data-streams-pv.yaml | 3 +- .../package-helm/templates/storage-class.yaml | 10 ++ tools/deployment/package-helm/test-multi.sh | 84 +++++++++++ tools/deployment/package-helm/test.sh | 135 +++++------------- tools/deployment/package-helm/values.yaml | 47 +++++- 31 files changed, 430 insertions(+), 233 deletions(-) create mode 100644 tools/deployment/package-helm/.test-common.sh create mode 100644 tools/deployment/package-helm/templates/storage-class.yaml create mode 100755 tools/deployment/package-helm/test-multi.sh diff --git a/docs/src/user-docs/guides-k8s-deployment.md b/docs/src/user-docs/guides-k8s-deployment.md index 211f360d0c..689b41260d 100644 --- a/docs/src/user-docs/guides-k8s-deployment.md +++ b/docs/src/user-docs/guides-k8s-deployment.md @@ -102,36 +102,6 @@ and kubeadm on all nodes. If you lost the command, regenerate it on the control-plane node with `kubeadm token create --print-join-command`. -4. **Label worker nodes** (on the control-plane node): - - CLP's Helm chart uses node affinity to schedule PersistentVolumes. Worker nodes must be labeled - so that storage can be properly allocated: - - ```bash - # List nodes to get their names - kubectl get nodes - - # Label each worker node - kubectl label node node-role.kubernetes.io/worker= - - # Verify labels - kubectl get nodes --show-labels - ``` - -5. **Allow scheduling on control-plane**: - - By default, kubeadm taints the control-plane node to prevent workloads from running on it. You - must remove this taint so CLP pods can be scheduled: - - ```bash - kubectl taint nodes --selector node-role.kubernetes.io/control-plane= node-role.kubernetes.io/control-plane- - ``` - - :::{note} - The CLP Helm chart currently requires some pods (database, queue, redis) to run on the - control-plane node. We plan to address this limitation in a future release. - ::: - --- ## Installing the Helm chart @@ -235,8 +205,67 @@ helm install clp . \ --set credentials.database.root_password="$CLP_DB_ROOT_PASS" \ --set credentials.queue.password="$CLP_QUEUE_PASS" \ --set credentials.redis.password="$CLP_REDIS_PASS" \ - --set replicas.compressionWorker="$CLP_COMPRESSION_WORKER_REPLICAS" \ - --set replicas.queryWorker="$CLP_QUERY_WORKER_REPLICAS" + --set compressionWorker.replicas="$CLP_COMPRESSION_WORKER_REPLICAS" \ + --set queryWorker.replicas="$CLP_QUERY_WORKER_REPLICAS" +``` + +### Multi-node deployment + +For multi-node clusters with shared storage mounted on all nodes (e.g., NFS/CephFS via +`/etc/fstab`), enable distributed storage mode and configure multiple worker replicas: + +```bash +helm install clp . \ + --set distributed=true \ + --set compressionWorker.replicas=3 \ + --set queryWorker.replicas=3 +``` + +#### Worker scheduling + +You can control where compression and query workers are scheduled using standard Kubernetes +scheduling primitives (`nodeSelector`, `affinity`, `tolerations`, `topologySpreadConstraints`). + +First, label your nodes: + +```bash +kubectl label nodes yscope.io/nodeType=compute +``` + +Then configure worker scheduling: + +```{code-block} yaml +:caption: worker-scheduling.yaml + +compressionWorker: + replicas: 2 + scheduling: + nodeSelector: + yscope.io/nodeType: compute + tolerations: + - key: "yscope.io/dedicated" + operator: "Equal" + value: "compression" + effect: "NoSchedule" + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: "DoNotSchedule" + labelSelector: + matchLabels: + app.kubernetes.io/component: compression-worker + +queryWorker: + replicas: 2 + scheduling: + nodeSelector: + yscope.io/nodeType: compute +``` + +Install with the scheduling configuration: + +```bash +helm install clp . -f worker-scheduling.yaml --set distributed=true ``` ### Installation with custom values @@ -317,6 +346,12 @@ chart directory. | `image.clpPackage.repository` | CLP package image repository | `ghcr.io/y-scope/clp/clp-package` | | `image.clpPackage.tag` | Image tag | `main` | | `workerConcurrency` | Number of worker processes | `8` | +| `distributed` | Distributed/multi-node deployment mode | `false` | +| `compressionWorker.replicas` | Number of compression worker replicas | `1` | +| `compressionWorker.scheduling` | Scheduling config for compression workers | `{}` | +| `queryWorker.replicas` | Number of query worker replicas | `1` | +| `queryWorker.scheduling` | Scheduling config for query workers | `{}` | +| `storage.storageClassName` | StorageClass name (created if "local-storage") | `local-storage` | | `allowHostAccessForSbinScripts` | Expose database/cache for sbin scripts | `true` | | `clpConfig.package.storage_engine` | Storage engine (`clp-s` or `clp`) | `clp-s` | | `clpConfig.package.query_engine` | Query engine (`clp-s`, `clp`, or `presto`) | `clp-s` | diff --git a/tools/deployment/package-helm/.test-common.sh b/tools/deployment/package-helm/.test-common.sh new file mode 100644 index 0000000000..3333fdc544 --- /dev/null +++ b/tools/deployment/package-helm/.test-common.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +# Common utilities for CLP Helm chart tests + +set -o errexit +set -o nounset +set -o pipefail + +CLP_HOME="${CLP_HOME:-/tmp/clp}" + +# Waits for all jobs to complete and all non-job pods to be ready. +# +# @param {int} timeout_seconds Overall timeout in seconds +# @param {int} poll_interval_seconds Interval between status checks +# @param {int} wait_timeout_seconds Timeout for each kubectl wait call +# @param {string} kubectl_get_opts Additional options for kubectl get pods (optional) +# @return {int} 0 on success, 1 on timeout +wait_for_pods() { + local timeout_seconds=$1 + local poll_interval_seconds=$2 + local wait_timeout_seconds=$3 + local kubectl_get_opts="${4:-}" + + echo "Waiting for all pods to be ready" \ + "(timeout=${timeout_seconds}s, poll=${poll_interval_seconds}s," \ + "wait=${wait_timeout_seconds}s)..." + + # Reset bash built-in SECONDS counter + SECONDS=0 + + while true; do + sleep "${poll_interval_seconds}" + # shellcheck disable=SC2086 + kubectl get pods ${kubectl_get_opts} + + if kubectl wait job \ + --all \ + --for=condition=Complete \ + --timeout="${wait_timeout_seconds}s" 2>/dev/null \ + && kubectl wait pods \ + --all \ + --selector='!job-name' \ + --for=condition=Ready \ + --timeout="${wait_timeout_seconds}s" 2>/dev/null + then + echo "All jobs completed and services are ready." + return 0 + fi + + if [[ ${SECONDS} -ge ${timeout_seconds} ]]; then + echo "ERROR: Timed out waiting for pods to be ready" + return 1 + fi + + echo "---" + done +} + +# Initializes the CLP home directory structure. +init_clp_home() { + rm -rf "$CLP_HOME" + mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ + "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \ + "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ + "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ + "$CLP_HOME/var/log/"{garbage_collector,api_server,log_ingestor,mcp_server} \ + "$CLP_HOME/var/tmp" \ + "$CLP_HOME/samples" +} + +# Downloads sample datasets in the background. +# Sets SAMPLE_DOWNLOAD_PID to the background process ID. +download_samples() { + wget -O - https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1 \ + | tar xz -C "$CLP_HOME/samples" & + SAMPLE_DOWNLOAD_PID=$! +} + +# Waits for sample download to complete. +wait_for_samples() { + wait "$SAMPLE_DOWNLOAD_PID" + echo "Sample download and extraction complete" +} + +# Generates kind extra port mappings for control-plane node. +# These are the NodePort services exposed by the Helm chart. +generate_kind_port_mappings() { + cat <<'EOF' + extraPortMappings: + - containerPort: 30306 + hostPort: 30306 + protocol: TCP + - containerPort: 30017 + hostPort: 30017 + protocol: TCP + - containerPort: 30000 + hostPort: 30000 + protocol: TCP + - containerPort: 30301 + hostPort: 30301 + protocol: TCP + - containerPort: 30302 + hostPort: 30302 + protocol: TCP + - containerPort: 30800 + hostPort: 30800 + protocol: TCP +EOF +} diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index a023ac2189..f948742c86 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -23,9 +23,9 @@ used as a full name. {{- .Release.Name | trunc 63 | trimSuffix "-" }} {{- else }} {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} +{{- end }}{{/* if contains $name .Release.Name */}} +{{- end }}{{/* if .Values.fullnameOverride */}} +{{- end }}{{/* define "clp.fullname" */}} {{/* Creates chart name and version as used by the chart label. @@ -111,20 +111,17 @@ Used for: {{- end }} {{/* -Creates a local PersistentVolume. +Creates a PersistentVolume that does not use dynamic provisioning. @param {object} root Root template context @param {string} component_category (e.g., "database", "shared-data") @param {string} name (e.g., "archives", "data", "logs") -@param {string} nodeRole Node role for affinity. Targets nodes with label - "node-role.kubernetes.io/". Always falls back to - "node-role.kubernetes.io/control-plane" @param {string} capacity Storage capacity @param {string[]} accessModes Access modes @param {string} hostPath Absolute path on host @return {string} YAML-formatted PersistentVolume resource */}} -{{- define "clp.createLocalPv" -}} +{{- define "clp.createStaticPv" -}} apiVersion: "v1" kind: "PersistentVolume" metadata: @@ -137,19 +134,22 @@ spec: storage: {{ .capacity }} accessModes: {{ .accessModes }} persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" + storageClassName: {{ .root.Values.storage.storageClassName | quote }} + {{- if .root.Values.distributed }} + hostPath: + path: {{ .hostPath | quote }} + type: "DirectoryOrCreate" + {{- else }} local: path: {{ .hostPath | quote }} nodeAffinity: required: nodeSelectorTerms: - - matchExpressions: - - key: {{ printf "node-role.kubernetes.io/%s" .nodeRole | quote }} - operator: "Exists" - matchExpressions: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" -{{- end }} + {{- end }}{{/* if .root.Values.distributed */}} +{{- end }}{{/* define "clp.createStaticPv" */}} {{/* Creates a PersistentVolumeClaim for the given component. @@ -171,7 +171,7 @@ metadata: app.kubernetes.io/component: {{ .component_category | quote }} spec: accessModes: {{ .accessModes }} - storageClassName: "local-storage" + storageClassName: {{ .root.Values.storage.storageClassName | quote }} selector: matchLabels: {{- include "clp.selectorLabels" .root | nindent 6 }} @@ -248,36 +248,45 @@ hostPath: {{- end }} {{/* -Creates a nodeAffinity that prefers scheduling on worker nodes. +Creates scheduling configuration (nodeSelector, affinity, tolerations, topologySpreadConstraints) +for a component. -@return {string} YAML-formatted nodeAffinity definition -*/}} -{{- define "clp.preferWorkerNodeAffinity" -}} -nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: "node-role.kubernetes.io/worker" - operator: "Exists" -{{- end }} - -{{/* -Creates a topologySpreadConstraint to ensure at most one pod per node. +When distributed is false (single-node mode), a control-plane toleration is automatically added +so pods can be scheduled on tainted control-plane nodes without manual untainting. @param {object} root Root template context -@param {string} component The component name (e.g., "compression-worker", "query-worker") -@return {string} YAML-formatted topologySpreadConstraint definition +@param {string} component Top-level values key (e.g., "compressionWorker", "queryWorker") +@return {string} YAML-formatted scheduling fields (nodeSelector, affinity, tolerations, + topologySpreadConstraints) */}} -{{- define "clp.uniquePodPerNodeConstraint" -}} -maxSkew: 1 -topologyKey: "kubernetes.io/hostname" -whenUnsatisfiable: "DoNotSchedule" -labelSelector: - matchLabels: - {{- include "clp.selectorLabels" .root | nindent 4 }} - app.kubernetes.io/component: {{ .component | quote }} +{{- define "clp.createSchedulingConfigs" -}} +{{- $componentConfig := index .root.Values .component | default dict -}} +{{- $scheduling := $componentConfig.scheduling | default dict -}} +{{- $tolerations := $scheduling.tolerations | default list -}} +{{- if not .root.Values.distributed -}} +{{- $tolerations = append $tolerations (dict + "key" "node-role.kubernetes.io/control-plane" + "operator" "Exists" + "effect" "NoSchedule" +) -}} +{{- end -}} +{{- with $scheduling.nodeSelector }} +nodeSelector: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with $scheduling.affinity }} +affinity: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with $tolerations }} +tolerations: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with $scheduling.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} {{- end }} +{{- end }}{{/* define "clp.createSchedulingConfigs" */}} {{/* Creates an initContainer that waits for a Kubernetes resource to be ready. diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index 89fd30dda8..e117f83bd1 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1,9 +1,8 @@ {{- if .Values.clpConfig.api_server }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "api-server" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/api_server" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 565f5dc40b..0a886c7c53 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "compression-scheduler" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression_scheduler" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml index 9b667a58bf..2e30551fbd 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "compression-scheduler" "name" "user-logs" - "nodeRole" "control-plane" "capacity" "10Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/user" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 2897ada94b..192d521ba7 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-worker" spec: - replicas: {{ .Values.replicas.compressionWorker }} + replicas: {{ .Values.compressionWorker.replicas }} selector: matchLabels: {{- include "clp.selectorLabels" . | nindent 6 }} @@ -17,13 +17,10 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "compression-worker" spec: - affinity: - {{- include "clp.preferWorkerNodeAffinity" . | nindent 8 }} - topologySpreadConstraints: - - {{- include "clp.uniquePodPerNodeConstraint" (dict - "root" . - "component" "compression-worker" - ) | nindent 10 }} + {{- include "clp.createSchedulingConfigs" (dict + "root" . + "component" "compressionWorker" + ) | nindent 6 }} terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index 02955db86e..55f243cf8c 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "compression-worker" "name" "logs" - "nodeRole" "worker" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression_worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index c00226b797..26f20ed732 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -1,9 +1,8 @@ {{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "compression-worker" "name" "staged-archives" - "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/staged-archives" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml index e73a08add5..d7107e14ae 100644 --- a/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "compression-worker" "name" "tmp" - "nodeRole" "worker" "capacity" "10Gi" "accessModes" (list "ReadWriteOnce") "hostPath" .Values.clpConfig.tmp_directory diff --git a/tools/deployment/package-helm/templates/database-data-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml index 3bd6ea2b9a..1456cb9f9f 100644 --- a/tools/deployment/package-helm/templates/database-data-pv.yaml +++ b/tools/deployment/package-helm/templates/database-data-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "database" "name" "data" - "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/database" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml index 794215cf3a..c9f2e63793 100644 --- a/tools/deployment/package-helm/templates/database-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "database" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/database" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index 47c5417a64..31a3484de4 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,10 +1,9 @@ {{- if or .Values.clpConfig.archive_output.retention_period .Values.clpConfig.results_cache.retention_period }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "garbage-collector" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/garbage_collector" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml b/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml index 1abb5d03c4..0324e4127e 100644 --- a/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml @@ -1,9 +1,8 @@ {{- if eq .Values.clpConfig.logs_input.type "s3" }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "log-ingestor" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/log_ingestor" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index 9c53c9c7f1..fc297f5020 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -1,9 +1,8 @@ {{- if .Values.clpConfig.mcp_server }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "mcp-server" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/mcp_server" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index de7633da55..bc66ac1ae3 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "query-scheduler" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query_scheduler" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 19151a259c..0fed67a154 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-worker" spec: - replicas: {{ .Values.replicas.queryWorker }} + replicas: {{ .Values.queryWorker.replicas }} selector: matchLabels: {{- include "clp.selectorLabels" . | nindent 6 }} @@ -17,13 +17,10 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "query-worker" spec: - affinity: - {{- include "clp.preferWorkerNodeAffinity" . | nindent 8 }} - topologySpreadConstraints: - - {{- include "clp.uniquePodPerNodeConstraint" (dict - "root" . - "component" "query-worker" - ) | nindent 10 }} + {{- include "clp.createSchedulingConfigs" (dict + "root" . + "component" "queryWorker" + ) | nindent 6 }} terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index f7a04c5061..1fd12c2e84 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "query-worker" "name" "logs" - "nodeRole" "worker" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query_worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index c3f14f6b02..a404a36f81 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -1,9 +1,8 @@ {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "query-worker" "name" "staged-streams" - "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml index c9315630f5..37f45a5d4e 100644 --- a/tools/deployment/package-helm/templates/queue-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "queue" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/queue" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml index 56efc9d19c..5e30c1c1c4 100644 --- a/tools/deployment/package-helm/templates/redis-data-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "redis" "name" "data" - "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/redis" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml index 7f03c8cdad..811b48a4d8 100644 --- a/tools/deployment/package-helm/templates/redis-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "redis" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/redis" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index 0aab54233a..4c467877ae 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "reducer" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/reducer" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml index 8410734b2e..9e8ccea38e 100644 --- a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "results-cache" "name" "data" - "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/results_cache" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index e76aaf90af..2064685ee0 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "results-cache" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/results_cache" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml index 06850b9867..4260827682 100644 --- a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "shared-data" "name" "archives" - "nodeRole" "worker" "capacity" "50Gi" "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.archive_output.storage.directory diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml index 50c6f35de8..d235870390 100644 --- a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "shared-data" "name" "streams" - "nodeRole" "worker" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.stream_output.storage.directory diff --git a/tools/deployment/package-helm/templates/storage-class.yaml b/tools/deployment/package-helm/templates/storage-class.yaml new file mode 100644 index 0000000000..acbb3c4334 --- /dev/null +++ b/tools/deployment/package-helm/templates/storage-class.yaml @@ -0,0 +1,10 @@ +{{- if eq .Values.storage.storageClassName "local-storage" }} +apiVersion: "storage.k8s.io/v1" +kind: "StorageClass" +metadata: + name: "local-storage" + labels: + {{- include "clp.labels" . | nindent 4 }} +provisioner: "kubernetes.io/no-provisioner" +volumeBindingMode: "WaitForFirstConsumer" +{{- end }} diff --git a/tools/deployment/package-helm/test-multi.sh b/tools/deployment/package-helm/test-multi.sh new file mode 100755 index 0000000000..712aedfb4a --- /dev/null +++ b/tools/deployment/package-helm/test-multi.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +# Multi-node cluster test for CLP Helm chart +# Tests distributed storage mode with multiple worker replicas + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.test-common.sh +source "${script_dir}/.test-common.sh" + +CLUSTER_NAME="${CLUSTER_NAME:-clp-test-multi}" +NUM_WORKER_NODES="${NUM_WORKER_NODES:-2}" +COMPRESSION_WORKER_REPLICAS="${COMPRESSION_WORKER_REPLICAS:-2}" +QUERY_WORKER_REPLICAS="${QUERY_WORKER_REPLICAS:-2}" + +echo "=== Multi-node CLP Helm Chart Test ===" +echo "Worker nodes: ${NUM_WORKER_NODES}" +echo "Compression worker replicas: ${COMPRESSION_WORKER_REPLICAS}" +echo "Query worker replicas: ${QUERY_WORKER_REPLICAS}" +echo "" + +kind delete cluster --name "${CLUSTER_NAME}" 2>/dev/null || true +init_clp_home +download_samples + +echo "Creating kind cluster with 1 control-plane + ${NUM_WORKER_NODES} worker nodes..." +{ + cat </dev/null || true + +helm uninstall test --ignore-not-found 2>/dev/null || true +sleep 2 + +echo "Installing Helm chart with distributed storage mode..." +helm install test . \ + --set "distributed=true" \ + --set "compressionWorker.replicas=${COMPRESSION_WORKER_REPLICAS}" \ + --set "queryWorker.replicas=${QUERY_WORKER_REPLICAS}" + +wait_for_samples +wait_for_pods 300 5 5 + +echo "" +echo "=== Pod Distribution ===" +kubectl get pods -o wide + +echo "" +echo "To clean up:" +echo " kind delete cluster --name ${CLUSTER_NAME}" diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 69b8139904..ab70528e3b 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -1,109 +1,46 @@ #!/usr/bin/env bash +# Single-node cluster test for CLP Helm chart # TODO: Migrate into integration test -set -o errexit -set -o nounset -set -o pipefail - -CLP_HOME="/tmp/clp" - -# Waits for all jobs to complete and all non-job pods to be ready. -# -# @param {int} timeout_seconds Overall timeout in seconds -# @param {int} poll_interval_seconds Interval between status checks -# @param {int} wait_timeout_seconds Timeout for each kubectl wait call -# @return {int} 0 on success, 1 on timeout -wait_for_pods() { - local timeout_seconds=$1 - local poll_interval_seconds=$2 - local wait_timeout_seconds=$3 - - echo "Waiting for all pods to be ready" \ - "(timeout=${timeout_seconds}s, poll=${poll_interval_seconds}s," \ - "wait=${wait_timeout_seconds}s)..." - - # Reset bash built-in SECONDS counter - SECONDS=0 - - while true; do - sleep "${poll_interval_seconds}" - kubectl get pods - - if kubectl wait job \ - --all \ - --for=condition=Complete \ - --timeout="${wait_timeout_seconds}s" 2>/dev/null \ - && kubectl wait pods \ - --all \ - --selector='!job-name' \ - --for=condition=Ready \ - --timeout="${wait_timeout_seconds}s" 2>/dev/null - then - echo "All jobs completed and services are ready." - return 0 - fi - - if [[ ${SECONDS} -ge ${timeout_seconds} ]]; then - echo "ERROR: Timed out waiting for pods to be ready" - return 1 - fi - - echo "---" - done -} - -kind delete cluster --name clp-test -rm -rf "$CLP_HOME" -mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ - "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \ - "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ - "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ - "$CLP_HOME/var/log/"{garbage_collector,api_server,log_ingestor,mcp_server} \ - "$CLP_HOME/var/tmp" \ - "$CLP_HOME/samples" - -# Download sample datasets in the background -wget -O - https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1 \ - | tar xz -C "$CLP_HOME/samples" & -SAMPLE_DOWNLOAD_PID=$! - -cat </dev/null || true +init_clp_home +download_samples + +echo "Creating kind cluster..." +{ + cat </dev/null || true sleep 2 -helm install test . -wait $SAMPLE_DOWNLOAD_PID -echo "Sample download and extraction complete" +echo "Installing Helm chart..." +helm install test . +wait_for_samples wait_for_pods 300 5 5 + +echo "" +echo "To clean up:" +echo " kind delete cluster --name ${CLUSTER_NAME}" diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index e3692f692f..421d1ba006 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -18,12 +18,53 @@ image: pullPolicy: "Always" tag: "main" +# Deployment mode: +# - false: Single-node deployment. PVs use local volumes bound to one node. Pods automatically +# tolerate control-plane taints. Only works with worker replicas=1. +# - true: Multi-node deployment. PVs use hostPath without node affinity, assuming unmanaged +# shared storage (e.g., NFS/CephFS mounted via /etc/fstab) at the same path on all nodes. +distributed: false + # Number of concurrent processes per worker pod. workerConcurrency: 8 -replicas: - compressionWorker: 1 - queryWorker: 1 +compressionWorker: + replicas: 1 + # Controls which nodes run compression workers + # scheduling: + # nodeSelector: + # yscope.io/nodeType: compute + # tolerations: + # - key: "yscope.io/dedicated" + # operator: "Equal" + # value: "compression" + # effect: "NoSchedule" + # topologySpreadConstraints: + # - maxSkew: 1 + # topologyKey: "kubernetes.io/hostname" + # whenUnsatisfiable: "DoNotSchedule" + +queryWorker: + replicas: 1 + # Controls which nodes run query workers + # scheduling: + # nodeSelector: + # yscope.io/nodeType: compute + # tolerations: + # - key: "yscope.io/dedicated" + # operator: "Equal" + # value: "query" + # effect: "NoSchedule" + # topologySpreadConstraints: + # - maxSkew: 1 + # topologyKey: "kubernetes.io/hostname" + # whenUnsatisfiable: "DoNotSchedule" + +storage: + # Name of the StorageClass for PVs and PVCs. + # - If "local-storage" (default), the chart creates a StorageClass with WaitForFirstConsumer + # - If a different name, the StorageClass must already exist in your cluster + storageClassName: "local-storage" clpConfig: package: From 39e90695037e841bc3a0a5f2dbf1d48f62f37a72 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 26 Dec 2025 02:50:38 -0500 Subject: [PATCH 152/155] docs(helm): Update Kubernetes deployment guide to improve worker scheduling examples and cases Add revised examples for scheduling workers with separate node pools or shared nodes, including updated Helm configurations. Introduce a test script for multi-node deployment with dedicated workers and modify the shared worker test script accordingly. --- docs/src/user-docs/guides-k8s-deployment.md | 131 +++++++++++------- .../package-helm/test-multi-dedicated.sh | 95 +++++++++++++ .../{test-multi.sh => test-multi-shared.sh} | 4 +- 3 files changed, 181 insertions(+), 49 deletions(-) create mode 100755 tools/deployment/package-helm/test-multi-dedicated.sh rename tools/deployment/package-helm/{test-multi.sh => test-multi-shared.sh} (95%) diff --git a/docs/src/user-docs/guides-k8s-deployment.md b/docs/src/user-docs/guides-k8s-deployment.md index 689b41260d..6353211bdb 100644 --- a/docs/src/user-docs/guides-k8s-deployment.md +++ b/docs/src/user-docs/guides-k8s-deployment.md @@ -221,53 +221,6 @@ helm install clp . \ --set queryWorker.replicas=3 ``` -#### Worker scheduling - -You can control where compression and query workers are scheduled using standard Kubernetes -scheduling primitives (`nodeSelector`, `affinity`, `tolerations`, `topologySpreadConstraints`). - -First, label your nodes: - -```bash -kubectl label nodes yscope.io/nodeType=compute -``` - -Then configure worker scheduling: - -```{code-block} yaml -:caption: worker-scheduling.yaml - -compressionWorker: - replicas: 2 - scheduling: - nodeSelector: - yscope.io/nodeType: compute - tolerations: - - key: "yscope.io/dedicated" - operator: "Equal" - value: "compression" - effect: "NoSchedule" - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: "kubernetes.io/hostname" - whenUnsatisfiable: "DoNotSchedule" - labelSelector: - matchLabels: - app.kubernetes.io/component: compression-worker - -queryWorker: - replicas: 2 - scheduling: - nodeSelector: - yscope.io/nodeType: compute -``` - -Install with the scheduling configuration: - -```bash -helm install clp . -f worker-scheduling.yaml --set distributed=true -``` - ### Installation with custom values For highly customized deployments, create a values file instead of using many `--set` flags: @@ -336,6 +289,90 @@ helm template clp . -f custom-values.yaml ``` :::: +### Worker scheduling + +You can control where workers are scheduled using standard Kubernetes scheduling primitives +(`nodeSelector`, `affinity`, `tolerations`, `topologySpreadConstraints`). + +#### Dedicated node pools + +To run compression and query workers on separate node pools: + +1. Label your nodes: + + ```bash + # Label compression nodes + kubectl label nodes node1 node2 yscope.io/nodeType=compression + + # Label query nodes + kubectl label nodes node3 node4 yscope.io/nodeType=query + ``` + +2. Configure scheduling: + + ```{code-block} yaml + :caption: dedicated-scheduling.yaml + + compressionWorker: + replicas: 2 + scheduling: + nodeSelector: + yscope.io/nodeType: compression + + queryWorker: + replicas: 2 + scheduling: + nodeSelector: + yscope.io/nodeType: query + ``` + +3. Install: + + ```bash + helm install clp . -f dedicated-scheduling.yaml --set distributed=true + ``` + +#### Shared node pool + +To run both worker types on the same node pool: + +1. Label your nodes: + + ```bash + kubectl label nodes node1 node2 node3 node4 yscope.io/nodeType=compute + ``` + +2. Configure scheduling: + + ```{code-block} yaml + :caption: shared-scheduling.yaml + + compressionWorker: + replicas: 2 + scheduling: + nodeSelector: + yscope.io/nodeType: compute + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: "DoNotSchedule" + labelSelector: + matchLabels: + app.kubernetes.io/component: compression-worker + + queryWorker: + replicas: 2 + scheduling: + nodeSelector: + yscope.io/nodeType: compute + ``` + +3. Install: + + ```bash + helm install clp . -f shared-scheduling.yaml --set distributed=true + ``` + ### Common configuration options The following table lists commonly used Helm values. For a complete list, see `values.yaml` in the diff --git a/tools/deployment/package-helm/test-multi-dedicated.sh b/tools/deployment/package-helm/test-multi-dedicated.sh new file mode 100755 index 0000000000..357f76ca99 --- /dev/null +++ b/tools/deployment/package-helm/test-multi-dedicated.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +# Multi-node cluster test with dedicated worker nodes for each worker type +# Demonstrates nodeSelector scheduling with separate node pools + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.test-common.sh +source "${script_dir}/.test-common.sh" + +CLUSTER_NAME="${CLUSTER_NAME:-clp-test-dedicated}" +NUM_COMPRESSION_NODES="${NUM_COMPRESSION_NODES:-2}" +NUM_QUERY_NODES="${NUM_QUERY_NODES:-2}" +COMPRESSION_WORKER_REPLICAS="${COMPRESSION_WORKER_REPLICAS:-2}" +QUERY_WORKER_REPLICAS="${QUERY_WORKER_REPLICAS:-2}" + +echo "=== Multi-node CLP Helm Chart Test (Dedicated Workers) ===" +echo "Compression nodes: ${NUM_COMPRESSION_NODES}" +echo "Query nodes: ${NUM_QUERY_NODES}" +echo "Compression worker replicas: ${COMPRESSION_WORKER_REPLICAS}" +echo "Query worker replicas: ${QUERY_WORKER_REPLICAS}" +echo "" + +kind delete cluster --name "${CLUSTER_NAME}" 2>/dev/null || true +init_clp_home +download_samples + +total_workers=$((NUM_COMPRESSION_NODES + NUM_QUERY_NODES)) +echo "Creating kind cluster with 1 control-plane + ${total_workers} worker nodes..." +{ + cat </dev/null || true + +helm uninstall test --ignore-not-found 2>/dev/null || true +sleep 2 + +echo "Installing Helm chart with dedicated worker nodes..." +helm install test . \ + --set "distributed=true" \ + --set "compressionWorker.replicas=${COMPRESSION_WORKER_REPLICAS}" \ + --set "compressionWorker.scheduling.nodeSelector.yscope\.io/nodeType=compression" \ + --set "queryWorker.replicas=${QUERY_WORKER_REPLICAS}" \ + --set "queryWorker.scheduling.nodeSelector.yscope\.io/nodeType=query" + +wait_for_samples +wait_for_pods 300 5 5 + +echo "" +echo "=== Pod Distribution ===" +kubectl get pods -o wide + +echo "" +echo "To clean up:" +echo " kind delete cluster --name ${CLUSTER_NAME}" diff --git a/tools/deployment/package-helm/test-multi.sh b/tools/deployment/package-helm/test-multi-shared.sh similarity index 95% rename from tools/deployment/package-helm/test-multi.sh rename to tools/deployment/package-helm/test-multi-shared.sh index 712aedfb4a..d72f5e8974 100755 --- a/tools/deployment/package-helm/test-multi.sh +++ b/tools/deployment/package-helm/test-multi-shared.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# Multi-node cluster test for CLP Helm chart -# Tests distributed storage mode with multiple worker replicas +# Multi-node cluster test with shared worker nodes +# Both compression and query workers share the same node pool script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=.test-common.sh From 46587a91567be0d127af43497d288943f084e9c1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 5 Jan 2026 01:34:41 -0500 Subject: [PATCH 153/155] fix(helm): update termination grace period for api-server and mcp-server deployments to match docker compose --- .../package-helm/templates/api-server-deployment.yaml | 2 +- .../package-helm/templates/mcp-server-deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 26da76486f..a36cfde8fd 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -19,7 +19,7 @@ spec: app.kubernetes.io/component: "api-server" spec: serviceAccountName: {{ include "clp.fullname" . }}-job-watcher - terminationGracePeriodSeconds: 10 + terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 09244b1b6b..f648a6f232 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -19,7 +19,7 @@ spec: app.kubernetes.io/component: "mcp-server" spec: serviceAccountName: {{ include "clp.fullname" . }}-job-watcher - terminationGracePeriodSeconds: 10 + terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} From 1c4807fd1441642a502abc7c993a17ec3837933e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 5 Jan 2026 01:57:07 -0500 Subject: [PATCH 154/155] fix(helm): update query scheduler health checks to use named port for tcpSocket --- .../package-helm/templates/query-scheduler-deployment.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index b42c6546da..5eba71bc30 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -87,12 +87,11 @@ spec: ] readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} - tcpSocket: - port: 7000 + tcpSocket: &query-scheduler-health-check + port: "query-scheduler" livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} - tcpSocket: - port: 7000 + tcpSocket: *query-scheduler-health-check volumes: - {{- include "clp.pvcVolume" (dict "root" . From 905677c3227262d2f31738ac1b78d40f2b1561c7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 5 Jan 2026 02:10:47 -0500 Subject: [PATCH 155/155] fix order --- .../package-helm/templates/redis-statefulset.yaml | 10 +++++----- .../templates/results-cache-statefulset.yaml | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 3929561c01..96f13eec8d 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -26,11 +26,6 @@ spec: - name: "redis" image: "redis:7.2.4" imagePullPolicy: "Always" - command: [ - "redis-server", - "/etc/redis/redis.conf", - "--requirepass", "$(REDIS_PASSWORD)" - ] env: - name: "REDIS_PASSWORD" valueFrom: @@ -55,6 +50,11 @@ spec: "name" "logs" ) | quote }} mountPath: "/var/log/redis" + command: [ + "redis-server", + "/etc/redis/redis.conf", + "--requirepass", "$(REDIS_PASSWORD)" + ] readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &redis-health-check diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 56efc28ee0..bc3f5aea63 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -26,10 +26,6 @@ spec: - name: "results-cache" image: "mongo:7.0.1" imagePullPolicy: "Always" - args: [ - "--config", "/etc/mongo/mongod.conf", - "--bind_ip", "0.0.0.0" - ] ports: - name: "results-cache" containerPort: 27017 @@ -48,6 +44,10 @@ spec: "name" "logs" ) | quote }} mountPath: "/var/log/mongodb" + args: [ + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0" + ] readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &results-cache-health-check