From aa64df9d06ddd46fe71d89930856c3b6d26adc11 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 14 Nov 2025 12:42:22 -0500 Subject: [PATCH 001/149] feat(deployment): Add k8s Helm chart for CLP package deployment. --- tools/deployment/package-helm/.helmignore | 23 ++ tools/deployment/package-helm/Chart.yaml | 17 ++ .../package-helm/templates/00-local-pvs.yaml | 25 ++ .../package-helm/templates/NOTES.txt | 1 + .../package-helm/templates/_helpers.tpl | 215 ++++++++++++++++++ .../compression-scheduler-deployment.yaml | 128 +++++++++++ .../compression-worker-deployment.yaml | 143 ++++++++++++ .../package-helm/templates/configmap.yaml | 150 ++++++++++++ .../templates/database-statefulset.yaml | 133 +++++++++++ .../templates/db-table-creator-job.yaml | 58 +++++ .../templates/query-scheduler-deployment.yaml | 121 ++++++++++ .../templates/query-worker-deployment.yaml | 118 ++++++++++ .../templates/queue-statefulset.yaml | 83 +++++++ .../templates/redis-statefulset.yaml | 110 +++++++++ .../results-cache-indices-creator-job.yaml | 39 ++++ .../templates/results-cache-statefulset.yaml | 121 ++++++++++ .../package-helm/templates/secret.yaml | 13 ++ .../templates/webui-deployment.yaml | 132 +++++++++++ tools/deployment/package-helm/test.sh | 37 +++ tools/deployment/package-helm/values.yaml | 193 ++++++++++++++++ 20 files changed, 1860 insertions(+) create mode 100644 tools/deployment/package-helm/.helmignore create mode 100644 tools/deployment/package-helm/Chart.yaml create mode 100644 tools/deployment/package-helm/templates/00-local-pvs.yaml create mode 100644 tools/deployment/package-helm/templates/NOTES.txt create mode 100644 tools/deployment/package-helm/templates/_helpers.tpl create mode 100644 tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/compression-worker-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/configmap.yaml create mode 100644 tools/deployment/package-helm/templates/database-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/db-table-creator-job.yaml create mode 100644 tools/deployment/package-helm/templates/query-scheduler-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/query-worker-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/queue-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/redis-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/secret.yaml create mode 100644 tools/deployment/package-helm/templates/webui-deployment.yaml create mode 100755 tools/deployment/package-helm/test.sh create mode 100644 tools/deployment/package-helm/values.yaml diff --git a/tools/deployment/package-helm/.helmignore b/tools/deployment/package-helm/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/tools/deployment/package-helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml new file mode 100644 index 0000000000..6621aa2842 --- /dev/null +++ b/tools/deployment/package-helm/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: "v2" +name: "clp-package" +description: "A Helm chart for CLP (Compressed Log Processor) package deployment" +type: "application" +version: "0.1.0" +appVersion: "0.6.1-dev" +home: "https://github.com/y-scope/clp" +icon: "https://github.com/y-scope/clp/raw/main/docs/src/clp-logo.png" +sources: ["https://github.com/y-scope/clp"] +maintainers: [{name: "YScope Inc.", email: "dev@yscope.com"}] +keywords: + - "analytics" + - "compression" + - "log-management" + - "log-parser" + - "logging" + - "search" diff --git a/tools/deployment/package-helm/templates/00-local-pvs.yaml b/tools/deployment/package-helm/templates/00-local-pvs.yaml new file mode 100644 index 0000000000..94da93965a --- /dev/null +++ b/tools/deployment/package-helm/templates/00-local-pvs.yaml @@ -0,0 +1,25 @@ +{{- include "clp-package.createLocalDataPv" (list . "database") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "database") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "queue") }} +--- +{{- include "clp-package.createLocalDataPv" (list . "redis") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "redis") }} +--- +{{- include "clp-package.createLocalDataPv" (list . "results-cache") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "results-cache") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "compression-scheduler") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "compression-worker") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "query-scheduler") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "query-worker") }} +--- +{{- include "clp-package.createStreamsPv" . }} +--- +{{- include "clp-package.createTmpPv" . }} diff --git a/tools/deployment/package-helm/templates/NOTES.txt b/tools/deployment/package-helm/templates/NOTES.txt new file mode 100644 index 0000000000..97a261da8a --- /dev/null +++ b/tools/deployment/package-helm/templates/NOTES.txt @@ -0,0 +1 @@ +TODO: This should be filled with usage instructions. \ No newline at end of file diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl new file mode 100644 index 0000000000..a02d98809e --- /dev/null +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -0,0 +1,215 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "clp-package.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "clp-package.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "clp-package.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "clp-package.labels" -}} +helm.sh/chart: {{ include "clp-package.chart" . }} +{{ include "clp-package.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "clp-package.selectorLabels" -}} +app.kubernetes.io/name: {{ include "clp-package.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Image reference for CLP Package +*/}} +{{- define "clp-package.image.ref" -}} +{{- $tag := .Values.image.clpPackage.tag | default .Chart.AppVersion }} +{{- printf "%s:%s" .Values.image.clpPackage.repository $tag }} +{{- end }} + +{{/* +Require a value with a standard error message +*/}} +{{- define "clp-package.requireInput" -}} +{{- . | required "Please set a value." }} +{{- end }} + +{{/* +Creates timings for readiness probes (faster checks for quicker startup). +*/}} +{{- define "clp-package.readinessProbeTimings" -}} +initialDelaySeconds: 1 +periodSeconds: 1 +timeoutSeconds: 2 +failureThreshold: 3 +{{- end }} + +{{/* +Creates timings for liveness probes. +*/}} +{{- define "clp-package.livenessProbeTimings" -}} +initialDelaySeconds: 30 +periodSeconds: 10 +timeoutSeconds: 5 +failureThreshold: 3 +{{- end }} + +{{/* +CLP logs directory path on host +*/}} +{{- define "clp-package.logsDirHost" -}} +{{ .Values.storage.localPathBase }}/{{ .Values.clpConfig.logs_directory }} +{{- end }} + +{{/* +CLP data directory path on host +*/}} +{{- define "clp-package.dataDirHost" -}} +{{ .Values.storage.localPathBase }}/{{ .Values.clpConfig.data_directory }} +{{- end }} + +{{/* +Creates a local PersistentVolume for data storage. +*/}} +{{- define "clp-package.createLocalDataPv" -}} +{{- $root := index . 0 -}} +{{- $component := index . 1 -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-data-pv-{{ $component }} + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: {{ $component | quote }} +spec: + capacity: + storage: "20Gi" + accessModes: ["ReadWriteOnce"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ include "clp-package.dataDirHost" $root }}/{{ $component }} + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + +{{/* +Creates a local PersistentVolume for logs storage. +*/}} +{{- define "clp-package.createLocalLogsPv" -}} +{{- $root := index . 0 -}} +{{- $component := index . 1 -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-logs-pv-{{ $component }} + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: {{ $component | quote }} +spec: + capacity: + storage: "5Gi" + accessModes: ["ReadWriteOnce"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ include "clp-package.logsDirHost" $root }}/{{ $component }} + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + +{{/* +Creates a local PersistentVolume for streams storage (ReadWriteMany). +*/}} +{{- define "clp-package.createStreamsPv" -}} +{{- $root := . -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-streams-pv + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: "streams" +spec: + capacity: + storage: "20Gi" + accessModes: ["ReadWriteMany"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ include "clp-package.dataDirHost" $root }}/streams + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + +{{/* +Creates a local PersistentVolume for tmp storage (ReadWriteMany). +*/}} +{{- define "clp-package.createTmpPv" -}} +{{- $root := . -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-tmp-pv + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: "tmp" +spec: + capacity: + storage: "10Gi" + accessModes: ["ReadWriteMany"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ .Values.storage.localPathBase }}/var/tmp + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml new file mode 100644 index 0000000000..458cc0dc96 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -0,0 +1,128 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-compression-scheduler + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" + ports: + - port: 7000 + targetPort: 7000 +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-compression-scheduler + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-scheduler" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "compression-scheduler" + spec: + terminationGracePeriodSeconds: 300 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + - name: "wait-for-queue" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + ] + - name: "wait-for-redis" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + ] + containers: + - name: "compression-scheduler" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "BROKER_URL" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/compression_scheduler" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + - name: "RESULT_BACKEND" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "compression-scheduler-logs" + mountPath: "/var/log/compression_scheduler" + - name: "logs-input" + mountPath: "/mnt/logs" + readOnly: true + command: [ + "python3", "-u", + "-m", "job_orchestration.scheduler.compress.compression_scheduler", + "--config", "/etc/clp-config.yml" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "compression-scheduler-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-compression-scheduler-logs + - name: "logs-input" + hostPath: + path: "{{ .Values.clpConfig.logs_input.directory }}" + type: "Directory" +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-compression-scheduler-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-scheduler" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml new file mode 100644 index 0000000000..3964667b6f --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -0,0 +1,143 @@ +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-compression-worker + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-worker" +spec: + replicas: {{ .Values.workerConcurrency }} + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-worker" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "compression-worker" + spec: + terminationGracePeriodSeconds: 60 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-queue" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + ] + - name: "wait-for-redis" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + ] + containers: + - name: "compression-worker" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "BROKER_URL" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + - name: "CLP_CONFIG_PATH" + value: "/etc/clp-config.yml" + - name: "CLP_HOME" + value: "/opt/clp" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.compression_worker.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/compression_worker" + - name: "CLP_WORKER_LOG_PATH" + value: "/var/log/compression_worker/worker.log" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + - name: "RESULT_BACKEND" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "compression-worker-logs" + mountPath: "/var/log/compression_worker" + - name: "archives" + mountPath: "/var/data/archives" + - name: "logs-input" + mountPath: "/mnt/logs" + readOnly: true + - name: "staged-archives" + mountPath: "/var/data/staged-archives" + - name: "tmp" + mountPath: "/var/tmp" + command: [ + "python3", "-u", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.compress", + "worker", + "--concurrency", "{{ .Values.workerConcurrency }}", + "--loglevel", "WARNING", + "-f", "/var/log/compression_worker/worker.log", + "-Q", "compression", + "-n", "compression-worker" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "compression-worker-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-compression-worker-logs + - name: "archives" + hostPath: + path: {{ include "clp-package.dataDirHost" . }}/archives + type: "DirectoryOrCreate" + - name: "logs-input" + hostPath: + path: "{{ .Values.clpConfig.logs_input.directory }}" + type: "Directory" + - name: "staged-archives" + hostPath: + path: {{ include "clp-package.dataDirHost" . }}/staged-archives + type: "DirectoryOrCreate" + - name: "tmp" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-tmp +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-compression-worker-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-worker" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-worker" + resources: + requests: + storage: "5Gi" +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-tmp + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "tmp" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "tmp" + resources: + requests: + storage: "10Gi" diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml new file mode 100644 index 0000000000..47aa6a0d5a --- /dev/null +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -0,0 +1,150 @@ +apiVersion: "v1" +kind: "ConfigMap" +metadata: + name: {{ include "clp-package.fullname" . }}-config + labels: + {{- include "clp-package.labels" . | nindent 4 }} +data: + clp-config.yml: | + api_server: + default_max_num_query_results: {{ .Values.clpConfig.api_server.default_max_num_query_results }} + host: {{ .Values.clpConfig.api_server.host | quote }} + port: {{ .Values.clpConfig.api_server.port }} + query_job_polling: + initial_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.initial_backoff_ms }} + max_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.max_backoff_ms }} + archive_output: + compression_level: {{ .Values.clpConfig.archive_output.compression_level }} + retention_period: {{ .Values.clpConfig.archive_output.retention_period }} + storage: + directory: "/var/data/archives" + type: "fs" + target_archive_size: {{ .Values.clpConfig.archive_output.target_archive_size }} + target_dictionaries_size: {{ .Values.clpConfig.archive_output.target_dictionaries_size }} + target_encoded_file_size: {{ .Values.clpConfig.archive_output.target_encoded_file_size }} + target_segment_size: {{ .Values.clpConfig.archive_output.target_segment_size }} + aws_config_directory: {{ .Values.clpConfig.aws_config_directory }} + compression_scheduler: + jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} + logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} + compression_worker: + logging_level: {{ .Values.clpConfig.compression_worker.logging_level | quote }} + data_directory: "/var/data" + database: + auto_commit: false + compress: true + host: {{ include "clp-package.fullname" . }}-database + name: {{ .Values.clpConfig.database.name | quote }} + port: {{ .Values.clpConfig.database.port }} + ssl_cert: null + type: {{ .Values.clpConfig.database.type | quote }} + garbage_collector: + logging_level: {{ .Values.clpConfig.garbage_collector.logging_level | quote }} + sweep_interval: + archive: {{ .Values.clpConfig.garbage_collector.sweep_interval.archive }} + search_result: {{ .Values.clpConfig.garbage_collector.sweep_interval.search_result }} + logs_directory: "/var/log" + logs_input: + directory: "/mnt/logs" + type: "fs" + mcp_server: {{ .Values.clpConfig.mcp_server }} + package: + query_engine: {{ .Values.clpConfig.package.query_engine | quote }} + storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} + presto: {{ .Values.clpConfig.presto }} + query_scheduler: + host: {{ include "clp-package.fullname" . }}-query-scheduler + jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }} + logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} + num_archives_to_search_per_sub_job: {{ .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job }} + port: {{ .Values.clpConfig.query_scheduler.port }} + query_worker: + logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }} + queue: + host: {{ include "clp-package.fullname" . }}-queue + port: {{ .Values.clpConfig.queue.port }} + redis: + compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} + host: {{ include "clp-package.fullname" . }}-redis + port: {{ .Values.clpConfig.redis.port }} + query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} + reducer: + base_port: {{ .Values.clpConfig.reducer.base_port }} + host: {{ include "clp-package.fullname" . }}-reducer + logging_level: {{ .Values.clpConfig.reducer.logging_level | quote }} + upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval }} + results_cache: + db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} + host: {{ include "clp-package.fullname" . }}-results-cache + port: {{ .Values.clpConfig.results_cache.port }} + retention_period: {{ .Values.clpConfig.results_cache.retention_period }} + stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} + stream_output: + storage: + directory: "/var/data/streams" + type: "fs" + target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size }} + tmp_directory: "/var/tmp" + webui: + host: {{ .Values.clpConfig.webui.host | quote }} + port: {{ .Values.clpConfig.webui.port }} + rate_limit: {{ .Values.clpConfig.webui.rate_limit }} + results_metadata_collection_name: {{ .Values.clpConfig.webui.results_metadata_collection_name | quote }} + mysql-logging.cnf: | + [mysqld] + # https://dev.mysql.com/doc/refman/8.0/en/log-destinations.html + # Enable the general log + general_log=1 + # Set the destination file + general_log_file=/var/log/mysql/mysql.log + + # https://dev.mysql.com/doc/refman/8.0/en/error-log-destination-configuration.html + # Send error logs to file + log_error=/var/log/mysql/mysql-error.log + redis.conf: | + # For full reference of configuration options see + # https://raw.githubusercontent.com/redis/redis/7.2/redis.conf + + databases 2 + logfile "/var/log/redis/redis.log" + mongod.conf: | + replication: + replSetName: "rs0" + systemLog: + destination: file + path: /var/log/mongodb/mongod.log + logAppend: true + timeStampFormat: iso8601-local + webui-client-settings.json: | + { + "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, + "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, + "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, + "SqlDbClpArchivesTableName": "", + "SqlDbClpDatasetsTableName": "clp_datasets", + "SqlDbClpFilesTableName": "", + "SqlDbClpTablePrefix": "clp_", + "SqlDbCompressionJobsTableName": "compression_jobs" + } + webui-server-settings.json: | + { + "SqlDbHost": "{{ include "clp-package.fullname" . }}-database", + "SqlDbPort": {{ .Values.clpConfig.database.port }}, + "SqlDbName": {{ .Values.clpConfig.database.name | quote }}, + "SqlDbQueryJobsTableName": "query_jobs", + "MongoDbHost": "{{ include "clp-package.fullname" . }}-results-cache", + "MongoDbPort": 27017, + "MongoDbName": {{ .Values.clpConfig.results_cache.db_name | quote }}, + "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, + "MongoDbStreamFilesCollectionName": "stream-files", + "ClientDir": "/opt/clp/var/www/webui/client", + "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer", + "StreamFilesDir": "/var/data/streams", + "StreamTargetUncompressedSize": 134217728, + "StreamFilesS3Region": null, + "StreamFilesS3PathPrefix": null, + "StreamFilesS3Profile": null, + "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, + "PrestoHost": null, + "PrestoPort": null + } diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml new file mode 100644 index 0000000000..3721691dce --- /dev/null +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -0,0 +1,133 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-database + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "database" + ports: + - port: 3306 + targetPort: 3306 +--- +{{- if .Values.allowSbinScripts }} +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-database-public + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +spec: + type: "NodePort" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "database" + ports: + - port: 3306 + nodePort: 30306 +--- +{{- end }} +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp-package.fullname" . }}-database + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +spec: + serviceName: {{ include "clp-package.fullname" . }}-database + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "database" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "database" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "database" + image: "mariadb:10-jammy" + imagePullPolicy: "Always" + env: + - name: "MYSQL_DATABASE" + value: {{ .Values.clpConfig.database.name | quote }} + - name: "MYSQL_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "MYSQL_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "MYSQL_ROOT_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + ports: + - name: "database" + containerPort: 3306 + volumeMounts: + - name: "database-data" + mountPath: "/var/lib/mysql" + - name: "database-logs" + mountPath: "/var/log/mysql" + - name: "config" + mountPath: "/etc/mysql/conf.d/logging.cnf" + subPath: "mysql-logging.cnf" + readOnly: true + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + exec: &database-health-check + command: [ + "mysqladmin", "ping", + "--silent", + "-h", "127.0.0.1" + ] + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + exec: + <<: *database-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + volumeClaimTemplates: + - metadata: + name: "database-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "database" + resources: + requests: + storage: "20Gi" + - metadata: + name: "database-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "database" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml new file mode 100644 index 0000000000..97dd731c67 --- /dev/null +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -0,0 +1,58 @@ +apiVersion: "batch/v1" +kind: "Job" +metadata: + name: {{ include "clp-package.fullname" . }}-db-table-creator + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "db-table-creator" +spec: + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "db-table-creator" + spec: + restartPolicy: "OnFailure" + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + containers: + - name: "db-table-creator" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + command: [ + "python3", "-u", + "-m", "clp_py_utils.create-db-tables", + "--config", "/etc/clp-config.yml", + "--storage-engine", {{ .Values.clpConfig.package.storage_engine | quote }} + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml new file mode 100644 index 0000000000..7104650c1c --- /dev/null +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -0,0 +1,121 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-query-scheduler + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" + ports: + - port: 7000 + targetPort: 7000 +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-query-scheduler + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "query-scheduler" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "query-scheduler" + spec: + terminationGracePeriodSeconds: 10 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + - name: "wait-for-queue" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + ] + - name: "wait-for-redis" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + ] + containers: + - name: "query-scheduler" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "BROKER_URL" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/query_scheduler" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + - name: "RESULT_BACKEND" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "query-scheduler-logs" + mountPath: "/var/log/query_scheduler" + command: [ + "python3", "-u", + "-m", "job_orchestration.scheduler.query.query_scheduler", + "--config", "/etc/clp-config.yml" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "query-scheduler-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-query-scheduler-logs +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-query-scheduler-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "query-scheduler" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml new file mode 100644 index 0000000000..c21a90380d --- /dev/null +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -0,0 +1,118 @@ +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-query-worker + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-worker" +spec: + replicas: {{ .Values.workerConcurrency }} + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "query-worker" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "query-worker" + spec: + terminationGracePeriodSeconds: 60 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-queue" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + ] + - name: "wait-for-redis" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + ] + containers: + - name: "query-worker" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "BROKER_URL" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + - name: "CLP_CONFIG_PATH" + value: "/etc/clp-config.yml" + - name: "CLP_HOME" + value: "/opt/clp" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.query_worker.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/query_worker" + - name: "CLP_WORKER_LOG_PATH" + value: "/var/log/query_worker/worker.log" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + - name: "RESULT_BACKEND" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "query-worker-logs" + mountPath: "/var/log/query_worker" + - name: "archives" + mountPath: "/var/data/archives" + - name: "staged-streams" + mountPath: "/var/data/staged-streams" + - name: "streams" + mountPath: "/var/data/streams" + command: [ + "python3", "-u", + "/opt/clp/lib/python3/site-packages/bin/celery", + "-A", "job_orchestration.executor.query", + "worker", + "--concurrency", "{{ .Values.workerConcurrency }}", + "--loglevel", "WARNING", + "-f", "/var/log/query_worker/worker.log", + "-Q", "query", + "-n", "query-worker" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "query-worker-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-query-worker-logs + - name: "archives" + hostPath: + path: {{ include "clp-package.dataDirHost" . }}/archives + type: "DirectoryOrCreate" + - name: "staged-streams" + hostPath: + path: {{ include "clp-package.dataDirHost" . }}/staged-streams + type: "DirectoryOrCreate" + - name: "streams" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-streams +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-query-worker-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-worker" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "query-worker" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml new file mode 100644 index 0000000000..f0f5cf1d60 --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -0,0 +1,83 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-queue + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "queue" + ports: + - port: 5672 + targetPort: 5672 +--- +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp-package.fullname" . }}-queue + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +spec: + serviceName: {{ include "clp-package.fullname" . }}-queue + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "queue" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "queue" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "queue" + image: "rabbitmq:3.9.8" + imagePullPolicy: "Always" + env: + - name: "RABBITMQ_DEFAULT_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "queue-username" + - name: "RABBITMQ_DEFAULT_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "queue-password" + - name: "RABBITMQ_LOGS" + value: "/var/log/rabbitmq/rabbitmq.log" + ports: + - containerPort: 5672 + volumeMounts: + - name: "queue-logs" + mountPath: "/var/log/rabbitmq" + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + exec: &queue-health-check + command: ["rabbitmq-diagnostics", "check_running"] + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + exec: + <<: *queue-health-check + volumeClaimTemplates: + - metadata: + name: "queue-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "queue" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml new file mode 100644 index 0000000000..56866947f0 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -0,0 +1,110 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-redis + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "redis" + ports: + - port: 6379 + targetPort: 6379 +--- +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp-package.fullname" . }}-redis + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +spec: + serviceName: {{ include "clp-package.fullname" . }}-redis + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "redis" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "redis" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "redis" + image: "redis:7.2.4" + imagePullPolicy: "Always" + command: [ + "redis-server", "/etc/redis/redis.conf", + "--requirepass", "$(REDIS_PASSWORD)" + ] + env: + - name: "REDIS_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "redis-password" + ports: + - containerPort: 6379 + volumeMounts: + - name: "redis-data" + mountPath: "/data" + - name: "redis-logs" + mountPath: "/var/log/redis" + - name: "config" + mountPath: "/etc/redis/redis.conf" + subPath: "redis.conf" + readOnly: true + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + exec: &redis-health-check + command: + - "redis-cli" + - "-h" + - "127.0.0.1" + - "-p" + - "6379" + - "-a" + - "$(REDIS_PASSWORD)" + - "PING" + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + exec: + <<: *redis-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + volumeClaimTemplates: + - metadata: + name: "redis-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "redis" + resources: + requests: + storage: "10Gi" + - metadata: + name: "redis-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "redis" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml new file mode 100644 index 0000000000..bdc479a54e --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -0,0 +1,39 @@ +apiVersion: "batch/v1" +kind: "Job" +metadata: + name: {{ include "clp-package.fullname" . }}-results-cache-indices-creator + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache-indices-creator" +spec: + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "results-cache-indices-creator" + spec: + restartPolicy: "OnFailure" + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-results-cache" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + ] + containers: + - name: "results-cache-indices-creator" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + command: [ + "python3", "-u", + "-m", "clp_py_utils.initialize-results-cache", + "--uri", "mongodb://{{ include "clp-package.fullname" . }}-results-cache:27017/{{ .Values.clpConfig.results_cache.db_name }}", + "--stream-collection", {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} + ] diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml new file mode 100644 index 0000000000..2673cb4ecd --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -0,0 +1,121 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-results-cache + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" + ports: + - port: 27017 + targetPort: 27017 +--- +{{- if .Values.allowSbinScripts }} +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-results-cache-public + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + type: "NodePort" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" + ports: + - port: 27017 + targetPort: 27017 + nodePort: 30017 +--- +{{- end }} +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp-package.fullname" . }}-results-cache + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + serviceName: {{ include "clp-package.fullname" . }}-results-cache + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "results-cache" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "results-cache" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "results-cache" + image: "mongo:7.0.1" + imagePullPolicy: "Always" + command: [ + "mongod", + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0" + ] + ports: + - containerPort: 27017 + volumeMounts: + - name: "results-cache-data" + mountPath: "/data/db" + - name: "results-cache-logs" + mountPath: "/var/log/mongodb" + - name: "config" + mountPath: "/etc/mongo/mongod.conf" + subPath: "mongod.conf" + readOnly: true + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + exec: &results-cache-health-check + command: + - "mongosh" + - "127.0.0.1:27017/test" + - "--quiet" + - "--eval" + - "db.runCommand('ping').ok" + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + exec: + <<: *results-cache-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + volumeClaimTemplates: + - metadata: + name: "results-cache-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "results-cache" + resources: + requests: + storage: "20Gi" + - metadata: + name: "results-cache-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "results-cache" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/secret.yaml b/tools/deployment/package-helm/templates/secret.yaml new file mode 100644 index 0000000000..b50867fcbc --- /dev/null +++ b/tools/deployment/package-helm/templates/secret.yaml @@ -0,0 +1,13 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp-package.fullname" . }}-secret +type: "Opaque" +stringData: + db-username: {{ .Values.credentials.database.user | include "clp-package.requireInput" | quote }} + db-password: {{ .Values.credentials.database.password | include "clp-package.requireInput" | quote }} + + queue-username: {{ .Values.credentials.queue.user | include "clp-package.requireInput" | quote }} + queue-password: {{ .Values.credentials.queue.password | include "clp-package.requireInput" | quote }} + + redis-password: {{ .Values.credentials.redis.password | include "clp-package.requireInput" | quote }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml new file mode 100644 index 0000000000..61d6f53bc6 --- /dev/null +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -0,0 +1,132 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-webui + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "webui" +spec: + type: "NodePort" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "webui" + ports: + - port: 4000 + targetPort: 4000 + nodePort: 30400 +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-webui + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "webui" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "webui" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "webui" + spec: + terminationGracePeriodSeconds: 60 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + - name: "wait-for-results-cache" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + ] + containers: + - name: "webui" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "HOST" + value: "0.0.0.0" + - name: "NODE_ENV" + value: "production" + - name: "NODE_PATH" + value: "/opt/clp/var/www/webui/server/node_modules" + - name: "PORT" + value: "4000" + - name: "RATE_LIMIT" + value: "1000" + ports: + - containerPort: 4000 + volumeMounts: + - name: "client-settings" + mountPath: "/opt/clp/var/www/webui/client/settings.json" + subPath: "webui-client-settings.json" + readOnly: true + - name: "server-settings" + mountPath: "/opt/clp/var/www/webui/server/dist/settings.json" + subPath: "webui-server-settings.json" + readOnly: true + - name: "streams" + mountPath: "/var/data/streams" + command: [ + "/opt/clp/bin/node-22", + "/opt/clp/var/www/webui/server/dist/src/main.js" + ] + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + tcpSocket: + port: 4000 + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + tcpSocket: + port: 4000 + volumes: + - name: "client-settings" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "server-settings" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "streams" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-streams +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-streams + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "streams" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "streams" + resources: + requests: + storage: "20Gi" diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh new file mode 100755 index 0000000000..e7985c8de8 --- /dev/null +++ b/tools/deployment/package-helm/test.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# TODO: to be deleted / migrated to integration tests + +kind delete cluster --name clp-test +sudo rm -rf /tmp/clp +mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker} +mkdir -p /tmp/clp/var/data/{archives,staged-streams,streams} +mkdir -p /tmp/clp/var/tmp +mkdir -p /tmp/clp/samples + +cat < Date: Sat, 15 Nov 2025 20:29:03 -0500 Subject: [PATCH 002/149] Add reducer service and deployment to Helm chart. --- .../package-helm/templates/00-local-pvs.yaml | 2 + .../templates/reducer-deployment.yaml | 110 ++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 tools/deployment/package-helm/templates/reducer-deployment.yaml diff --git a/tools/deployment/package-helm/templates/00-local-pvs.yaml b/tools/deployment/package-helm/templates/00-local-pvs.yaml index 94da93965a..7f57dd0ef7 100644 --- a/tools/deployment/package-helm/templates/00-local-pvs.yaml +++ b/tools/deployment/package-helm/templates/00-local-pvs.yaml @@ -20,6 +20,8 @@ --- {{- include "clp-package.createLocalLogsPv" (list . "query-worker") }} --- +{{- include "clp-package.createLocalLogsPv" (list . "reducer") }} +--- {{- include "clp-package.createStreamsPv" . }} --- {{- include "clp-package.createTmpPv" . }} diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml new file mode 100644 index 0000000000..869be59d5b --- /dev/null +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -0,0 +1,110 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-reducer + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" +spec: + clusterIP: "None" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" + ports: + {{- range $i := until (int .Values.workerConcurrency) }} + - name: "reducer-{{ $i }}" + port: {{ add $.Values.clpConfig.reducer.base_port $i }} + targetPort: {{ add $.Values.clpConfig.reducer.base_port $i }} + {{- end }} +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-reducer + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "reducer" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "reducer" + spec: + hostname: {{ include "clp-package.fullname" . }}-reducer + subdomain: {{ include "clp-package.fullname" . }}-reducer + terminationGracePeriodSeconds: 10 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-query-scheduler" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-query-scheduler 7000; do echo waiting for query-scheduler; sleep 2; done" + ] + - name: "wait-for-results-cache" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + ] + containers: + - name: "reducer" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_HOME" + value: "/opt/clp" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.reducer.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/reducer" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "reducer-logs" + mountPath: "/var/log/reducer" + command: [ + "python3", "-u", + "-m", "job_orchestration.reducer.reducer", + "--config", "/etc/clp-config.yml", + "--concurrency", "{{ .Values.workerConcurrency }}", + "--upsert-interval", "{{ .Values.clpConfig.reducer.upsert_interval }}" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "reducer-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-reducer-logs +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-reducer-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "reducer" + resources: + requests: + storage: "5Gi" From c0c9d02dbdf8d2ea7d4a46ff5138b181e3c13a08 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 20:39:17 -0500 Subject: [PATCH 003/149] Replace hostPath volumes with PersistentVolumeClaims in Helm chart; fix query scheduler hostname binding issue. --- .../package-helm/templates/00-local-pvs.yaml | 6 ++ .../package-helm/templates/_helpers.tpl | 84 +++++++++++++++++++ .../compression-worker-deployment.yaml | 46 ++++++++-- .../templates/query-scheduler-deployment.yaml | 2 + .../templates/query-worker-deployment.yaml | 28 +++++-- 5 files changed, 154 insertions(+), 12 deletions(-) diff --git a/tools/deployment/package-helm/templates/00-local-pvs.yaml b/tools/deployment/package-helm/templates/00-local-pvs.yaml index 7f57dd0ef7..96bff3f9a0 100644 --- a/tools/deployment/package-helm/templates/00-local-pvs.yaml +++ b/tools/deployment/package-helm/templates/00-local-pvs.yaml @@ -25,3 +25,9 @@ {{- include "clp-package.createStreamsPv" . }} --- {{- include "clp-package.createTmpPv" . }} +--- +{{- include "clp-package.createArchivesPv" . }} +--- +{{- include "clp-package.createStagedArchivesPv" . }} +--- +{{- include "clp-package.createStagedStreamsPv" . }} diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index a02d98809e..1763654a02 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -213,3 +213,87 @@ spec: operator: "Exists" {{- end }} +{{/* +Creates a local PersistentVolume for archives storage (ReadWriteMany). +*/}} +{{- define "clp-package.createArchivesPv" -}} +{{- $root := . -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-archives-pv + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: "archives" +spec: + capacity: + storage: "50Gi" + accessModes: ["ReadWriteMany"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ include "clp-package.dataDirHost" $root }}/archives + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + +{{/* +Creates a local PersistentVolume for staged-archives storage (ReadWriteMany). +*/}} +{{- define "clp-package.createStagedArchivesPv" -}} +{{- $root := . -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-staged-archives-pv + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: "staged-archives" +spec: + capacity: + storage: "20Gi" + accessModes: ["ReadWriteMany"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ include "clp-package.dataDirHost" $root }}/staged-archives + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + +{{/* +Creates a local PersistentVolume for staged-streams storage (ReadWriteMany). +*/}} +{{- define "clp-package.createStagedStreamsPv" -}} +{{- $root := . -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ include "clp-package.fullname" $root }}-staged-streams-pv + labels: + {{- include "clp-package.labels" $root | nindent 4 }} + app.kubernetes.io/component: "staged-streams" +spec: + capacity: + storage: "20Gi" + accessModes: ["ReadWriteMany"] + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ include "clp-package.dataDirHost" $root }}/staged-streams + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 3964667b6f..2f9449fbe4 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -91,17 +91,15 @@ spec: persistentVolumeClaim: claimName: {{ include "clp-package.fullname" . }}-compression-worker-logs - name: "archives" - hostPath: - path: {{ include "clp-package.dataDirHost" . }}/archives - type: "DirectoryOrCreate" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-archives - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" type: "Directory" - name: "staged-archives" - hostPath: - path: {{ include "clp-package.dataDirHost" . }}/staged-archives - type: "DirectoryOrCreate" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-staged-archives - name: "tmp" persistentVolumeClaim: claimName: {{ include "clp-package.fullname" . }}-tmp @@ -126,6 +124,42 @@ spec: --- apiVersion: "v1" kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-archives + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "archives" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "archives" + resources: + requests: + storage: "50Gi" +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-staged-archives + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "staged-archives" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "staged-archives" + resources: + requests: + storage: "20Gi" +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" metadata: name: {{ include "clp-package.fullname" . }}-tmp labels: diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 7104650c1c..a71e33ef5d 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -33,6 +33,8 @@ spec: {{- include "clp-package.labels" . | nindent 8 }} app.kubernetes.io/component: "query-scheduler" spec: + hostname: {{ include "clp-package.fullname" . }}-query-scheduler + subdomain: {{ include "clp-package.fullname" . }}-query-scheduler terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index c21a90380d..83e9198010 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -88,13 +88,11 @@ spec: persistentVolumeClaim: claimName: {{ include "clp-package.fullname" . }}-query-worker-logs - name: "archives" - hostPath: - path: {{ include "clp-package.dataDirHost" . }}/archives - type: "DirectoryOrCreate" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-archives - name: "staged-streams" - hostPath: - path: {{ include "clp-package.dataDirHost" . }}/staged-streams - type: "DirectoryOrCreate" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-staged-streams - name: "streams" persistentVolumeClaim: claimName: {{ include "clp-package.fullname" . }}-streams @@ -116,3 +114,21 @@ spec: resources: requests: storage: "5Gi" +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-staged-streams + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "staged-streams" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "staged-streams" + resources: + requests: + storage: "20Gi" From 8bfd5e1469c31a38cdb96fa9849a582c70ff4ba4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 20:41:04 -0500 Subject: [PATCH 004/149] update test.sh --- tools/deployment/package-helm/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index e7985c8de8..85485148fd 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -4,7 +4,7 @@ kind delete cluster --name clp-test sudo rm -rf /tmp/clp -mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker} +mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker,reducer} mkdir -p /tmp/clp/var/data/{archives,staged-streams,streams} mkdir -p /tmp/clp/var/tmp mkdir -p /tmp/clp/samples From a3a1356cfa5498585de52bcf4af7e7579bbcb997 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 20:43:27 -0500 Subject: [PATCH 005/149] update test.sh --- tools/deployment/package-helm/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 85485148fd..9adbe38baa 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -5,7 +5,7 @@ kind delete cluster --name clp-test sudo rm -rf /tmp/clp mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker,reducer} -mkdir -p /tmp/clp/var/data/{archives,staged-streams,streams} +mkdir -p /tmp/clp/var/data/{archives,staged-archives,staged-streams,streams} mkdir -p /tmp/clp/var/tmp mkdir -p /tmp/clp/samples From 37000f699300c0a5734d3e6fdb55e88d9f99968c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 20:53:25 -0500 Subject: [PATCH 006/149] Update readiness probe timeoutSeconds to 1. --- tools/deployment/package-helm/templates/_helpers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 1763654a02..10479cd318 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -71,7 +71,7 @@ Creates timings for readiness probes (faster checks for quicker startup). {{- define "clp-package.readinessProbeTimings" -}} initialDelaySeconds: 1 periodSeconds: 1 -timeoutSeconds: 2 +timeoutSeconds: 1 failureThreshold: 3 {{- end }} From 34b3dca852aea281a2eb44cf518470d1a3e88c75 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 21:52:42 -0500 Subject: [PATCH 007/149] feat(deployment): Add deployments for MCP server, API server, and garbage collector. --- .../package-helm/templates/00-local-pvs.yaml | 8 ++ .../templates/api-server-deployment.yaml | 111 +++++++++++++++ .../garbage-collector-deployment.yaml | 107 +++++++++++++++ .../templates/mcp-server-deployment.yaml | 126 ++++++++++++++++++ tools/deployment/package-helm/test.sh | 8 +- 5 files changed, 359 insertions(+), 1 deletion(-) create mode 100644 tools/deployment/package-helm/templates/api-server-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/garbage-collector-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/mcp-server-deployment.yaml diff --git a/tools/deployment/package-helm/templates/00-local-pvs.yaml b/tools/deployment/package-helm/templates/00-local-pvs.yaml index 96bff3f9a0..0bd1f663ed 100644 --- a/tools/deployment/package-helm/templates/00-local-pvs.yaml +++ b/tools/deployment/package-helm/templates/00-local-pvs.yaml @@ -22,6 +22,14 @@ --- {{- include "clp-package.createLocalLogsPv" (list . "reducer") }} --- +{{- include "clp-package.createLocalLogsPv" (list . "garbage-collector") }} +--- +{{- include "clp-package.createLocalLogsPv" (list . "api-server") }} +--- +{{- if .Values.clpConfig.mcp_server }} +{{- include "clp-package.createLocalLogsPv" (list . "mcp-server") }} +--- +{{- end }} {{- include "clp-package.createStreamsPv" . }} --- {{- include "clp-package.createTmpPv" . }} diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml new file mode 100644 index 0000000000..be76af0b35 --- /dev/null +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -0,0 +1,111 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-api-server + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" +spec: + type: "NodePort" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" + ports: + - port: {{ .Values.clpConfig.api_server.port }} + targetPort: {{ .Values.clpConfig.api_server.port }} + nodePort: 30301 +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-api-server + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "api-server" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "api-server" + spec: + terminationGracePeriodSeconds: 10 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + - name: "wait-for-results-cache" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + ] + containers: + - name: "api-server" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "CLP_LOGS_DIR" + value: "/var/log/api_server" + - name: "RUST_LOG" + value: "DEBUG" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "api-server-logs" + mountPath: "/var/log/api_server" + command: [ + "/opt/clp/bin/api_server", + "--host", "0.0.0.0", + "--port", "{{ .Values.clpConfig.api_server.port }}", + "--config", "/etc/clp-config.yml" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "api-server-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-api-server-logs +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-api-server-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "api-server" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml new file mode 100644 index 0000000000..47b989b372 --- /dev/null +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -0,0 +1,107 @@ +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-garbage-collector + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "garbage-collector" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "garbage-collector" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "garbage-collector" + spec: + terminationGracePeriodSeconds: 10 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + - name: "wait-for-results-cache" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + ] + containers: + - name: "garbage-collector" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "CLP_HOME" + value: "/opt/clp" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.garbage_collector.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/garbage_collector" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "garbage-collector-logs" + mountPath: "/var/log/garbage_collector" + - name: "archives" + mountPath: "/var/data/archives" + - name: "streams" + mountPath: "/var/data/streams" + command: [ + "python3", "-u", + "-m", "job_orchestration.garbage_collector.garbage_collector", + "--config", "/etc/clp-config.yml" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "garbage-collector-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-garbage-collector-logs + - name: "archives" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-archives + - name: "streams" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-streams +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-garbage-collector-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "garbage-collector" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "garbage-collector" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml new file mode 100644 index 0000000000..c86fb16efd --- /dev/null +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -0,0 +1,126 @@ +{{- if .Values.clpConfig.mcp_server }} +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp-package.fullname" . }}-mcp-server + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" +spec: + type: "NodePort" + selector: + {{- include "clp-package.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" + ports: + - port: 8000 + targetPort: 8000 + nodePort: 30800 +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp-package.fullname" . }}-mcp-server + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "mcp-server" + template: + metadata: + labels: + {{- include "clp-package.labels" . | nindent 8 }} + app.kubernetes.io/component: "mcp-server" + spec: + terminationGracePeriodSeconds: 10 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + - name: "wait-for-results-cache" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + ] + containers: + - name: "mcp-server" + image: "{{ include "clp-package.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp-package.fullname" . }}-secret + key: "db-password" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.mcp_server.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/mcp_server" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + - name: "mcp-server-logs" + mountPath: "/var/log/mcp_server" + command: [ + "python3", "-u", + "-m", "clp_mcp_server.clp_mcp_server", + "--host", "0.0.0.0", + "--port", "8000", + "--config-path", "/etc/clp-config.yml" + ] + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + httpGet: + path: "/health" + port: 8000 + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + httpGet: + path: "/health" + port: 8000 + volumes: + - name: "config" + configMap: + name: {{ include "clp-package.fullname" . }}-config + - name: "mcp-server-logs" + persistentVolumeClaim: + claimName: {{ include "clp-package.fullname" . }}-mcp-server-logs +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp-package.fullname" . }}-mcp-server-logs + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp-package.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "mcp-server" + resources: + requests: + storage: "5Gi" +{{- end }} diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 9adbe38baa..33fbc61651 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -4,7 +4,7 @@ kind delete cluster --name clp-test sudo rm -rf /tmp/clp -mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker,reducer} +mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker,reducer,garbage-collector,api-server,mcp-server} mkdir -p /tmp/clp/var/data/{archives,staged-archives,staged-streams,streams} mkdir -p /tmp/clp/var/tmp mkdir -p /tmp/clp/samples @@ -27,6 +27,12 @@ cat < Date: Sat, 15 Nov 2025 21:55:03 -0500 Subject: [PATCH 008/149] feat(deployment): Add liveness and readiness probes to API server deployment. --- .../package-helm/templates/api-server-deployment.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index be76af0b35..c9e04643b3 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -84,6 +84,16 @@ spec: "--port", "{{ .Values.clpConfig.api_server.port }}", "--config", "/etc/clp-config.yml" ] + livenessProbe: + {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + httpGet: + path: "/health" + port: {{ .Values.clpConfig.api_server.port }} + readinessProbe: + {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + httpGet: + path: "/health" + port: {{ .Values.clpConfig.api_server.port }} volumes: - name: "config" configMap: From a1afc7f5d32bf3335709733185dd4c9ef523e8a2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 22:12:23 -0500 Subject: [PATCH 009/149] Simplify service templates with conditional NodePort configuration. --- .../templates/database-statefulset.yaml | 23 +++++------------- .../templates/results-cache-statefulset.yaml | 24 +++++-------------- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 3721691dce..456c0128d4 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -6,32 +6,21 @@ metadata: {{- include "clp-package.labels" . | nindent 4 }} app.kubernetes.io/component: "database" spec: + {{- if .Values.allowSbinScripts }} + type: "NodePort" + {{- else }} clusterIP: "None" + {{- end }} selector: {{- include "clp-package.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "database" ports: - port: 3306 targetPort: 3306 ---- -{{- if .Values.allowSbinScripts }} -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp-package.fullname" . }}-database-public - labels: - {{- include "clp-package.labels" . | nindent 4 }} - app.kubernetes.io/component: "database" -spec: - type: "NodePort" - selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "database" - ports: - - port: 3306 + {{- if .Values.allowSbinScripts }} nodePort: 30306 + {{- end }} --- -{{- end }} apiVersion: "apps/v1" kind: "StatefulSet" metadata: diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 2673cb4ecd..1e25c5493d 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -6,33 +6,21 @@ metadata: {{- include "clp-package.labels" . | nindent 4 }} app.kubernetes.io/component: "results-cache" spec: - clusterIP: "None" - selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "results-cache" - ports: - - port: 27017 - targetPort: 27017 ---- -{{- if .Values.allowSbinScripts }} -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp-package.fullname" . }}-results-cache-public - labels: - {{- include "clp-package.labels" . | nindent 4 }} - app.kubernetes.io/component: "results-cache" -spec: + {{- if .Values.allowSbinScripts }} type: "NodePort" + {{- else }} + clusterIP: "None" + {{- end }} selector: {{- include "clp-package.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "results-cache" ports: - port: 27017 targetPort: 27017 + {{- if .Values.allowSbinScripts }} nodePort: 30017 + {{- end }} --- -{{- end }} apiVersion: "apps/v1" kind: "StatefulSet" metadata: From e3d78a42bc49a8a55e031b5241f6b58444d2016e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 22:18:41 -0500 Subject: [PATCH 010/149] Add named ports in Helm templates for standardization. --- .../package-helm/templates/api-server-deployment.yaml | 5 ++++- .../package-helm/templates/database-statefulset.yaml | 2 +- .../package-helm/templates/mcp-server-deployment.yaml | 5 ++++- .../package-helm/templates/query-scheduler-deployment.yaml | 5 ++++- .../deployment/package-helm/templates/queue-statefulset.yaml | 5 +++-- .../deployment/package-helm/templates/redis-statefulset.yaml | 5 +++-- .../package-helm/templates/results-cache-statefulset.yaml | 5 +++-- .../deployment/package-helm/templates/webui-deployment.yaml | 5 +++-- 8 files changed, 25 insertions(+), 12 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index c9e04643b3..9294df9014 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -12,7 +12,7 @@ spec: app.kubernetes.io/component: "api-server" ports: - port: {{ .Values.clpConfig.api_server.port }} - targetPort: {{ .Values.clpConfig.api_server.port }} + targetPort: "http" nodePort: 30301 --- apiVersion: "apps/v1" @@ -71,6 +71,9 @@ spec: value: "/var/log/api_server" - name: "RUST_LOG" value: "DEBUG" + ports: + - name: "http" + containerPort: {{ .Values.clpConfig.api_server.port }} volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 456c0128d4..ab7848fa23 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -16,7 +16,7 @@ spec: app.kubernetes.io/component: "database" ports: - port: 3306 - targetPort: 3306 + targetPort: "database" {{- if .Values.allowSbinScripts }} nodePort: 30306 {{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index c86fb16efd..d00c37aad2 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -13,7 +13,7 @@ spec: app.kubernetes.io/component: "mcp-server" ports: - port: 8000 - targetPort: 8000 + targetPort: "http" nodePort: 30800 --- apiVersion: "apps/v1" @@ -74,6 +74,9 @@ spec: value: "/var/log/mcp_server" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" + ports: + - name: "http" + containerPort: 8000 volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index a71e33ef5d..ea193637dc 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -12,7 +12,7 @@ spec: app.kubernetes.io/component: "query-scheduler" ports: - port: 7000 - targetPort: 7000 + targetPort: "query-scheduler" --- apiVersion: "apps/v1" kind: "Deployment" @@ -84,6 +84,9 @@ spec: value: "/opt/clp/lib/python3/site-packages" - name: "RESULT_BACKEND" value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" + ports: + - name: "query-scheduler" + containerPort: 7000 volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index f0f5cf1d60..20ebf719a1 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -12,7 +12,7 @@ spec: app.kubernetes.io/component: "queue" ports: - port: 5672 - targetPort: 5672 + targetPort: "amqp" --- apiVersion: "apps/v1" kind: "StatefulSet" @@ -56,7 +56,8 @@ spec: - name: "RABBITMQ_LOGS" value: "/var/log/rabbitmq/rabbitmq.log" ports: - - containerPort: 5672 + - name: "amqp" + containerPort: 5672 volumeMounts: - name: "queue-logs" mountPath: "/var/log/rabbitmq" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 56866947f0..882c8b33db 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -12,7 +12,7 @@ spec: app.kubernetes.io/component: "redis" ports: - port: 6379 - targetPort: 6379 + targetPort: "redis" --- apiVersion: "apps/v1" kind: "StatefulSet" @@ -53,7 +53,8 @@ spec: name: {{ include "clp-package.fullname" . }}-secret key: "redis-password" ports: - - containerPort: 6379 + - name: "redis" + containerPort: 6379 volumeMounts: - name: "redis-data" mountPath: "/data" diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 1e25c5493d..4d45371858 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -16,7 +16,7 @@ spec: app.kubernetes.io/component: "results-cache" ports: - port: 27017 - targetPort: 27017 + targetPort: "mongodb" {{- if .Values.allowSbinScripts }} nodePort: 30017 {{- end }} @@ -55,7 +55,8 @@ spec: "--bind_ip", "0.0.0.0" ] ports: - - containerPort: 27017 + - name: "mongodb" + containerPort: 27017 volumeMounts: - name: "results-cache-data" mountPath: "/data/db" diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 61d6f53bc6..9f2f783683 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -12,7 +12,7 @@ spec: app.kubernetes.io/component: "webui" ports: - port: 4000 - targetPort: 4000 + targetPort: "http" nodePort: 30400 --- apiVersion: "apps/v1" @@ -78,7 +78,8 @@ spec: - name: "RATE_LIMIT" value: "1000" ports: - - containerPort: 4000 + - name: "http" + containerPort: 4000 volumeMounts: - name: "client-settings" mountPath: "/opt/clp/var/www/webui/client/settings.json" From f679f3803a085a70e34570fa95c5a1ff322ab2d0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 22:36:41 -0500 Subject: [PATCH 011/149] fix nodePort 's value --- .../templates/api-server-deployment.yaml | 12 ++++++------ .../package-helm/templates/configmap.yaml | 10 +++++----- .../templates/database-statefulset.yaml | 2 +- .../templates/mcp-server-deployment.yaml | 2 +- .../templates/results-cache-statefulset.yaml | 2 +- .../package-helm/templates/webui-deployment.yaml | 2 +- tools/deployment/package-helm/values.yaml | 13 +++++++++---- 7 files changed, 24 insertions(+), 19 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 9294df9014..e764af866d 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -11,9 +11,9 @@ spec: {{- include "clp-package.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "api-server" ports: - - port: {{ .Values.clpConfig.api_server.port }} + - port: 3001 targetPort: "http" - nodePort: 30301 + nodePort: {{ .Values.clpConfig.api_server.port }} --- apiVersion: "apps/v1" kind: "Deployment" @@ -73,7 +73,7 @@ spec: value: "DEBUG" ports: - name: "http" - containerPort: {{ .Values.clpConfig.api_server.port }} + containerPort: 3001 volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" @@ -84,19 +84,19 @@ spec: command: [ "/opt/clp/bin/api_server", "--host", "0.0.0.0", - "--port", "{{ .Values.clpConfig.api_server.port }}", + "--port", "3001", "--config", "/etc/clp-config.yml" ] livenessProbe: {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} httpGet: path: "/health" - port: {{ .Values.clpConfig.api_server.port }} + port: 3001 readinessProbe: {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} httpGet: path: "/health" - port: {{ .Values.clpConfig.api_server.port }} + port: 3001 volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 47aa6a0d5a..a5de1ff6b5 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -9,7 +9,7 @@ data: api_server: default_max_num_query_results: {{ .Values.clpConfig.api_server.default_max_num_query_results }} host: {{ .Values.clpConfig.api_server.host | quote }} - port: {{ .Values.clpConfig.api_server.port }} + port: 3001 query_job_polling: initial_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.initial_backoff_ms }} max_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.max_backoff_ms }} @@ -35,7 +35,7 @@ data: compress: true host: {{ include "clp-package.fullname" . }}-database name: {{ .Values.clpConfig.database.name | quote }} - port: {{ .Values.clpConfig.database.port }} + port: 3306 ssl_cert: null type: {{ .Values.clpConfig.database.type | quote }} garbage_collector: @@ -76,7 +76,7 @@ data: results_cache: db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} host: {{ include "clp-package.fullname" . }}-results-cache - port: {{ .Values.clpConfig.results_cache.port }} + port: 27017 retention_period: {{ .Values.clpConfig.results_cache.retention_period }} stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} stream_output: @@ -87,7 +87,7 @@ data: tmp_directory: "/var/tmp" webui: host: {{ .Values.clpConfig.webui.host | quote }} - port: {{ .Values.clpConfig.webui.port }} + port: 4000 rate_limit: {{ .Values.clpConfig.webui.rate_limit }} results_metadata_collection_name: {{ .Values.clpConfig.webui.results_metadata_collection_name | quote }} mysql-logging.cnf: | @@ -129,7 +129,7 @@ data: webui-server-settings.json: | { "SqlDbHost": "{{ include "clp-package.fullname" . }}-database", - "SqlDbPort": {{ .Values.clpConfig.database.port }}, + "SqlDbPort": 3306, "SqlDbName": {{ .Values.clpConfig.database.name | quote }}, "SqlDbQueryJobsTableName": "query_jobs", "MongoDbHost": "{{ include "clp-package.fullname" . }}-results-cache", diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index ab7848fa23..95fdc13f06 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -18,7 +18,7 @@ spec: - port: 3306 targetPort: "database" {{- if .Values.allowSbinScripts }} - nodePort: 30306 + nodePort: {{ .Values.clpConfig.database.port }} {{- end }} --- apiVersion: "apps/v1" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index d00c37aad2..143a57b3ca 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -14,7 +14,7 @@ spec: ports: - port: 8000 targetPort: "http" - nodePort: 30800 + nodePort: {{ .Values.clpConfig.mcp_server.port }} --- apiVersion: "apps/v1" kind: "Deployment" diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 4d45371858..010fde0c98 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -18,7 +18,7 @@ spec: - port: 27017 targetPort: "mongodb" {{- if .Values.allowSbinScripts }} - nodePort: 30017 + nodePort: {{ .Values.clpConfig.results_cache.port }} {{- end }} --- apiVersion: "apps/v1" diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 9f2f783683..965a54420f 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -13,7 +13,7 @@ spec: ports: - port: 4000 targetPort: "http" - nodePort: 30400 + nodePort: {{ .Values.clpConfig.webui.port }} --- apiVersion: "apps/v1" kind: "Deployment" diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 65a54d0e7a..d542d108db 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -44,7 +44,7 @@ clpConfig: database: type: "mariadb" # "mariadb" or "mysql" host: "localhost" - port: 3306 + port: 30306 name: "clp-db" compression_scheduler: @@ -76,7 +76,7 @@ clpConfig: results_cache: host: "localhost" - port: 27017 + port: 30017 db_name: "clp-query-results" stream_collection_name: "stream-files" @@ -91,11 +91,14 @@ clpConfig: webui: host: "localhost" - port: 4000 + port: 30400 results_metadata_collection_name: "results-metadata" rate_limit: 1000 mcp_server: null + # host: "localhost" + # port: 30800 + # logging_level: "INFO" # Where archives should be output to archive_output: @@ -148,7 +151,7 @@ clpConfig: # API server config api_server: host: "localhost" - port: 3001 + port: 30301 default_max_num_query_results: 1000 query_job_polling: initial_backoff_ms: 100 @@ -156,6 +159,8 @@ clpConfig: # Presto client config presto: null + # host: "localhost" + # port: 30889 # Location where other data (besides archives) are stored. It will be created if # it doesn't exist. From 3108ecad4b410b86938367438f10cb99795b9550 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 22:40:05 -0500 Subject: [PATCH 012/149] fix(helm): Update default port values in configmap for query scheduler, queue, and redis. --- tools/deployment/package-helm/templates/configmap.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index a5de1ff6b5..fa570eacf0 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -57,16 +57,16 @@ data: jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} num_archives_to_search_per_sub_job: {{ .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job }} - port: {{ .Values.clpConfig.query_scheduler.port }} + port: 7000 query_worker: logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }} queue: host: {{ include "clp-package.fullname" . }}-queue - port: {{ .Values.clpConfig.queue.port }} + port: 5672 redis: compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} host: {{ include "clp-package.fullname" . }}-redis - port: {{ .Values.clpConfig.redis.port }} + port: 6379 query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} reducer: base_port: {{ .Values.clpConfig.reducer.base_port }} From 597de93c7c9ce8396577c427d1e6943e9f8c2526 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 23:13:01 -0500 Subject: [PATCH 013/149] remove comment --- tools/deployment/package-helm/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index d542d108db..023ebb93cd 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -8,7 +8,6 @@ storage: # Base directory on host when using local PVs. localPathBase: "/tmp/clp" -# UID and GID for services securityContext: # UID/GID for first-party CLP services firstParty: From 9c68929e1eff024565331186af723f9fb00333bf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 23:23:55 -0500 Subject: [PATCH 014/149] Rename `user` to `username` across all components --- components/clp-package-utils/clp_package_utils/general.py | 4 ++-- components/clp-py-utils/clp_py_utils/clp_config.py | 4 ++-- .../package-template/src/etc/credentials.template.yml | 4 ++-- tools/deployment/package-helm/templates/secret.yaml | 4 ++-- tools/deployment/package-helm/values.yaml | 4 ++-- tools/deployment/presto-clp/scripts/init.py | 6 +++--- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a1c5fb5e15..d0ad00034c 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -453,8 +453,8 @@ def load_config_file( def generate_credentials_file(credentials_file_path: pathlib.Path): credentials = { - DB_COMPONENT_NAME: {"user": "clp-user", "password": secrets.token_urlsafe(8)}, - QUEUE_COMPONENT_NAME: {"user": "clp-user", "password": secrets.token_urlsafe(8)}, + DB_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)}, + QUEUE_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)}, REDIS_COMPONENT_NAME: {"password": secrets.token_urlsafe(16)}, } diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index c35a30dc0f..9e791754b0 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -232,7 +232,7 @@ def load_credentials_from_file(self, credentials_file_path: pathlib.Path): if config is None: raise ValueError(f"Credentials file '{credentials_file_path}' is empty.") try: - self.username = get_config_value(config, f"{DB_COMPONENT_NAME}.user") + self.username = get_config_value(config, f"{DB_COMPONENT_NAME}.username") self.password = get_config_value(config, f"{DB_COMPONENT_NAME}.password") except KeyError as ex: raise ValueError( @@ -360,7 +360,7 @@ def load_credentials_from_file(self, credentials_file_path: pathlib.Path): if config is None: raise ValueError(f"Credentials file '{credentials_file_path}' is empty.") try: - self.username = get_config_value(config, f"{QUEUE_COMPONENT_NAME}.user") + self.username = get_config_value(config, f"{QUEUE_COMPONENT_NAME}.username") self.password = get_config_value(config, f"{QUEUE_COMPONENT_NAME}.password") except KeyError as ex: raise ValueError( diff --git a/components/package-template/src/etc/credentials.template.yml b/components/package-template/src/etc/credentials.template.yml index 9a0dd73fa2..eb2f9b73f7 100644 --- a/components/package-template/src/etc/credentials.template.yml +++ b/components/package-template/src/etc/credentials.template.yml @@ -1,11 +1,11 @@ ## Database credentials #database: -# user: "user" +# username: "clp-user" # password: "pass" # ## Queue credentials #queue: -# user: "user" +# username: "clp-user" # password: "pass" # ## Redis credentials diff --git a/tools/deployment/package-helm/templates/secret.yaml b/tools/deployment/package-helm/templates/secret.yaml index b50867fcbc..145d85f568 100644 --- a/tools/deployment/package-helm/templates/secret.yaml +++ b/tools/deployment/package-helm/templates/secret.yaml @@ -4,10 +4,10 @@ metadata: name: {{ include "clp-package.fullname" . }}-secret type: "Opaque" stringData: - db-username: {{ .Values.credentials.database.user | include "clp-package.requireInput" | quote }} + db-username: {{ .Values.credentials.database.username | include "clp-package.requireInput" | quote }} db-password: {{ .Values.credentials.database.password | include "clp-package.requireInput" | quote }} - queue-username: {{ .Values.credentials.queue.user | include "clp-package.requireInput" | quote }} + queue-username: {{ .Values.credentials.queue.username | include "clp-package.requireInput" | quote }} queue-password: {{ .Values.credentials.queue.password | include "clp-package.requireInput" | quote }} redis-password: {{ .Values.credentials.redis.password | include "clp-package.requireInput" | quote }} diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 023ebb93cd..7a69d9aeb3 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -184,12 +184,12 @@ clpConfig: credentials: # Database credentials database: - user: "clp-user" + username: "clp-user" password: "pass" # Queue credentials queue: - user: "clp-user" + username: "clp-user" password: "pass" # Redis credentials diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py index 608f938c29..43a748f491 100644 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -164,15 +164,15 @@ def _add_clp_env_vars( credentials = yaml.load(credentials_file, Loader=yaml.CSafeLoader) try: - database_user = _get_required_config_value( - credentials, "database.user", credentials_file_path + database_username = _get_required_config_value( + credentials, "database.username", credentials_file_path ) database_password = _get_required_config_value( credentials, "database.password", credentials_file_path ) except KeyError: return False - env_vars["PRESTO_COORDINATOR_CLPPROPERTIES_METADATA_DATABASE_USER"] = database_user + env_vars["PRESTO_COORDINATOR_CLPPROPERTIES_METADATA_DATABASE_USER"] = database_username env_vars["PRESTO_COORDINATOR_CLPPROPERTIES_METADATA_DATABASE_PASSWORD"] = database_password instance_id = _get_clp_package_instance_id(clp_config, clp_package_dir) From 1f6a5f09c12e767803712061f9d8daf04933479d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 23:30:36 -0500 Subject: [PATCH 015/149] refactor(helm): Split secrets into component-specific files; remove requireInput --- .../deployment/package-helm/templates/_helpers.tpl | 7 ------- .../templates/api-server-deployment.yaml | 8 ++++---- .../templates/compression-scheduler-deployment.yaml | 8 ++++---- .../package-helm/templates/database-secret.yaml | 11 +++++++++++ .../templates/database-statefulset.yaml | 12 ++++++------ .../templates/db-table-creator-job.yaml | 8 ++++---- .../templates/garbage-collector-deployment.yaml | 8 ++++---- .../templates/mcp-server-deployment.yaml | 8 ++++---- .../templates/query-scheduler-deployment.yaml | 8 ++++---- .../package-helm/templates/queue-secret.yaml | 11 +++++++++++ .../package-helm/templates/queue-statefulset.yaml | 8 ++++---- .../package-helm/templates/redis-secret.yaml | 10 ++++++++++ .../package-helm/templates/redis-statefulset.yaml | 4 ++-- tools/deployment/package-helm/templates/secret.yaml | 13 ------------- .../package-helm/templates/webui-deployment.yaml | 8 ++++---- 15 files changed, 72 insertions(+), 60 deletions(-) create mode 100644 tools/deployment/package-helm/templates/database-secret.yaml create mode 100644 tools/deployment/package-helm/templates/queue-secret.yaml create mode 100644 tools/deployment/package-helm/templates/redis-secret.yaml delete mode 100644 tools/deployment/package-helm/templates/secret.yaml diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 10479cd318..db7c62bc1c 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -58,13 +58,6 @@ Image reference for CLP Package {{- printf "%s:%s" .Values.image.clpPackage.repository $tag }} {{- end }} -{{/* -Require a value with a standard error message -*/}} -{{- define "clp-package.requireInput" -}} -{{- . | required "Please set a value." }} -{{- end }} - {{/* Creates timings for readiness probes (faster checks for quicker startup). */}} diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index e764af866d..4085d5ccb7 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -60,13 +60,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "CLP_LOGS_DIR" value: "/var/log/api_server" - name: "RUST_LOG" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 458cc0dc96..df89829013 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -67,13 +67,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} - name: "CLP_LOGS_DIR" diff --git a/tools/deployment/package-helm/templates/database-secret.yaml b/tools/deployment/package-helm/templates/database-secret.yaml new file mode 100644 index 0000000000..8ed317c108 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp-package.fullname" . }}-database-secret + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +type: "Opaque" +stringData: + username: {{ .Values.credentials.database.username | quote }} + password: {{ .Values.credentials.database.password | quote }} diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 95fdc13f06..7a14774276 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -55,18 +55,18 @@ spec: - name: "MYSQL_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "MYSQL_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "MYSQL_ROOT_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" ports: - name: "database" containerPort: 3306 diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index 97dd731c67..367e8934ff 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -32,13 +32,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 47b989b372..d1a3bfdf7f 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -43,13 +43,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "CLP_HOME" value: "/opt/clp" - name: "CLP_LOGGING_LEVEL" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 143a57b3ca..0cb9f3da9f 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -61,13 +61,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.mcp_server.logging_level | quote }} - name: "CLP_LOGS_DIR" diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index ea193637dc..6fc9565df3 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -69,13 +69,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} - name: "CLP_LOGS_DIR" diff --git a/tools/deployment/package-helm/templates/queue-secret.yaml b/tools/deployment/package-helm/templates/queue-secret.yaml new file mode 100644 index 0000000000..8896941986 --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp-package.fullname" . }}-queue-secret + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +type: "Opaque" +stringData: + username: {{ .Values.credentials.queue.username | quote }} + password: {{ .Values.credentials.queue.password | quote }} diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index 20ebf719a1..631cc2e1c0 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -46,13 +46,13 @@ spec: - name: "RABBITMQ_DEFAULT_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "queue-username" + name: {{ include "clp-package.fullname" . }}-queue-secret + key: "username" - name: "RABBITMQ_DEFAULT_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "queue-password" + name: {{ include "clp-package.fullname" . }}-queue-secret + key: "password" - name: "RABBITMQ_LOGS" value: "/var/log/rabbitmq/rabbitmq.log" ports: diff --git a/tools/deployment/package-helm/templates/redis-secret.yaml b/tools/deployment/package-helm/templates/redis-secret.yaml new file mode 100644 index 0000000000..4b1dae946a --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp-package.fullname" . }}-redis-secret + labels: + {{- include "clp-package.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +type: "Opaque" +stringData: + password: {{ .Values.credentials.redis.password | quote }} diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 882c8b33db..204c9b99c8 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -50,8 +50,8 @@ spec: - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "redis-password" + name: {{ include "clp-package.fullname" . }}-redis-secret + key: "password" ports: - name: "redis" containerPort: 6379 diff --git a/tools/deployment/package-helm/templates/secret.yaml b/tools/deployment/package-helm/templates/secret.yaml deleted file mode 100644 index 145d85f568..0000000000 --- a/tools/deployment/package-helm/templates/secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: "v1" -kind: "Secret" -metadata: - name: {{ include "clp-package.fullname" . }}-secret -type: "Opaque" -stringData: - db-username: {{ .Values.credentials.database.username | include "clp-package.requireInput" | quote }} - db-password: {{ .Values.credentials.database.password | include "clp-package.requireInput" | quote }} - - queue-username: {{ .Values.credentials.queue.username | include "clp-package.requireInput" | quote }} - queue-password: {{ .Values.credentials.queue.password | include "clp-package.requireInput" | quote }} - - redis-password: {{ .Values.credentials.redis.password | include "clp-package.requireInput" | quote }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 965a54420f..f08022b2fb 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -60,13 +60,13 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-username" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-secret - key: "db-password" + name: {{ include "clp-package.fullname" . }}-database-secret + key: "password" - name: "HOST" value: "0.0.0.0" - name: "NODE_ENV" From 89af6958d63e56a1faf854cf9660c9538a9934f4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 15 Nov 2025 23:33:38 -0500 Subject: [PATCH 016/149] Replace `clp-package` references with `clp` across templates. --- tools/deployment/package-helm/Chart.yaml | 2 +- .../package-helm/templates/00-local-pvs.yaml | 40 +++++----- .../package-helm/templates/_helpers.tpl | 80 +++++++++---------- .../templates/api-server-deployment.yaml | 38 ++++----- .../compression-scheduler-deployment.yaml | 40 +++++----- .../compression-worker-deployment.yaml | 52 ++++++------ .../package-helm/templates/configmap.yaml | 20 ++--- .../templates/database-secret.yaml | 4 +- .../templates/database-statefulset.yaml | 32 ++++---- .../templates/db-table-creator-job.yaml | 16 ++-- .../garbage-collector-deployment.yaml | 32 ++++---- .../templates/mcp-server-deployment.yaml | 38 ++++----- .../templates/query-scheduler-deployment.yaml | 44 +++++----- .../templates/query-worker-deployment.yaml | 40 +++++----- .../package-helm/templates/queue-secret.yaml | 4 +- .../templates/queue-statefulset.yaml | 26 +++--- .../package-helm/templates/redis-secret.yaml | 4 +- .../templates/redis-statefulset.yaml | 28 +++---- .../templates/reducer-deployment.yaml | 34 ++++---- .../results-cache-indices-creator-job.yaml | 12 +-- .../templates/results-cache-statefulset.yaml | 26 +++--- .../templates/webui-deployment.yaml | 40 +++++----- 22 files changed, 326 insertions(+), 326 deletions(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 6621aa2842..035b371050 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: "v2" -name: "clp-package" +name: "clp" description: "A Helm chart for CLP (Compressed Log Processor) package deployment" type: "application" version: "0.1.0" diff --git a/tools/deployment/package-helm/templates/00-local-pvs.yaml b/tools/deployment/package-helm/templates/00-local-pvs.yaml index 0bd1f663ed..94ebc58cd7 100644 --- a/tools/deployment/package-helm/templates/00-local-pvs.yaml +++ b/tools/deployment/package-helm/templates/00-local-pvs.yaml @@ -1,41 +1,41 @@ -{{- include "clp-package.createLocalDataPv" (list . "database") }} +{{- include "clp.createLocalDataPv" (list . "database") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "database") }} +{{- include "clp.createLocalLogsPv" (list . "database") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "queue") }} +{{- include "clp.createLocalLogsPv" (list . "queue") }} --- -{{- include "clp-package.createLocalDataPv" (list . "redis") }} +{{- include "clp.createLocalDataPv" (list . "redis") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "redis") }} +{{- include "clp.createLocalLogsPv" (list . "redis") }} --- -{{- include "clp-package.createLocalDataPv" (list . "results-cache") }} +{{- include "clp.createLocalDataPv" (list . "results-cache") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "results-cache") }} +{{- include "clp.createLocalLogsPv" (list . "results-cache") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "compression-scheduler") }} +{{- include "clp.createLocalLogsPv" (list . "compression-scheduler") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "compression-worker") }} +{{- include "clp.createLocalLogsPv" (list . "compression-worker") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "query-scheduler") }} +{{- include "clp.createLocalLogsPv" (list . "query-scheduler") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "query-worker") }} +{{- include "clp.createLocalLogsPv" (list . "query-worker") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "reducer") }} +{{- include "clp.createLocalLogsPv" (list . "reducer") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "garbage-collector") }} +{{- include "clp.createLocalLogsPv" (list . "garbage-collector") }} --- -{{- include "clp-package.createLocalLogsPv" (list . "api-server") }} +{{- include "clp.createLocalLogsPv" (list . "api-server") }} --- {{- if .Values.clpConfig.mcp_server }} -{{- include "clp-package.createLocalLogsPv" (list . "mcp-server") }} +{{- include "clp.createLocalLogsPv" (list . "mcp-server") }} --- {{- end }} -{{- include "clp-package.createStreamsPv" . }} +{{- include "clp.createStreamsPv" . }} --- -{{- include "clp-package.createTmpPv" . }} +{{- include "clp.createTmpPv" . }} --- -{{- include "clp-package.createArchivesPv" . }} +{{- include "clp.createArchivesPv" . }} --- -{{- include "clp-package.createStagedArchivesPv" . }} +{{- include "clp.createStagedArchivesPv" . }} --- -{{- include "clp-package.createStagedStreamsPv" . }} +{{- include "clp.createStagedStreamsPv" . }} diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index db7c62bc1c..3c6815ab24 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "clp-package.name" -}} +{{- define "clp.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "clp-package.fullname" -}} +{{- define "clp.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "clp-package.chart" -}} +{{- define "clp.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "clp-package.labels" -}} -helm.sh/chart: {{ include "clp-package.chart" . }} -{{ include "clp-package.selectorLabels" . }} +{{- define "clp.labels" -}} +helm.sh/chart: {{ include "clp.chart" . }} +{{ include "clp.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,15 +45,15 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "clp-package.selectorLabels" -}} -app.kubernetes.io/name: {{ include "clp-package.name" . }} +{{- define "clp.selectorLabels" -}} +app.kubernetes.io/name: {{ include "clp.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* Image reference for CLP Package */}} -{{- define "clp-package.image.ref" -}} +{{- define "clp.image.ref" -}} {{- $tag := .Values.image.clpPackage.tag | default .Chart.AppVersion }} {{- printf "%s:%s" .Values.image.clpPackage.repository $tag }} {{- end }} @@ -61,7 +61,7 @@ Image reference for CLP Package {{/* Creates timings for readiness probes (faster checks for quicker startup). */}} -{{- define "clp-package.readinessProbeTimings" -}} +{{- define "clp.readinessProbeTimings" -}} initialDelaySeconds: 1 periodSeconds: 1 timeoutSeconds: 1 @@ -71,7 +71,7 @@ failureThreshold: 3 {{/* Creates timings for liveness probes. */}} -{{- define "clp-package.livenessProbeTimings" -}} +{{- define "clp.livenessProbeTimings" -}} initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 @@ -81,29 +81,29 @@ failureThreshold: 3 {{/* CLP logs directory path on host */}} -{{- define "clp-package.logsDirHost" -}} +{{- define "clp.logsDirHost" -}} {{ .Values.storage.localPathBase }}/{{ .Values.clpConfig.logs_directory }} {{- end }} {{/* CLP data directory path on host */}} -{{- define "clp-package.dataDirHost" -}} +{{- define "clp.dataDirHost" -}} {{ .Values.storage.localPathBase }}/{{ .Values.clpConfig.data_directory }} {{- end }} {{/* Creates a local PersistentVolume for data storage. */}} -{{- define "clp-package.createLocalDataPv" -}} +{{- define "clp.createLocalDataPv" -}} {{- $root := index . 0 -}} {{- $component := index . 1 -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-data-pv-{{ $component }} + name: {{ include "clp.fullname" $root }}-data-pv-{{ $component }} labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: {{ $component | quote }} spec: capacity: @@ -112,7 +112,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp-package.dataDirHost" $root }}/{{ $component }} + path: {{ include "clp.dataDirHost" $root }}/{{ $component }} nodeAffinity: required: nodeSelectorTerms: @@ -124,15 +124,15 @@ spec: {{/* Creates a local PersistentVolume for logs storage. */}} -{{- define "clp-package.createLocalLogsPv" -}} +{{- define "clp.createLocalLogsPv" -}} {{- $root := index . 0 -}} {{- $component := index . 1 -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-logs-pv-{{ $component }} + name: {{ include "clp.fullname" $root }}-logs-pv-{{ $component }} labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: {{ $component | quote }} spec: capacity: @@ -141,7 +141,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp-package.logsDirHost" $root }}/{{ $component }} + path: {{ include "clp.logsDirHost" $root }}/{{ $component }} nodeAffinity: required: nodeSelectorTerms: @@ -153,14 +153,14 @@ spec: {{/* Creates a local PersistentVolume for streams storage (ReadWriteMany). */}} -{{- define "clp-package.createStreamsPv" -}} +{{- define "clp.createStreamsPv" -}} {{- $root := . -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-streams-pv + name: {{ include "clp.fullname" $root }}-streams-pv labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: "streams" spec: capacity: @@ -169,7 +169,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp-package.dataDirHost" $root }}/streams + path: {{ include "clp.dataDirHost" $root }}/streams nodeAffinity: required: nodeSelectorTerms: @@ -181,14 +181,14 @@ spec: {{/* Creates a local PersistentVolume for tmp storage (ReadWriteMany). */}} -{{- define "clp-package.createTmpPv" -}} +{{- define "clp.createTmpPv" -}} {{- $root := . -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-tmp-pv + name: {{ include "clp.fullname" $root }}-tmp-pv labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: "tmp" spec: capacity: @@ -209,14 +209,14 @@ spec: {{/* Creates a local PersistentVolume for archives storage (ReadWriteMany). */}} -{{- define "clp-package.createArchivesPv" -}} +{{- define "clp.createArchivesPv" -}} {{- $root := . -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-archives-pv + name: {{ include "clp.fullname" $root }}-archives-pv labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: "archives" spec: capacity: @@ -225,7 +225,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp-package.dataDirHost" $root }}/archives + path: {{ include "clp.dataDirHost" $root }}/archives nodeAffinity: required: nodeSelectorTerms: @@ -237,14 +237,14 @@ spec: {{/* Creates a local PersistentVolume for staged-archives storage (ReadWriteMany). */}} -{{- define "clp-package.createStagedArchivesPv" -}} +{{- define "clp.createStagedArchivesPv" -}} {{- $root := . -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-staged-archives-pv + name: {{ include "clp.fullname" $root }}-staged-archives-pv labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: "staged-archives" spec: capacity: @@ -253,7 +253,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp-package.dataDirHost" $root }}/staged-archives + path: {{ include "clp.dataDirHost" $root }}/staged-archives nodeAffinity: required: nodeSelectorTerms: @@ -265,14 +265,14 @@ spec: {{/* Creates a local PersistentVolume for staged-streams storage (ReadWriteMany). */}} -{{- define "clp-package.createStagedStreamsPv" -}} +{{- define "clp.createStagedStreamsPv" -}} {{- $root := . -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp-package.fullname" $root }}-staged-streams-pv + name: {{ include "clp.fullname" $root }}-staged-streams-pv labels: - {{- include "clp-package.labels" $root | nindent 4 }} + {{- include "clp.labels" $root | nindent 4 }} app.kubernetes.io/component: "staged-streams" spec: capacity: @@ -281,7 +281,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp-package.dataDirHost" $root }}/staged-streams + path: {{ include "clp.dataDirHost" $root }}/staged-streams nodeAffinity: required: nodeSelectorTerms: diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 4085d5ccb7..1f6752909e 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-api-server + name: {{ include "clp.fullname" . }}-api-server labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "api-server" spec: type: "NodePort" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "api-server" ports: - port: 3001 @@ -18,20 +18,20 @@ spec: apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-api-server + name: {{ include "clp.fullname" . }}-api-server labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "api-server" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "api-server" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "api-server" spec: terminationGracePeriodSeconds: 10 @@ -44,28 +44,28 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] - name: "wait-for-results-cache" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" ] containers: - name: "api-server" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "CLP_LOGS_DIR" value: "/var/log/api_server" @@ -88,36 +88,36 @@ spec: "--config", "/etc/clp-config.yml" ] livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} httpGet: path: "/health" port: 3001 readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} httpGet: path: "/health" port: 3001 volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "api-server-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-api-server-logs + claimName: {{ include "clp.fullname" . }}-api-server-logs --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-api-server-logs + name: {{ include "clp.fullname" . }}-api-server-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "api-server" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "api-server" resources: requests: diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index df89829013..fe43c9e896 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-compression-scheduler + name: {{ include "clp.fullname" . }}-compression-scheduler labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-scheduler" spec: clusterIP: "None" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "compression-scheduler" ports: - port: 7000 @@ -17,20 +17,20 @@ spec: apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-compression-scheduler + name: {{ include "clp.fullname" . }}-compression-scheduler labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-scheduler" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "compression-scheduler" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "compression-scheduler" spec: terminationGracePeriodSeconds: 300 @@ -43,36 +43,36 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] - name: "wait-for-queue" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" ] - name: "wait-for-redis" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" ] containers: - name: "compression-scheduler" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} @@ -81,7 +81,7 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" @@ -100,10 +100,10 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "compression-scheduler-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-compression-scheduler-logs + claimName: {{ include "clp.fullname" . }}-compression-scheduler-logs - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" @@ -112,16 +112,16 @@ spec: apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-compression-scheduler-logs + name: {{ include "clp.fullname" . }}-compression-scheduler-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-scheduler" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "compression-scheduler" resources: requests: diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 2f9449fbe4..6ce71cb371 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -1,20 +1,20 @@ apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-compression-worker + name: {{ include "clp.fullname" . }}-compression-worker labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-worker" spec: replicas: {{ .Values.workerConcurrency }} selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "compression-worker" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "compression-worker" spec: terminationGracePeriodSeconds: 60 @@ -27,21 +27,21 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" ] - name: "wait-for-redis" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" ] containers: - name: "compression-worker" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yml" - name: "CLP_HOME" @@ -55,7 +55,7 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" @@ -86,37 +86,37 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "compression-worker-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-compression-worker-logs + claimName: {{ include "clp.fullname" . }}-compression-worker-logs - name: "archives" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-archives + claimName: {{ include "clp.fullname" . }}-archives - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" type: "Directory" - name: "staged-archives" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-staged-archives + claimName: {{ include "clp.fullname" . }}-staged-archives - name: "tmp" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-tmp + claimName: {{ include "clp.fullname" . }}-tmp --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-compression-worker-logs + name: {{ include "clp.fullname" . }}-compression-worker-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-worker" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "compression-worker" resources: requests: @@ -125,16 +125,16 @@ spec: apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-archives + name: {{ include "clp.fullname" . }}-archives labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "archives" spec: accessModes: ["ReadWriteMany"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "archives" resources: requests: @@ -143,16 +143,16 @@ spec: apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-staged-archives + name: {{ include "clp.fullname" . }}-staged-archives labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "staged-archives" spec: accessModes: ["ReadWriteMany"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "staged-archives" resources: requests: @@ -161,16 +161,16 @@ spec: apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-tmp + name: {{ include "clp.fullname" . }}-tmp labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "tmp" spec: accessModes: ["ReadWriteMany"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "tmp" resources: requests: diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index fa570eacf0..cd94d9d489 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -1,9 +1,9 @@ apiVersion: "v1" kind: "ConfigMap" metadata: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} data: clp-config.yml: | api_server: @@ -33,7 +33,7 @@ data: database: auto_commit: false compress: true - host: {{ include "clp-package.fullname" . }}-database + host: {{ include "clp.fullname" . }}-database name: {{ .Values.clpConfig.database.name | quote }} port: 3306 ssl_cert: null @@ -53,7 +53,7 @@ data: storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} presto: {{ .Values.clpConfig.presto }} query_scheduler: - host: {{ include "clp-package.fullname" . }}-query-scheduler + host: {{ include "clp.fullname" . }}-query-scheduler jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} num_archives_to_search_per_sub_job: {{ .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job }} @@ -61,21 +61,21 @@ data: query_worker: logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }} queue: - host: {{ include "clp-package.fullname" . }}-queue + host: {{ include "clp.fullname" . }}-queue port: 5672 redis: compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} - host: {{ include "clp-package.fullname" . }}-redis + host: {{ include "clp.fullname" . }}-redis port: 6379 query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} reducer: base_port: {{ .Values.clpConfig.reducer.base_port }} - host: {{ include "clp-package.fullname" . }}-reducer + host: {{ include "clp.fullname" . }}-reducer logging_level: {{ .Values.clpConfig.reducer.logging_level | quote }} upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval }} results_cache: db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} - host: {{ include "clp-package.fullname" . }}-results-cache + host: {{ include "clp.fullname" . }}-results-cache port: 27017 retention_period: {{ .Values.clpConfig.results_cache.retention_period }} stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} @@ -128,11 +128,11 @@ data: } webui-server-settings.json: | { - "SqlDbHost": "{{ include "clp-package.fullname" . }}-database", + "SqlDbHost": "{{ include "clp.fullname" . }}-database", "SqlDbPort": 3306, "SqlDbName": {{ .Values.clpConfig.database.name | quote }}, "SqlDbQueryJobsTableName": "query_jobs", - "MongoDbHost": "{{ include "clp-package.fullname" . }}-results-cache", + "MongoDbHost": "{{ include "clp.fullname" . }}-results-cache", "MongoDbPort": 27017, "MongoDbName": {{ .Values.clpConfig.results_cache.db_name | quote }}, "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, diff --git a/tools/deployment/package-helm/templates/database-secret.yaml b/tools/deployment/package-helm/templates/database-secret.yaml index 8ed317c108..f2814d6877 100644 --- a/tools/deployment/package-helm/templates/database-secret.yaml +++ b/tools/deployment/package-helm/templates/database-secret.yaml @@ -1,9 +1,9 @@ apiVersion: "v1" kind: "Secret" metadata: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "database" type: "Opaque" stringData: diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 7a14774276..ca2796fc04 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -1,9 +1,9 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-database + name: {{ include "clp.fullname" . }}-database labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "database" spec: {{- if .Values.allowSbinScripts }} @@ -12,7 +12,7 @@ spec: clusterIP: "None" {{- end }} selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "database" ports: - port: 3306 @@ -24,21 +24,21 @@ spec: apiVersion: "apps/v1" kind: "StatefulSet" metadata: - name: {{ include "clp-package.fullname" . }}-database + name: {{ include "clp.fullname" . }}-database labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "database" spec: - serviceName: {{ include "clp-package.fullname" . }}-database + serviceName: {{ include "clp.fullname" . }}-database replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "database" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "database" spec: securityContext: @@ -55,17 +55,17 @@ spec: - name: "MYSQL_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "MYSQL_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "MYSQL_ROOT_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" ports: - name: "database" @@ -80,7 +80,7 @@ spec: subPath: "mysql-logging.cnf" readOnly: true livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: &database-health-check command: [ "mysqladmin", "ping", @@ -88,13 +88,13 @@ spec: "-h", "127.0.0.1" ] readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: <<: *database-health-check volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config volumeClaimTemplates: - metadata: name: "database-data" @@ -103,7 +103,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "database" resources: requests: @@ -115,7 +115,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "database" resources: requests: diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index 367e8934ff..f8f89a260a 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -1,15 +1,15 @@ apiVersion: "batch/v1" kind: "Job" metadata: - name: {{ include "clp-package.fullname" . }}-db-table-creator + name: {{ include "clp.fullname" . }}-db-table-creator labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "db-table-creator" spec: template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "db-table-creator" spec: restartPolicy: "OnFailure" @@ -22,22 +22,22 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] containers: - name: "db-table-creator" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" @@ -55,4 +55,4 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index d1a3bfdf7f..483946dd36 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -1,20 +1,20 @@ apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-garbage-collector + name: {{ include "clp.fullname" . }}-garbage-collector labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "garbage-collector" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "garbage-collector" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "garbage-collector" spec: terminationGracePeriodSeconds: 10 @@ -27,28 +27,28 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] - name: "wait-for-results-cache" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" ] containers: - name: "garbage-collector" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "CLP_HOME" value: "/opt/clp" @@ -77,30 +77,30 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "garbage-collector-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-garbage-collector-logs + claimName: {{ include "clp.fullname" . }}-garbage-collector-logs - name: "archives" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-archives + claimName: {{ include "clp.fullname" . }}-archives - name: "streams" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-streams + claimName: {{ include "clp.fullname" . }}-streams --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-garbage-collector-logs + name: {{ include "clp.fullname" . }}-garbage-collector-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "garbage-collector" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "garbage-collector" resources: requests: diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 0cb9f3da9f..e3de1b3550 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -2,14 +2,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-mcp-server + name: {{ include "clp.fullname" . }}-mcp-server labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "mcp-server" spec: type: "NodePort" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "mcp-server" ports: - port: 8000 @@ -19,20 +19,20 @@ spec: apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-mcp-server + name: {{ include "clp.fullname" . }}-mcp-server labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "mcp-server" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "mcp-server" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "mcp-server" spec: terminationGracePeriodSeconds: 10 @@ -45,28 +45,28 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] - name: "wait-for-results-cache" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" ] containers: - name: "mcp-server" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.mcp_server.logging_level | quote }} @@ -92,36 +92,36 @@ spec: "--config-path", "/etc/clp-config.yml" ] livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} httpGet: path: "/health" port: 8000 readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} httpGet: path: "/health" port: 8000 volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "mcp-server-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-mcp-server-logs + claimName: {{ include "clp.fullname" . }}-mcp-server-logs --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-mcp-server-logs + name: {{ include "clp.fullname" . }}-mcp-server-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "mcp-server" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "mcp-server" resources: requests: diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 6fc9565df3..97f1d133f9 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-query-scheduler + name: {{ include "clp.fullname" . }}-query-scheduler labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-scheduler" spec: clusterIP: "None" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "query-scheduler" ports: - port: 7000 @@ -17,24 +17,24 @@ spec: apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-query-scheduler + name: {{ include "clp.fullname" . }}-query-scheduler labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-scheduler" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "query-scheduler" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "query-scheduler" spec: - hostname: {{ include "clp-package.fullname" . }}-query-scheduler - subdomain: {{ include "clp-package.fullname" . }}-query-scheduler + hostname: {{ include "clp.fullname" . }}-query-scheduler + subdomain: {{ include "clp.fullname" . }}-query-scheduler terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} @@ -45,36 +45,36 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] - name: "wait-for-queue" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" ] - name: "wait-for-redis" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" ] containers: - name: "query-scheduler" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} @@ -83,7 +83,7 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" ports: - name: "query-scheduler" containerPort: 7000 @@ -102,24 +102,24 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "query-scheduler-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-query-scheduler-logs + claimName: {{ include "clp.fullname" . }}-query-scheduler-logs --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-query-scheduler-logs + name: {{ include "clp.fullname" . }}-query-scheduler-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-scheduler" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "query-scheduler" resources: requests: diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 83e9198010..1de4df841d 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -1,20 +1,20 @@ apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-query-worker + name: {{ include "clp.fullname" . }}-query-worker labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-worker" spec: replicas: {{ .Values.workerConcurrency }} selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "query-worker" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "query-worker" spec: terminationGracePeriodSeconds: 60 @@ -27,21 +27,21 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" ] - name: "wait-for-redis" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" ] containers: - name: "query-worker" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp-package.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yml" - name: "CLP_HOME" @@ -55,7 +55,7 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp-package.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" + value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" volumeMounts: - name: "config" mountPath: "/etc/clp-config.yml" @@ -83,33 +83,33 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "query-worker-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-query-worker-logs + claimName: {{ include "clp.fullname" . }}-query-worker-logs - name: "archives" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-archives + claimName: {{ include "clp.fullname" . }}-archives - name: "staged-streams" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-staged-streams + claimName: {{ include "clp.fullname" . }}-staged-streams - name: "streams" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-streams + claimName: {{ include "clp.fullname" . }}-streams --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-query-worker-logs + name: {{ include "clp.fullname" . }}-query-worker-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-worker" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "query-worker" resources: requests: @@ -118,16 +118,16 @@ spec: apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-staged-streams + name: {{ include "clp.fullname" . }}-staged-streams labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "staged-streams" spec: accessModes: ["ReadWriteMany"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "staged-streams" resources: requests: diff --git a/tools/deployment/package-helm/templates/queue-secret.yaml b/tools/deployment/package-helm/templates/queue-secret.yaml index 8896941986..8b909ce22a 100644 --- a/tools/deployment/package-helm/templates/queue-secret.yaml +++ b/tools/deployment/package-helm/templates/queue-secret.yaml @@ -1,9 +1,9 @@ apiVersion: "v1" kind: "Secret" metadata: - name: {{ include "clp-package.fullname" . }}-queue-secret + name: {{ include "clp.fullname" . }}-queue-secret labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "queue" type: "Opaque" stringData: diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index 631cc2e1c0..93c0733b27 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-queue + name: {{ include "clp.fullname" . }}-queue labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "queue" spec: clusterIP: "None" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "queue" ports: - port: 5672 @@ -17,21 +17,21 @@ spec: apiVersion: "apps/v1" kind: "StatefulSet" metadata: - name: {{ include "clp-package.fullname" . }}-queue + name: {{ include "clp.fullname" . }}-queue labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "queue" spec: - serviceName: {{ include "clp-package.fullname" . }}-queue + serviceName: {{ include "clp.fullname" . }}-queue replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "queue" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "queue" spec: securityContext: @@ -46,12 +46,12 @@ spec: - name: "RABBITMQ_DEFAULT_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-queue-secret + name: {{ include "clp.fullname" . }}-queue-secret key: "username" - name: "RABBITMQ_DEFAULT_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-queue-secret + name: {{ include "clp.fullname" . }}-queue-secret key: "password" - name: "RABBITMQ_LOGS" value: "/var/log/rabbitmq/rabbitmq.log" @@ -62,11 +62,11 @@ spec: - name: "queue-logs" mountPath: "/var/log/rabbitmq" livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: &queue-health-check command: ["rabbitmq-diagnostics", "check_running"] readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: <<: *queue-health-check volumeClaimTemplates: @@ -77,7 +77,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "queue" resources: requests: diff --git a/tools/deployment/package-helm/templates/redis-secret.yaml b/tools/deployment/package-helm/templates/redis-secret.yaml index 4b1dae946a..4923e0ce5d 100644 --- a/tools/deployment/package-helm/templates/redis-secret.yaml +++ b/tools/deployment/package-helm/templates/redis-secret.yaml @@ -1,9 +1,9 @@ apiVersion: "v1" kind: "Secret" metadata: - name: {{ include "clp-package.fullname" . }}-redis-secret + name: {{ include "clp.fullname" . }}-redis-secret labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "redis" type: "Opaque" stringData: diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 204c9b99c8..5db313885d 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-redis + name: {{ include "clp.fullname" . }}-redis labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "redis" spec: clusterIP: "None" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "redis" ports: - port: 6379 @@ -17,21 +17,21 @@ spec: apiVersion: "apps/v1" kind: "StatefulSet" metadata: - name: {{ include "clp-package.fullname" . }}-redis + name: {{ include "clp.fullname" . }}-redis labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "redis" spec: - serviceName: {{ include "clp-package.fullname" . }}-redis + serviceName: {{ include "clp.fullname" . }}-redis replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "redis" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "redis" spec: securityContext: @@ -50,7 +50,7 @@ spec: - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-redis-secret + name: {{ include "clp.fullname" . }}-redis-secret key: "password" ports: - name: "redis" @@ -65,7 +65,7 @@ spec: subPath: "redis.conf" readOnly: true livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: &redis-health-check command: - "redis-cli" @@ -77,13 +77,13 @@ spec: - "$(REDIS_PASSWORD)" - "PING" readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: <<: *redis-health-check volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config volumeClaimTemplates: - metadata: name: "redis-data" @@ -92,7 +92,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "redis" resources: requests: @@ -104,7 +104,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "redis" resources: requests: diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index 869be59d5b..db4a7af1c3 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-reducer + name: {{ include "clp.fullname" . }}-reducer labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "reducer" spec: clusterIP: "None" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "reducer" ports: {{- range $i := until (int .Values.workerConcurrency) }} @@ -20,24 +20,24 @@ spec: apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-reducer + name: {{ include "clp.fullname" . }}-reducer labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "reducer" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "reducer" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "reducer" spec: - hostname: {{ include "clp-package.fullname" . }}-reducer - subdomain: {{ include "clp-package.fullname" . }}-reducer + hostname: {{ include "clp.fullname" . }}-reducer + subdomain: {{ include "clp.fullname" . }}-reducer terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} @@ -48,17 +48,17 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-query-scheduler 7000; do echo waiting for query-scheduler; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-query-scheduler 7000; do echo waiting for query-scheduler; sleep 2; done" ] - name: "wait-for-results-cache" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" ] containers: - name: "reducer" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "CLP_HOME" @@ -86,24 +86,24 @@ spec: volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "reducer-logs" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-reducer-logs + claimName: {{ include "clp.fullname" . }}-reducer-logs --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-reducer-logs + name: {{ include "clp.fullname" . }}-reducer-logs labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "reducer" spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "reducer" resources: requests: diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index bdc479a54e..4b45368f3f 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -1,15 +1,15 @@ apiVersion: "batch/v1" kind: "Job" metadata: - name: {{ include "clp-package.fullname" . }}-results-cache-indices-creator + name: {{ include "clp.fullname" . }}-results-cache-indices-creator labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "results-cache-indices-creator" spec: template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "results-cache-indices-creator" spec: restartPolicy: "OnFailure" @@ -22,11 +22,11 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" ] containers: - name: "results-cache-indices-creator" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "PYTHONPATH" @@ -34,6 +34,6 @@ spec: command: [ "python3", "-u", "-m", "clp_py_utils.initialize-results-cache", - "--uri", "mongodb://{{ include "clp-package.fullname" . }}-results-cache:27017/{{ .Values.clpConfig.results_cache.db_name }}", + "--uri", "mongodb://{{ include "clp.fullname" . }}-results-cache:27017/{{ .Values.clpConfig.results_cache.db_name }}", "--stream-collection", {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} ] diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 010fde0c98..c4453cd9d0 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -1,9 +1,9 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-results-cache + name: {{ include "clp.fullname" . }}-results-cache labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "results-cache" spec: {{- if .Values.allowSbinScripts }} @@ -12,7 +12,7 @@ spec: clusterIP: "None" {{- end }} selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "results-cache" ports: - port: 27017 @@ -24,21 +24,21 @@ spec: apiVersion: "apps/v1" kind: "StatefulSet" metadata: - name: {{ include "clp-package.fullname" . }}-results-cache + name: {{ include "clp.fullname" . }}-results-cache labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "results-cache" spec: - serviceName: {{ include "clp-package.fullname" . }}-results-cache + serviceName: {{ include "clp.fullname" . }}-results-cache replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "results-cache" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "results-cache" spec: securityContext: @@ -67,7 +67,7 @@ spec: subPath: "mongod.conf" readOnly: true livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: &results-cache-health-check command: - "mongosh" @@ -76,13 +76,13 @@ spec: - "--eval" - "db.runCommand('ping').ok" readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: <<: *results-cache-health-check volumes: - name: "config" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config volumeClaimTemplates: - metadata: name: "results-cache-data" @@ -91,7 +91,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "results-cache" resources: requests: @@ -103,7 +103,7 @@ spec: storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 12 }} + {{- include "clp.selectorLabels" . | nindent 12 }} app.kubernetes.io/component: "results-cache" resources: requests: diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index f08022b2fb..873e56c8f7 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -1,14 +1,14 @@ apiVersion: "v1" kind: "Service" metadata: - name: {{ include "clp-package.fullname" . }}-webui + name: {{ include "clp.fullname" . }}-webui labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "webui" spec: type: "NodePort" selector: - {{- include "clp-package.selectorLabels" . | nindent 4 }} + {{- include "clp.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: "webui" ports: - port: 4000 @@ -18,20 +18,20 @@ spec: apiVersion: "apps/v1" kind: "Deployment" metadata: - name: {{ include "clp-package.fullname" . }}-webui + name: {{ include "clp.fullname" . }}-webui labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "webui" spec: replicas: 1 selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "webui" template: metadata: labels: - {{- include "clp-package.labels" . | nindent 8 }} + {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "webui" spec: terminationGracePeriodSeconds: 60 @@ -44,28 +44,28 @@ spec: image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" ] - name: "wait-for-results-cache" image: "busybox:latest" command: [ "sh", "-c", - "until nc -z {{ include "clp-package.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" + "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" ] containers: - name: "webui" - image: "{{ include "clp-package.image.ref" . }}" + image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "Always" env: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp-package.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database-secret key: "password" - name: "HOST" value: "0.0.0.0" @@ -96,37 +96,37 @@ spec: "/opt/clp/var/www/webui/server/dist/src/main.js" ] livenessProbe: - {{- include "clp-package.livenessProbeTimings" . | nindent 12 }} + {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: port: 4000 readinessProbe: - {{- include "clp-package.readinessProbeTimings" . | nindent 12 }} + {{- include "clp.readinessProbeTimings" . | nindent 12 }} tcpSocket: port: 4000 volumes: - name: "client-settings" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "server-settings" configMap: - name: {{ include "clp-package.fullname" . }}-config + name: {{ include "clp.fullname" . }}-config - name: "streams" persistentVolumeClaim: - claimName: {{ include "clp-package.fullname" . }}-streams + claimName: {{ include "clp.fullname" . }}-streams --- apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp-package.fullname" . }}-streams + name: {{ include "clp.fullname" . }}-streams labels: - {{- include "clp-package.labels" . | nindent 4 }} + {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "streams" spec: accessModes: ["ReadWriteMany"] storageClassName: "local-storage" selector: matchLabels: - {{- include "clp-package.selectorLabels" . | nindent 6 }} + {{- include "clp.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: "streams" resources: requests: From ca7d8200b854c2070c1bbce5b14b842fcd680e58 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 00:00:33 -0500 Subject: [PATCH 017/149] split yamls --- .../package-helm/templates/00-local-pvs.yaml | 41 ----------- .../templates/api-server-deployment.yaml | 35 --------- .../templates/api-server-logs-pv.yaml | 1 + .../templates/api-server-pvc.yaml | 17 +++++ .../templates/api-server-service.yaml | 16 +++++ .../package-helm/templates/archives-pv.yaml | 1 + .../compression-scheduler-deployment.yaml | 34 --------- .../compression-scheduler-logs-pv.yaml | 1 + .../templates/compression-scheduler-pvc.yaml | 17 +++++ .../compression-scheduler-service.yaml | 15 ++++ .../compression-worker-archives-pvc.yaml | 17 +++++ .../compression-worker-deployment.yaml | 72 ------------------- .../templates/compression-worker-logs-pv.yaml | 1 + .../compression-worker-logs-pvc.yaml | 17 +++++ ...ompression-worker-staged-archives-pvc.yaml | 17 +++++ .../templates/database-logs-pv.yaml | 1 + .../package-helm/templates/database-pv.yaml | 1 + .../templates/database-service.yaml | 22 ++++++ .../templates/database-statefulset.yaml | 23 ------ .../garbage-collector-deployment.yaml | 18 ----- .../templates/garbage-collector-logs-pv.yaml | 1 + .../templates/garbage-collector-pvc.yaml | 17 +++++ .../templates/mcp-server-deployment.yaml | 37 ---------- .../templates/mcp-server-logs-pv.yaml | 3 + .../templates/mcp-server-pvc.yaml | 18 +++++ .../templates/mcp-server-service.yaml | 17 +++++ .../templates/query-scheduler-deployment.yaml | 34 --------- .../templates/query-scheduler-logs-pv.yaml | 1 + .../templates/query-scheduler-pvc.yaml | 17 +++++ .../templates/query-scheduler-service.yaml | 15 ++++ .../templates/query-worker-archives-pvc.yaml | 17 +++++ .../templates/query-worker-deployment.yaml | 36 ---------- .../templates/query-worker-logs-pv.yaml | 1 + .../templates/query-worker-logs-pvc.yaml | 17 +++++ .../query-worker-staged-streams-pvc.yaml | 17 +++++ .../package-helm/templates/queue-logs-pv.yaml | 1 + .../package-helm/templates/queue-service.yaml | 15 ++++ .../templates/queue-statefulset.yaml | 16 ----- .../package-helm/templates/redis-logs-pv.yaml | 1 + .../package-helm/templates/redis-pv.yaml | 1 + .../package-helm/templates/redis-service.yaml | 15 ++++ .../templates/redis-statefulset.yaml | 16 ----- .../templates/reducer-deployment.yaml | 37 ---------- .../templates/reducer-logs-pv.yaml | 1 + .../package-helm/templates/reducer-pvc.yaml | 17 +++++ .../templates/reducer-service.yaml | 18 +++++ .../templates/results-cache-logs-pv.yaml | 1 + .../templates/results-cache-pv.yaml | 1 + .../templates/results-cache-service.yaml | 22 ++++++ .../templates/results-cache-statefulset.yaml | 23 ------ .../templates/staged-archives-pv.yaml | 1 + .../templates/staged-streams-pv.yaml | 1 + .../package-helm/templates/streams-pv.yaml | 1 + .../package-helm/templates/tmp-pv.yaml | 1 + .../templates/webui-deployment.yaml | 35 --------- .../package-helm/templates/webui-pvc.yaml | 17 +++++ .../package-helm/templates/webui-service.yaml | 16 +++++ 57 files changed, 415 insertions(+), 457 deletions(-) delete mode 100644 tools/deployment/package-helm/templates/00-local-pvs.yaml create mode 100644 tools/deployment/package-helm/templates/api-server-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/api-server-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/api-server-service.yaml create mode 100644 tools/deployment/package-helm/templates/archives-pv.yaml create mode 100644 tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/compression-scheduler-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/compression-scheduler-service.yaml create mode 100644 tools/deployment/package-helm/templates/compression-worker-archives-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/database-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/database-pv.yaml create mode 100644 tools/deployment/package-helm/templates/database-service.yaml create mode 100644 tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/garbage-collector-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/mcp-server-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/mcp-server-service.yaml create mode 100644 tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/query-scheduler-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/query-scheduler-service.yaml create mode 100644 tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/query-worker-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/queue-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/queue-service.yaml create mode 100644 tools/deployment/package-helm/templates/redis-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/redis-pv.yaml create mode 100644 tools/deployment/package-helm/templates/redis-service.yaml create mode 100644 tools/deployment/package-helm/templates/reducer-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/reducer-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/reducer-service.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-pv.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-service.yaml create mode 100644 tools/deployment/package-helm/templates/staged-archives-pv.yaml create mode 100644 tools/deployment/package-helm/templates/staged-streams-pv.yaml create mode 100644 tools/deployment/package-helm/templates/streams-pv.yaml create mode 100644 tools/deployment/package-helm/templates/tmp-pv.yaml create mode 100644 tools/deployment/package-helm/templates/webui-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/webui-service.yaml diff --git a/tools/deployment/package-helm/templates/00-local-pvs.yaml b/tools/deployment/package-helm/templates/00-local-pvs.yaml deleted file mode 100644 index 94ebc58cd7..0000000000 --- a/tools/deployment/package-helm/templates/00-local-pvs.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- include "clp.createLocalDataPv" (list . "database") }} ---- -{{- include "clp.createLocalLogsPv" (list . "database") }} ---- -{{- include "clp.createLocalLogsPv" (list . "queue") }} ---- -{{- include "clp.createLocalDataPv" (list . "redis") }} ---- -{{- include "clp.createLocalLogsPv" (list . "redis") }} ---- -{{- include "clp.createLocalDataPv" (list . "results-cache") }} ---- -{{- include "clp.createLocalLogsPv" (list . "results-cache") }} ---- -{{- include "clp.createLocalLogsPv" (list . "compression-scheduler") }} ---- -{{- include "clp.createLocalLogsPv" (list . "compression-worker") }} ---- -{{- include "clp.createLocalLogsPv" (list . "query-scheduler") }} ---- -{{- include "clp.createLocalLogsPv" (list . "query-worker") }} ---- -{{- include "clp.createLocalLogsPv" (list . "reducer") }} ---- -{{- include "clp.createLocalLogsPv" (list . "garbage-collector") }} ---- -{{- include "clp.createLocalLogsPv" (list . "api-server") }} ---- -{{- if .Values.clpConfig.mcp_server }} -{{- include "clp.createLocalLogsPv" (list . "mcp-server") }} ---- -{{- end }} -{{- include "clp.createStreamsPv" . }} ---- -{{- include "clp.createTmpPv" . }} ---- -{{- include "clp.createArchivesPv" . }} ---- -{{- include "clp.createStagedArchivesPv" . }} ---- -{{- include "clp.createStagedStreamsPv" . }} diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 1f6752909e..a55769e02c 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -1,20 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-api-server - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "api-server" -spec: - type: "NodePort" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "api-server" - ports: - - port: 3001 - targetPort: "http" - nodePort: {{ .Values.clpConfig.api_server.port }} ---- apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -104,21 +87,3 @@ spec: - name: "api-server-logs" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-api-server-logs ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-api-server-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "api-server" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "api-server" - resources: - requests: - storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml new file mode 100644 index 0000000000..c717de287a --- /dev/null +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "api-server") }} diff --git a/tools/deployment/package-helm/templates/api-server-pvc.yaml b/tools/deployment/package-helm/templates/api-server-pvc.yaml new file mode 100644 index 0000000000..53db492b5a --- /dev/null +++ b/tools/deployment/package-helm/templates/api-server-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-api-server-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "api-server" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/api-server-service.yaml b/tools/deployment/package-helm/templates/api-server-service.yaml new file mode 100644 index 0000000000..7c24c0f6e5 --- /dev/null +++ b/tools/deployment/package-helm/templates/api-server-service.yaml @@ -0,0 +1,16 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-api-server + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" +spec: + type: "NodePort" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "api-server" + ports: + - port: 3001 + targetPort: "http" + nodePort: {{ .Values.clpConfig.api_server.port }} diff --git a/tools/deployment/package-helm/templates/archives-pv.yaml b/tools/deployment/package-helm/templates/archives-pv.yaml new file mode 100644 index 0000000000..84edd505ec --- /dev/null +++ b/tools/deployment/package-helm/templates/archives-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createArchivesPv" . }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index fe43c9e896..21d210e18b 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -1,19 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-compression-scheduler - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "compression-scheduler" -spec: - clusterIP: "None" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "compression-scheduler" - ports: - - port: 7000 - targetPort: 7000 ---- apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -108,21 +92,3 @@ spec: hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" type: "Directory" ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-compression-scheduler-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "compression-scheduler" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "compression-scheduler" - resources: - requests: - storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml new file mode 100644 index 0000000000..05becd1f26 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "compression-scheduler") }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-pvc.yaml new file mode 100644 index 0000000000..9e5a5170d9 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-scheduler-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-compression-scheduler-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-scheduler" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-service.yaml b/tools/deployment/package-helm/templates/compression-scheduler-service.yaml new file mode 100644 index 0000000000..605daf9055 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-scheduler-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-compression-scheduler + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "compression-scheduler" + ports: + - port: 7000 + targetPort: 7000 diff --git a/tools/deployment/package-helm/templates/compression-worker-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-archives-pvc.yaml new file mode 100644 index 0000000000..2534d4ca78 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-worker-archives-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-archives + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "archives" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "archives" + resources: + requests: + storage: "50Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 6ce71cb371..35db257d75 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -103,75 +103,3 @@ spec: - name: "tmp" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-tmp ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-compression-worker-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "compression-worker" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "compression-worker" - resources: - requests: - storage: "5Gi" ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-archives - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "archives" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "archives" - resources: - requests: - storage: "50Gi" ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-staged-archives - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "staged-archives" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "staged-archives" - resources: - requests: - storage: "20Gi" ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-tmp - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "tmp" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "tmp" - resources: - requests: - storage: "10Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml new file mode 100644 index 0000000000..737e205c52 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "compression-worker") }} diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml new file mode 100644 index 0000000000..617290ea74 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-compression-worker-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-worker" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-worker" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml new file mode 100644 index 0000000000..541516ed3b --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-staged-archives + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "staged-archives" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "staged-archives" + resources: + requests: + storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml new file mode 100644 index 0000000000..ada6357ba0 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "database") }} diff --git a/tools/deployment/package-helm/templates/database-pv.yaml b/tools/deployment/package-helm/templates/database-pv.yaml new file mode 100644 index 0000000000..a72331b6f1 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalDataPv" (list . "database") }} diff --git a/tools/deployment/package-helm/templates/database-service.yaml b/tools/deployment/package-helm/templates/database-service.yaml new file mode 100644 index 0000000000..fe6f2f823e --- /dev/null +++ b/tools/deployment/package-helm/templates/database-service.yaml @@ -0,0 +1,22 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-database + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +spec: + {{- if .Values.allowSbinScripts }} + type: "NodePort" + {{- else }} + clusterIP: "None" + {{- end }} + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "database" + ports: + - port: 3306 + targetPort: "database" + {{- if .Values.allowSbinScripts }} + nodePort: {{ .Values.clpConfig.database.port }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index ca2796fc04..911adbd117 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -1,26 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-database - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "database" -spec: - {{- if .Values.allowSbinScripts }} - type: "NodePort" - {{- else }} - clusterIP: "None" - {{- end }} - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "database" - ports: - - port: 3306 - targetPort: "database" - {{- if .Values.allowSbinScripts }} - nodePort: {{ .Values.clpConfig.database.port }} - {{- end }} ---- apiVersion: "apps/v1" kind: "StatefulSet" metadata: diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 483946dd36..2369b9538a 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -87,21 +87,3 @@ spec: - name: "streams" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-streams ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-garbage-collector-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "garbage-collector" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "garbage-collector" - resources: - requests: - storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml new file mode 100644 index 0000000000..a55e94608f --- /dev/null +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "garbage-collector") }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-pvc.yaml new file mode 100644 index 0000000000..ec16d99ae7 --- /dev/null +++ b/tools/deployment/package-helm/templates/garbage-collector-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-garbage-collector-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "garbage-collector" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "garbage-collector" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index e3de1b3550..8465ea59ae 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -1,21 +1,3 @@ -{{- if .Values.clpConfig.mcp_server }} -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-mcp-server - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "mcp-server" -spec: - type: "NodePort" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "mcp-server" - ports: - - port: 8000 - targetPort: "http" - nodePort: {{ .Values.clpConfig.mcp_server.port }} ---- apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -108,22 +90,3 @@ spec: - name: "mcp-server-logs" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-mcp-server-logs ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-mcp-server-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "mcp-server" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "mcp-server" - resources: - requests: - storage: "5Gi" -{{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml new file mode 100644 index 0000000000..48fd58f61f --- /dev/null +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -0,0 +1,3 @@ +{{- if .Values.clpConfig.mcp_server }} +{{- include "clp.createLocalLogsPv" (list . "mcp-server") }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-pvc.yaml b/tools/deployment/package-helm/templates/mcp-server-pvc.yaml new file mode 100644 index 0000000000..98caca2667 --- /dev/null +++ b/tools/deployment/package-helm/templates/mcp-server-pvc.yaml @@ -0,0 +1,18 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-mcp-server-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "mcp-server" + resources: + requests: + storage: "5Gi" +{{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-service.yaml b/tools/deployment/package-helm/templates/mcp-server-service.yaml new file mode 100644 index 0000000000..81e8261428 --- /dev/null +++ b/tools/deployment/package-helm/templates/mcp-server-service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.clpConfig.mcp_server }} +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-mcp-server + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" +spec: + type: "NodePort" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "mcp-server" + ports: + - port: 8000 + targetPort: "http" + nodePort: {{ .Values.clpConfig.mcp_server.port }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 97f1d133f9..f9620d4d8c 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -1,19 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-query-scheduler - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "query-scheduler" -spec: - clusterIP: "None" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "query-scheduler" - ports: - - port: 7000 - targetPort: "query-scheduler" ---- apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -106,21 +90,3 @@ spec: - name: "query-scheduler-logs" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-query-scheduler-logs ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-query-scheduler-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "query-scheduler" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "query-scheduler" - resources: - requests: - storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml new file mode 100644 index 0000000000..ff15477c5e --- /dev/null +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "query-scheduler") }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-pvc.yaml b/tools/deployment/package-helm/templates/query-scheduler-pvc.yaml new file mode 100644 index 0000000000..7b1c09aba6 --- /dev/null +++ b/tools/deployment/package-helm/templates/query-scheduler-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-query-scheduler-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "query-scheduler" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/query-scheduler-service.yaml b/tools/deployment/package-helm/templates/query-scheduler-service.yaml new file mode 100644 index 0000000000..9f2ffa7172 --- /dev/null +++ b/tools/deployment/package-helm/templates/query-scheduler-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-query-scheduler + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "query-scheduler" + ports: + - port: 7000 + targetPort: "query-scheduler" diff --git a/tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml new file mode 100644 index 0000000000..20b4dd1c75 --- /dev/null +++ b/tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-staged-streams + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "staged-streams" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "staged-streams" + resources: + requests: + storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 1de4df841d..ede349b8ed 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -96,39 +96,3 @@ spec: - name: "streams" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-streams ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-query-worker-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "query-worker" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "query-worker" - resources: - requests: - storage: "5Gi" ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-staged-streams - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "staged-streams" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "staged-streams" - resources: - requests: - storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml new file mode 100644 index 0000000000..9faec0d79d --- /dev/null +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "query-worker") }} diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml new file mode 100644 index 0000000000..b3948b64aa --- /dev/null +++ b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-query-worker-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "query-worker" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "query-worker" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml new file mode 100644 index 0000000000..20b4dd1c75 --- /dev/null +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-staged-streams + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "staged-streams" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "staged-streams" + resources: + requests: + storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml new file mode 100644 index 0000000000..57a08a0e5b --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "queue") }} diff --git a/tools/deployment/package-helm/templates/queue-service.yaml b/tools/deployment/package-helm/templates/queue-service.yaml new file mode 100644 index 0000000000..2971725fdb --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-queue + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "queue" + ports: + - port: 5672 + targetPort: "amqp" diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index 93c0733b27..7c7245a5e7 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -1,19 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-queue - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "queue" -spec: - clusterIP: "None" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "queue" - ports: - - port: 5672 - targetPort: "amqp" ---- apiVersion: "apps/v1" kind: "StatefulSet" metadata: diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml new file mode 100644 index 0000000000..5dcc2821de --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "redis") }} diff --git a/tools/deployment/package-helm/templates/redis-pv.yaml b/tools/deployment/package-helm/templates/redis-pv.yaml new file mode 100644 index 0000000000..418696c290 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalDataPv" (list . "redis") }} diff --git a/tools/deployment/package-helm/templates/redis-service.yaml b/tools/deployment/package-helm/templates/redis-service.yaml new file mode 100644 index 0000000000..5cd8e61386 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-redis + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "redis" + ports: + - port: 6379 + targetPort: "redis" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 5db313885d..c9ef5b8223 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -1,19 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-redis - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "redis" -spec: - clusterIP: "None" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "redis" - ports: - - port: 6379 - targetPort: "redis" ---- apiVersion: "apps/v1" kind: "StatefulSet" metadata: diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index db4a7af1c3..edd1693849 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -1,22 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-reducer - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "reducer" -spec: - clusterIP: "None" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "reducer" - ports: - {{- range $i := until (int .Values.workerConcurrency) }} - - name: "reducer-{{ $i }}" - port: {{ add $.Values.clpConfig.reducer.base_port $i }} - targetPort: {{ add $.Values.clpConfig.reducer.base_port $i }} - {{- end }} ---- apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -90,21 +71,3 @@ spec: - name: "reducer-logs" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-reducer-logs ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-reducer-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "reducer" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "reducer" - resources: - requests: - storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml new file mode 100644 index 0000000000..e94ed53016 --- /dev/null +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "reducer") }} diff --git a/tools/deployment/package-helm/templates/reducer-pvc.yaml b/tools/deployment/package-helm/templates/reducer-pvc.yaml new file mode 100644 index 0000000000..e4502864c7 --- /dev/null +++ b/tools/deployment/package-helm/templates/reducer-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-reducer-logs + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "reducer" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/reducer-service.yaml b/tools/deployment/package-helm/templates/reducer-service.yaml new file mode 100644 index 0000000000..5c3fc6d7b1 --- /dev/null +++ b/tools/deployment/package-helm/templates/reducer-service.yaml @@ -0,0 +1,18 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-reducer + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "reducer" + ports: + {{- range $i := until (int .Values.workerConcurrency) }} + - name: "reducer-{{ $i }}" + port: {{ add $.Values.clpConfig.reducer.base_port $i }} + targetPort: {{ add $.Values.clpConfig.reducer.base_port $i }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml new file mode 100644 index 0000000000..5547f06955 --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalLogsPv" (list . "results-cache") }} diff --git a/tools/deployment/package-helm/templates/results-cache-pv.yaml b/tools/deployment/package-helm/templates/results-cache-pv.yaml new file mode 100644 index 0000000000..bbaacab351 --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createLocalDataPv" (list . "results-cache") }} diff --git a/tools/deployment/package-helm/templates/results-cache-service.yaml b/tools/deployment/package-helm/templates/results-cache-service.yaml new file mode 100644 index 0000000000..2c8e540fd8 --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-service.yaml @@ -0,0 +1,22 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-results-cache + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + {{- if .Values.allowSbinScripts }} + type: "NodePort" + {{- else }} + clusterIP: "None" + {{- end }} + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" + ports: + - port: 27017 + targetPort: "mongodb" + {{- if .Values.allowSbinScripts }} + nodePort: {{ .Values.clpConfig.results_cache.port }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index c4453cd9d0..818093fde7 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -1,26 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-results-cache - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "results-cache" -spec: - {{- if .Values.allowSbinScripts }} - type: "NodePort" - {{- else }} - clusterIP: "None" - {{- end }} - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "results-cache" - ports: - - port: 27017 - targetPort: "mongodb" - {{- if .Values.allowSbinScripts }} - nodePort: {{ .Values.clpConfig.results_cache.port }} - {{- end }} ---- apiVersion: "apps/v1" kind: "StatefulSet" metadata: diff --git a/tools/deployment/package-helm/templates/staged-archives-pv.yaml b/tools/deployment/package-helm/templates/staged-archives-pv.yaml new file mode 100644 index 0000000000..28b5053f43 --- /dev/null +++ b/tools/deployment/package-helm/templates/staged-archives-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createStagedArchivesPv" . }} diff --git a/tools/deployment/package-helm/templates/staged-streams-pv.yaml b/tools/deployment/package-helm/templates/staged-streams-pv.yaml new file mode 100644 index 0000000000..ae7e52821b --- /dev/null +++ b/tools/deployment/package-helm/templates/staged-streams-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createStagedStreamsPv" . }} diff --git a/tools/deployment/package-helm/templates/streams-pv.yaml b/tools/deployment/package-helm/templates/streams-pv.yaml new file mode 100644 index 0000000000..7f773161a0 --- /dev/null +++ b/tools/deployment/package-helm/templates/streams-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createStreamsPv" . }} diff --git a/tools/deployment/package-helm/templates/tmp-pv.yaml b/tools/deployment/package-helm/templates/tmp-pv.yaml new file mode 100644 index 0000000000..1864277a59 --- /dev/null +++ b/tools/deployment/package-helm/templates/tmp-pv.yaml @@ -0,0 +1 @@ +{{- include "clp.createTmpPv" . }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 873e56c8f7..23c2b57320 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -1,20 +1,3 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-webui - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "webui" -spec: - type: "NodePort" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "webui" - ports: - - port: 4000 - targetPort: "http" - nodePort: {{ .Values.clpConfig.webui.port }} ---- apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -113,21 +96,3 @@ spec: - name: "streams" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-streams ---- -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-streams - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "streams" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "streams" - resources: - requests: - storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/webui-pvc.yaml b/tools/deployment/package-helm/templates/webui-pvc.yaml new file mode 100644 index 0000000000..dfde2e91e1 --- /dev/null +++ b/tools/deployment/package-helm/templates/webui-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-streams + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "streams" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "streams" + resources: + requests: + storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/webui-service.yaml b/tools/deployment/package-helm/templates/webui-service.yaml new file mode 100644 index 0000000000..ebe1ea2074 --- /dev/null +++ b/tools/deployment/package-helm/templates/webui-service.yaml @@ -0,0 +1,16 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-webui + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "webui" +spec: + type: "NodePort" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "webui" + ports: + - port: 4000 + targetPort: "http" + nodePort: {{ .Values.clpConfig.webui.port }} From 0b8ef72b8e23d1389d2b1dd60700993abf7eaa3c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 00:46:22 -0500 Subject: [PATCH 018/149] exclude helm directory from yaml lint --- taskfiles/lint.yaml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index ecfb24279f..a82ca09351 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -98,16 +98,17 @@ tasks: - |- . "{{.G_LINT_VENV_DIR}}/bin/activate" yamllint --strict \ - .github \ - .yamllint.yml \ - components/core/.clang-format \ - components/core/config \ - components/core/tools/ \ - components/package-template/src/etc \ - docs \ - taskfile.yaml \ - taskfiles \ - tools/deployment + ".github" \ + ".yamllint.yml" \ + "components/core/.clang-format" \ + "components/core/config" \ + "components/core/tools/" \ + "components/package-template/src/etc" \ + "docs" \ + "taskfile.yaml" \ + "taskfiles" \ + "tools/deployment/package" \ + "tools/deployment/presto-clp" check-cpp-format: sources: &cpp_source_files From 9b5b53b70ae34462c232ed4ab8efddb446997124 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 03:01:06 -0500 Subject: [PATCH 019/149] fix templates --- .../package-helm/templates/mcp-server-deployment.yaml | 2 ++ tools/deployment/package-helm/templates/mcp-server-pvc.yaml | 1 + tools/deployment/package-helm/templates/mcp-server-service.yaml | 1 + 3 files changed, 4 insertions(+) diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 8465ea59ae..436719558f 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -1,3 +1,4 @@ +{{- if .Values.clpConfig.mcp_server }} apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -90,3 +91,4 @@ spec: - name: "mcp-server-logs" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-mcp-server-logs +{{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-pvc.yaml b/tools/deployment/package-helm/templates/mcp-server-pvc.yaml index 98caca2667..4ab1171bc2 100644 --- a/tools/deployment/package-helm/templates/mcp-server-pvc.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-pvc.yaml @@ -1,3 +1,4 @@ +{{- if .Values.clpConfig.mcp_server }} apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: diff --git a/tools/deployment/package-helm/templates/mcp-server-service.yaml b/tools/deployment/package-helm/templates/mcp-server-service.yaml index 81e8261428..ee2ae9fbe9 100644 --- a/tools/deployment/package-helm/templates/mcp-server-service.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-service.yaml @@ -15,3 +15,4 @@ spec: - port: 8000 targetPort: "http" nodePort: {{ .Values.clpConfig.mcp_server.port }} +{{- end }} From c1dccd6c2d1c0302c1dc9c6fe794b4beac0978ef Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 04:08:19 -0500 Subject: [PATCH 020/149] fix(helm): Update maintainer name in Chart.yaml. --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 035b371050..ba2ac6ee3d 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -7,7 +7,7 @@ appVersion: "0.6.1-dev" home: "https://github.com/y-scope/clp" icon: "https://github.com/y-scope/clp/raw/main/docs/src/clp-logo.png" sources: ["https://github.com/y-scope/clp"] -maintainers: [{name: "YScope Inc.", email: "dev@yscope.com"}] +maintainers: [{name: "y-scope", email: "dev@yscope.com"}] keywords: - "analytics" - "compression" From 236c024d55aa704eefd62a354b206d7b6b989fa0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 04:08:32 -0500 Subject: [PATCH 021/149] fix(helm): Set default tag to "nightly" for clpPackage. --- tools/deployment/package-helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 7a69d9aeb3..ffd87b17ab 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -24,7 +24,7 @@ image: clpPackage: repository: "ghcr.io/y-scope/clp/clp-package" pullPolicy: "IfNotPresent" - tag: "" + tag: "nightly" clpConfig: package: From 5aa26cdded5243e45a56462de8deaea25b86fb12 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 05:11:02 -0500 Subject: [PATCH 022/149] Add Helm linting and toolchain support --- .github/workflows/clp-lint.yaml | 4 ++++ chart_schema.yaml | 38 +++++++++++++++++++++++++++++ ct.yaml | 8 +++++++ lint-requirements.txt | 8 ++++++- taskfiles/lint.yaml | 16 +++++++++++++ taskfiles/toolchains.yaml | 42 +++++++++++++++++++++++++++++++++ 6 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 chart_schema.yaml create mode 100644 ct.yaml diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml index 1d97737e74..933ac97325 100644 --- a/.github/workflows/clp-lint.yaml +++ b/.github/workflows/clp-lint.yaml @@ -78,3 +78,7 @@ jobs: - name: "Lint .yaml files" shell: "bash" run: "task lint:check-yaml" + + - name: "Lint Helm charts" + shell: "bash" + run: "task lint:check-helm" diff --git a/chart_schema.yaml b/chart_schema.yaml new file mode 100644 index 0000000000..4f1d69b73b --- /dev/null +++ b/chart_schema.yaml @@ -0,0 +1,38 @@ +# yamllint disable rule:document-start +name: "str()" +home: "str(required=False)" +version: "str()" +apiVersion: "str()" +appVersion: "any(str(), num(), required=False)" +description: "str(required=False)" +keywords: "list(str(), required=False)" +sources: "list(str(), required=False)" +maintainers: "list(include('maintainer'), required=False)" +dependencies: "list(include('dependency'), required=False)" +icon: "str(required=False)" +engine: "str(required=False)" +condition: "str(required=False)" +tags: "str(required=False)" +deprecated: "bool(required=False)" +kubeVersion: "str(required=False)" +annotations: "map(str(), str(), required=False)" +type: "str(required=False)" +--- +maintainer: + name: "str()" + email: "str(required=False)" + url: "str(required=False)" +--- +dependency: + name: "str()" + version: "str()" + repository: "str(required=False)" + condition: "str(required=False)" + tags: "list(str(), required=False)" + enabled: "bool(required=False)" + import-values: "list(any(str(), include('import-value')), required=False)" + alias: "str(required=False)" +--- +import-value: + child: "str()" + parent: "str()" diff --git a/ct.yaml b/ct.yaml new file mode 100644 index 0000000000..159fec6d15 --- /dev/null +++ b/ct.yaml @@ -0,0 +1,8 @@ +chart-dirs: + - "tools/deployment" +validate-maintainers: true +validate-chart-schema: true +check-version-increment: true +validate-yaml: true +lint-conf: ".yamllint.yml" +chart-yaml-schema: "chart_schema.yaml" diff --git a/lint-requirements.txt b/lint-requirements.txt index a5e2ef60d9..2cab04f19a 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,6 +1,12 @@ -black>=24.4.2 +# C++ clang-format>=21.1.2 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 + +# Python +black>=24.4.2 ruff>=0.4.4 + +# YAML yamllint>=1.35.1 +yamale>=6.0.0 \ No newline at end of file diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index a82ca09351..4d64625bcb 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -14,6 +14,7 @@ tasks: check: cmds: - task: "check-cpp-full" + - task: "check-helm" - task: "check-js" - task: "check-py" - task: "check-rust" @@ -32,6 +33,7 @@ tasks: fix: cmds: - task: "fix-cpp-full" + - task: "fix-helm" - task: "fix-js" - task: "fix-py" - task: "fix-rust" @@ -89,6 +91,18 @@ tasks: RUFF_CHECK_FLAGS: "--fix" RUFF_FORMAT_FLAGS: "" + helm: + aliases: + - "check-helm" + - "fix-helm" + deps: + - "toolchains:helm" + - "venv" + cmds: + - |- + . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + ct lint + yaml: aliases: - "check-yaml" @@ -104,6 +118,8 @@ tasks: "components/core/config" \ "components/core/tools/" \ "components/package-template/src/etc" \ + "chart_schema.yaml" \ + "ct.yaml" \ "docs" \ "taskfile.yaml" \ "taskfiles" \ diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 5dfae671ee..84f1694349 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -1,6 +1,48 @@ version: "3" +includes: + utils: "../tools/yscope-dev-utils/exports/taskfiles/utils/remote.yaml" + +vars: + G_HELM_TOOLCHAIN_DIR: "{{.G_BUILD_DIR}}/toolchains/helm" + G_HELM_TOOLCHAIN_ENV_FILE: "{{.G_HELM_TOOLCHAIN_DIR}}/env" + tasks: + helm: + run: "once" + vars: + CHART_TESTING_VERSION: "3.14.0" + cmds: + - "mkdir -p '{{.G_HELM_TOOLCHAIN_DIR}}'" + - task: "utils:download-and-extract-tar" + vars: + OUTPUT_DIR: "{{.G_BUILD_DIR}}/toolchains/helm" + URL: "https://github.com/helm/chart-testing/releases/download/v{{.CHART_TESTING_VERSION}}\ + /chart-testing_{{.CHART_TESTING_VERSION}}_{{OS}}_{{ARCH}}.tar.gz" + FILE_SHA256: >- + {{- if and (eq OS "linux") (eq ARCH "amd64") -}} + d16f0583616885423826241164ce1f6589c6fe5332fa74f374ebd2bd3cb3fe1f + {{- else if and (eq OS "darwin") (eq ARCH "arm64") -}} + db10dbbb42b110c7a5da5a3202908f32ad2ca6ad600d423426dc7886d09aad07 + {{- else -}} + unsupported + {{- end }} + INCLUDE_PATTERNS: ["ct"] + NUM_COMPONENTS_TO_STRIP: "0" + - |- + export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" + export HELM_INSTALL_DIR="{{.G_HELM_TOOLCHAIN_DIR}}" + curl --proto "=https" --tlsv1.2 --silent --show-error --fail \ + https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-4 | bash -s -- --no-sudo + + # Create an environment file to configure the custom Helm toolchain location. + - |- + cat < "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + #!/bin/sh + export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" + EOF + chmod +x "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + rust: run: "once" cmds: From 2ab3e74a9358c8fa94c5257de21e5fe96c66e4f9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 06:12:17 -0500 Subject: [PATCH 023/149] fix(helm): Rename `user` to `username` in query-worker-deployment template. --- .../templates/compression-scheduler-deployment.yaml | 2 +- .../package-helm/templates/compression-worker-deployment.yaml | 2 +- .../package-helm/templates/query-scheduler-deployment.yaml | 2 +- .../package-helm/templates/query-worker-deployment.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 21d210e18b..9365b60327 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -47,7 +47,7 @@ spec: imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_DB_USER" valueFrom: secretKeyRef: diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 35db257d75..6aee42b056 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -41,7 +41,7 @@ spec: imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yml" - name: "CLP_HOME" diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index f9620d4d8c..8728244139 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -49,7 +49,7 @@ spec: imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_DB_USER" valueFrom: secretKeyRef: diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index ede349b8ed..6f913e2a86 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -41,7 +41,7 @@ spec: imagePullPolicy: "Always" env: - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.user }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" + value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yml" - name: "CLP_HOME" From 2ac391be067ed66a529e91479cf5f37cadd512e0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 12:44:56 -0500 Subject: [PATCH 024/149] refactor(helm): Rename PVC and PV files; add tmp-pvc definition. --- ...rker-archives-pvc.yaml => archives-pvc.yaml} | 0 ... compression-worker-staged-archives-pv.yaml} | 0 ...yaml => query-worker-staged-streams-pv.yaml} | 0 .../package-helm/templates/tmp-pvc.yaml | 17 +++++++++++++++++ 4 files changed, 17 insertions(+) rename tools/deployment/package-helm/templates/{compression-worker-archives-pvc.yaml => archives-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{staged-archives-pv.yaml => compression-worker-staged-archives-pv.yaml} (100%) rename tools/deployment/package-helm/templates/{staged-streams-pv.yaml => query-worker-staged-streams-pv.yaml} (100%) create mode 100644 tools/deployment/package-helm/templates/tmp-pvc.yaml diff --git a/tools/deployment/package-helm/templates/compression-worker-archives-pvc.yaml b/tools/deployment/package-helm/templates/archives-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/compression-worker-archives-pvc.yaml rename to tools/deployment/package-helm/templates/archives-pvc.yaml diff --git a/tools/deployment/package-helm/templates/staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml similarity index 100% rename from tools/deployment/package-helm/templates/staged-archives-pv.yaml rename to tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml diff --git a/tools/deployment/package-helm/templates/staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml similarity index 100% rename from tools/deployment/package-helm/templates/staged-streams-pv.yaml rename to tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml diff --git a/tools/deployment/package-helm/templates/tmp-pvc.yaml b/tools/deployment/package-helm/templates/tmp-pvc.yaml new file mode 100644 index 0000000000..c0e4cffad0 --- /dev/null +++ b/tools/deployment/package-helm/templates/tmp-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" . }}-tmp + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "tmp" +spec: + accessModes: ["ReadWriteMany"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "tmp" + resources: + requests: + storage: "10Gi" From f1f3a541792b2c58cb8c38b2d8d09134fe457e61 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 13:12:33 -0500 Subject: [PATCH 025/149] refactor(helm): Remove localPathBase; redefine directory paths for storage and logs. --- .../package-helm/templates/_helpers.tpl | 28 +++++-------------- tools/deployment/package-helm/values.yaml | 14 ++++------ 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 3c6815ab24..0db2fb59b7 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -78,20 +78,6 @@ timeoutSeconds: 5 failureThreshold: 3 {{- end }} -{{/* -CLP logs directory path on host -*/}} -{{- define "clp.logsDirHost" -}} -{{ .Values.storage.localPathBase }}/{{ .Values.clpConfig.logs_directory }} -{{- end }} - -{{/* -CLP data directory path on host -*/}} -{{- define "clp.dataDirHost" -}} -{{ .Values.storage.localPathBase }}/{{ .Values.clpConfig.data_directory }} -{{- end }} - {{/* Creates a local PersistentVolume for data storage. */}} @@ -112,7 +98,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp.dataDirHost" $root }}/{{ $component }} + path: {{ $root.Values.clpConfig.data_directory }}/{{ $component }} nodeAffinity: required: nodeSelectorTerms: @@ -141,7 +127,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp.logsDirHost" $root }}/{{ $component }} + path: {{ $root.Values.clpConfig.logs_directory }}/{{ $component }} nodeAffinity: required: nodeSelectorTerms: @@ -169,7 +155,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp.dataDirHost" $root }}/streams + path: {{ $root.Values.clpConfig.stream_output.storage.directory }} nodeAffinity: required: nodeSelectorTerms: @@ -197,7 +183,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ .Values.storage.localPathBase }}/var/tmp + path: {{ .Values.clpConfig.tmp_directory }} nodeAffinity: required: nodeSelectorTerms: @@ -225,7 +211,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp.dataDirHost" $root }}/archives + path: {{ $root.Values.clpConfig.archive_output.storage.directory }} nodeAffinity: required: nodeSelectorTerms: @@ -253,7 +239,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp.dataDirHost" $root }}/staged-archives + path: "/tmp/clp/staged-archives" nodeAffinity: required: nodeSelectorTerms: @@ -281,7 +267,7 @@ spec: persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ include "clp.dataDirHost" $root }}/staged-streams + path: "/tmp/clp/staged-streams" nodeAffinity: required: nodeSelectorTerms: diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index ffd87b17ab..0b414da61d 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -4,10 +4,6 @@ fullnameOverride: "" # Whether to allow sbin script accesses to services on the node host. allowSbinScripts: true -storage: - # Base directory on host when using local PVs. - localPathBase: "/tmp/clp" - securityContext: # UID/GID for first-party CLP services firstParty: @@ -105,7 +101,7 @@ clpConfig: type: "fs" # NOTE: This directory must not overlap with any path used in CLP's execution container. An # error will be raised if so. - directory: "var/data/archives" + directory: "/tmp/clp/var/data/archives" # Retention period for archives, in minutes. Set to null to disable automatic deletion. retention_period: null @@ -133,7 +129,7 @@ clpConfig: type: "fs" # NOTE: This directory must not overlap with any path used in CLP's execution container. An # error will be raised if so. - directory: "var/data/streams" + directory: "/tmp/clp/var/data/streams" # How large each stream file should be before being split into a new stream file target_uncompressed_size: 134217728 # 128 MB @@ -165,18 +161,18 @@ clpConfig: # it doesn't exist. # NOTE: This directory must not overlap with any path used in CLP's execution container. An error # will be raised if so. - data_directory: "var/data" + data_directory: "/tmp/clp/var/data" # Location where logs are stored. It will be created if it doesn't exist. # NOTE: This directory must not overlap with any path used in CLP's execution container. An error # will be raised if so. - logs_directory: "var/log" + logs_directory: "/tmp/clp/var/log" # Location where temporary runtime data are stored. It will be created if # it doesn't exist. # NOTE: This directory must not overlap with any path used in CLP's execution container. An error # will be raised if so. - tmp_directory: "var/tmp" + tmp_directory: "/tmp/clp/var/tmp" # Location of the AWS tools' config files (e.g., `~/.aws`) aws_config_directory: null From 832eb1935e35d96783f8c5bba5d99420f57ccbb4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 18:04:12 -0500 Subject: [PATCH 026/149] refactor(helm): Consolidate PersistentVolume templates into `createLocalPv`. --- .../package-helm/templates/_helpers.tpl | 193 ++---------------- .../templates/api-server-logs-pv.yaml | 9 +- .../package-helm/templates/archives-pv.yaml | 9 +- .../compression-scheduler-logs-pv.yaml | 9 +- .../templates/compression-worker-logs-pv.yaml | 9 +- ...compression-worker-staged-archives-pv.yaml | 9 +- .../templates/database-logs-pv.yaml | 9 +- .../package-helm/templates/database-pv.yaml | 9 +- .../templates/garbage-collector-logs-pv.yaml | 9 +- .../templates/mcp-server-logs-pv.yaml | 11 +- .../templates/query-scheduler-logs-pv.yaml | 9 +- .../templates/query-worker-logs-pv.yaml | 9 +- .../query-worker-staged-streams-pv.yaml | 9 +- .../package-helm/templates/queue-logs-pv.yaml | 9 +- .../package-helm/templates/redis-logs-pv.yaml | 9 +- .../package-helm/templates/redis-pv.yaml | 9 +- .../templates/reducer-logs-pv.yaml | 9 +- .../templates/results-cache-logs-pv.yaml | 9 +- .../templates/results-cache-pv.yaml | 9 +- .../package-helm/templates/streams-pv.yaml | 9 +- .../package-helm/templates/tmp-pv.yaml | 9 +- 21 files changed, 175 insertions(+), 200 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 0db2fb59b7..249bcadaba 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -79,26 +79,31 @@ failureThreshold: 3 {{- end }} {{/* -Creates a local PersistentVolume for data storage. +Creates a local PersistentVolume. +Parameters (dict): + root: Root template context + name: PV name (full) + component: Component label + capacity: Storage capacity + accessModes: Access modes (list) + hostPath: Absolute path on host */}} -{{- define "clp.createLocalDataPv" -}} -{{- $root := index . 0 -}} -{{- $component := index . 1 -}} +{{- define "clp.createLocalPv" -}} apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp.fullname" $root }}-data-pv-{{ $component }} + name: {{ .name }} labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: {{ $component | quote }} + {{- include "clp.labels" .root | nindent 4 }} + app.kubernetes.io/component: {{ .component | quote }} spec: capacity: - storage: "20Gi" - accessModes: ["ReadWriteOnce"] + storage: {{ .capacity }} + accessModes: {{ .accessModes }} persistentVolumeReclaimPolicy: "Retain" storageClassName: "local-storage" local: - path: {{ $root.Values.clpConfig.data_directory }}/{{ $component }} + path: {{ .hostPath }} nodeAffinity: required: nodeSelectorTerms: @@ -107,172 +112,4 @@ spec: operator: "Exists" {{- end }} -{{/* -Creates a local PersistentVolume for logs storage. -*/}} -{{- define "clp.createLocalLogsPv" -}} -{{- $root := index . 0 -}} -{{- $component := index . 1 -}} -apiVersion: "v1" -kind: "PersistentVolume" -metadata: - name: {{ include "clp.fullname" $root }}-logs-pv-{{ $component }} - labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: {{ $component | quote }} -spec: - capacity: - storage: "5Gi" - accessModes: ["ReadWriteOnce"] - persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" - local: - path: {{ $root.Values.clpConfig.logs_directory }}/{{ $component }} - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: "Exists" -{{- end }} - -{{/* -Creates a local PersistentVolume for streams storage (ReadWriteMany). -*/}} -{{- define "clp.createStreamsPv" -}} -{{- $root := . -}} -apiVersion: "v1" -kind: "PersistentVolume" -metadata: - name: {{ include "clp.fullname" $root }}-streams-pv - labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: "streams" -spec: - capacity: - storage: "20Gi" - accessModes: ["ReadWriteMany"] - persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" - local: - path: {{ $root.Values.clpConfig.stream_output.storage.directory }} - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: "Exists" -{{- end }} - -{{/* -Creates a local PersistentVolume for tmp storage (ReadWriteMany). -*/}} -{{- define "clp.createTmpPv" -}} -{{- $root := . -}} -apiVersion: "v1" -kind: "PersistentVolume" -metadata: - name: {{ include "clp.fullname" $root }}-tmp-pv - labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: "tmp" -spec: - capacity: - storage: "10Gi" - accessModes: ["ReadWriteMany"] - persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" - local: - path: {{ .Values.clpConfig.tmp_directory }} - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: "Exists" -{{- end }} - -{{/* -Creates a local PersistentVolume for archives storage (ReadWriteMany). -*/}} -{{- define "clp.createArchivesPv" -}} -{{- $root := . -}} -apiVersion: "v1" -kind: "PersistentVolume" -metadata: - name: {{ include "clp.fullname" $root }}-archives-pv - labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: "archives" -spec: - capacity: - storage: "50Gi" - accessModes: ["ReadWriteMany"] - persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" - local: - path: {{ $root.Values.clpConfig.archive_output.storage.directory }} - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: "Exists" -{{- end }} - -{{/* -Creates a local PersistentVolume for staged-archives storage (ReadWriteMany). -*/}} -{{- define "clp.createStagedArchivesPv" -}} -{{- $root := . -}} -apiVersion: "v1" -kind: "PersistentVolume" -metadata: - name: {{ include "clp.fullname" $root }}-staged-archives-pv - labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: "staged-archives" -spec: - capacity: - storage: "20Gi" - accessModes: ["ReadWriteMany"] - persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" - local: - path: "/tmp/clp/staged-archives" - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: "Exists" -{{- end }} - -{{/* -Creates a local PersistentVolume for staged-streams storage (ReadWriteMany). -*/}} -{{- define "clp.createStagedStreamsPv" -}} -{{- $root := . -}} -apiVersion: "v1" -kind: "PersistentVolume" -metadata: - name: {{ include "clp.fullname" $root }}-staged-streams-pv - labels: - {{- include "clp.labels" $root | nindent 4 }} - app.kubernetes.io/component: "staged-streams" -spec: - capacity: - storage: "20Gi" - accessModes: ["ReadWriteMany"] - persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" - local: - path: "/tmp/clp/staged-streams" - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/hostname" - operator: "Exists" -{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index c717de287a..f229201f9f 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "api-server") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-api-server-logs-pv" (include "clp.fullname" .)) + "component" "api-server" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/api-server" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/archives-pv.yaml b/tools/deployment/package-helm/templates/archives-pv.yaml index 84edd505ec..bfd618723e 100644 --- a/tools/deployment/package-helm/templates/archives-pv.yaml +++ b/tools/deployment/package-helm/templates/archives-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createArchivesPv" . }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-archives-pv" (include "clp.fullname" .)) + "component" "archives" + "capacity" "50Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" .Values.clpConfig.archive_output.storage.directory +) }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 05becd1f26..14812f7384 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "compression-scheduler") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-compression-scheduler-logs-pv" (include "clp.fullname" .)) + "component" "compression-scheduler" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/compression-scheduler" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index 737e205c52..a1f5903e4f 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "compression-worker") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-compression-worker-logs-pv" (include "clp.fullname" .)) + "component" "compression-worker" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/compression-worker" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index 28b5053f43..c10c71642c 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createStagedArchivesPv" . }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-staged-archives-pv" (include "clp.fullname" .)) + "component" "staged-archives" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" "/tmp/clp/var/data/staged-archives" +) }} diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml index ada6357ba0..92d8f548f9 100644 --- a/tools/deployment/package-helm/templates/database-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "database") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-database-logs-pv" (include "clp.fullname" .)) + "component" "database" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/database" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/database-pv.yaml b/tools/deployment/package-helm/templates/database-pv.yaml index a72331b6f1..cf8bebe068 100644 --- a/tools/deployment/package-helm/templates/database-pv.yaml +++ b/tools/deployment/package-helm/templates/database-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalDataPv" (list . "database") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-database-data-pv" (include "clp.fullname" .)) + "component" "database" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/database" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index a55e94608f..4b1e709b6f 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "garbage-collector") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-garbage-collector-logs-pv" (include "clp.fullname" .)) + "component" "garbage-collector" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/garbage-collector" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index 48fd58f61f..ee6b204e65 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -1,3 +1,8 @@ -{{- if .Values.clpConfig.mcp_server }} -{{- include "clp.createLocalLogsPv" (list . "mcp-server") }} -{{- end }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-mcp-server-logs-pv" (include "clp.fullname" .)) + "component" "mcp-server" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/mcp-server" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index ff15477c5e..cf9b994d63 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "query-scheduler") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-query-scheduler-logs-pv" (include "clp.fullname" .)) + "component" "query-scheduler" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/query-scheduler" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index 9faec0d79d..fd52f15714 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "query-worker") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-query-worker-logs-pv" (include "clp.fullname" .)) + "component" "query-worker" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/query-worker" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index ae7e52821b..134ec7d14c 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createStagedStreamsPv" . }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-staged-streams-pv" (include "clp.fullname" .)) + "component" "staged-streams" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" "/tmp/clp/var/data/staged-streams" +) }} diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml index 57a08a0e5b..0db0788eaa 100644 --- a/tools/deployment/package-helm/templates/queue-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "queue") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-queue-logs-pv" (include "clp.fullname" .)) + "component" "queue" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/queue" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml index 5dcc2821de..d123c21d11 100644 --- a/tools/deployment/package-helm/templates/redis-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "redis") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-redis-logs-pv" (include "clp.fullname" .)) + "component" "redis" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/redis" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/redis-pv.yaml b/tools/deployment/package-helm/templates/redis-pv.yaml index 418696c290..139cfbed4e 100644 --- a/tools/deployment/package-helm/templates/redis-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalDataPv" (list . "redis") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-redis-data-pv" (include "clp.fullname" .)) + "component" "redis" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/redis" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index e94ed53016..898c88d8c4 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "reducer") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-reducer-logs-pv" (include "clp.fullname" .)) + "component" "reducer" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/reducer" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index 5547f06955..89a8fbf892 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalLogsPv" (list . "results-cache") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-results-cache-logs-pv" (include "clp.fullname" .)) + "component" "results-cache" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/results-cache" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/results-cache-pv.yaml b/tools/deployment/package-helm/templates/results-cache-pv.yaml index bbaacab351..a74fa2cb66 100644 --- a/tools/deployment/package-helm/templates/results-cache-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createLocalDataPv" (list . "results-cache") }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-results-cache-data-pv" (include "clp.fullname" .)) + "component" "results-cache" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/results-cache" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/streams-pv.yaml b/tools/deployment/package-helm/templates/streams-pv.yaml index 7f773161a0..477c23153d 100644 --- a/tools/deployment/package-helm/templates/streams-pv.yaml +++ b/tools/deployment/package-helm/templates/streams-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createStreamsPv" . }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-streams-pv" (include "clp.fullname" .)) + "component" "streams" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" .Values.clpConfig.stream_output.storage.directory +) }} diff --git a/tools/deployment/package-helm/templates/tmp-pv.yaml b/tools/deployment/package-helm/templates/tmp-pv.yaml index 1864277a59..2ddba79f4c 100644 --- a/tools/deployment/package-helm/templates/tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/tmp-pv.yaml @@ -1 +1,8 @@ -{{- include "clp.createTmpPv" . }} +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-tmp-pv" (include "clp.fullname" .)) + "component" "tmp" + "capacity" "10Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" .Values.clpConfig.tmp_directory +) }} From d2b4671abcb5da3d2aec8df5972063583d88753e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 16 Nov 2025 23:58:12 -0500 Subject: [PATCH 027/149] refactor(helm): Rename PVC and PV template files for consistency. --- .../templates/{api-server-pvc.yaml => api-server-logs-pvc.yaml} | 0 ...ion-scheduler-pvc.yaml => compression-scheduler-logs-pvc.yaml} | 0 .../templates/{database-pv.yaml => database-data-pv.yaml} | 0 ...garbage-collector-pvc.yaml => garbage-collector-logs-pvc.yaml} | 0 .../templates/{mcp-server-pvc.yaml => mcp-server-logs-pvc.yaml} | 0 .../{query-scheduler-pvc.yaml => query-scheduler-logs-pvc.yaml} | 0 .../package-helm/templates/{redis-pv.yaml => redis-data-pv.yaml} | 0 .../templates/{reducer-pvc.yaml => reducer-logs-pvc.yaml} | 0 .../{results-cache-pv.yaml => results-cache-data-pv.yaml} | 0 .../package-helm/templates/{webui-pvc.yaml => streams-pvc.yaml} | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename tools/deployment/package-helm/templates/{api-server-pvc.yaml => api-server-logs-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{compression-scheduler-pvc.yaml => compression-scheduler-logs-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{database-pv.yaml => database-data-pv.yaml} (100%) rename tools/deployment/package-helm/templates/{garbage-collector-pvc.yaml => garbage-collector-logs-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{mcp-server-pvc.yaml => mcp-server-logs-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{query-scheduler-pvc.yaml => query-scheduler-logs-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{redis-pv.yaml => redis-data-pv.yaml} (100%) rename tools/deployment/package-helm/templates/{reducer-pvc.yaml => reducer-logs-pvc.yaml} (100%) rename tools/deployment/package-helm/templates/{results-cache-pv.yaml => results-cache-data-pv.yaml} (100%) rename tools/deployment/package-helm/templates/{webui-pvc.yaml => streams-pvc.yaml} (100%) diff --git a/tools/deployment/package-helm/templates/api-server-pvc.yaml b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/api-server-pvc.yaml rename to tools/deployment/package-helm/templates/api-server-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/compression-scheduler-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/compression-scheduler-pvc.yaml rename to tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/database-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml similarity index 100% rename from tools/deployment/package-helm/templates/database-pv.yaml rename to tools/deployment/package-helm/templates/database-data-pv.yaml diff --git a/tools/deployment/package-helm/templates/garbage-collector-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/garbage-collector-pvc.yaml rename to tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/mcp-server-pvc.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/mcp-server-pvc.yaml rename to tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/query-scheduler-pvc.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/query-scheduler-pvc.yaml rename to tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/redis-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml similarity index 100% rename from tools/deployment/package-helm/templates/redis-pv.yaml rename to tools/deployment/package-helm/templates/redis-data-pv.yaml diff --git a/tools/deployment/package-helm/templates/reducer-pvc.yaml b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/reducer-pvc.yaml rename to tools/deployment/package-helm/templates/reducer-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/results-cache-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml similarity index 100% rename from tools/deployment/package-helm/templates/results-cache-pv.yaml rename to tools/deployment/package-helm/templates/results-cache-data-pv.yaml diff --git a/tools/deployment/package-helm/templates/webui-pvc.yaml b/tools/deployment/package-helm/templates/streams-pvc.yaml similarity index 100% rename from tools/deployment/package-helm/templates/webui-pvc.yaml rename to tools/deployment/package-helm/templates/streams-pvc.yaml From 8985355147ea5ed443ce6d5fc620d5373d8c64a9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 00:04:57 -0500 Subject: [PATCH 028/149] refactor(helm): Simplify PV and secret object names for consistency. --- tools/deployment/package-helm/templates/api-server-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/archives-pv.yaml | 2 +- .../package-helm/templates/compression-scheduler-logs-pv.yaml | 2 +- .../package-helm/templates/compression-worker-logs-pv.yaml | 2 +- .../templates/compression-worker-staged-archives-pv.yaml | 2 +- tools/deployment/package-helm/templates/database-data-pv.yaml | 2 +- tools/deployment/package-helm/templates/database-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/database-secret.yaml | 2 +- .../package-helm/templates/garbage-collector-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml | 2 +- .../package-helm/templates/query-scheduler-logs-pv.yaml | 2 +- .../deployment/package-helm/templates/query-worker-logs-pv.yaml | 2 +- .../package-helm/templates/query-worker-staged-streams-pv.yaml | 2 +- tools/deployment/package-helm/templates/queue-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/queue-secret.yaml | 2 +- tools/deployment/package-helm/templates/redis-data-pv.yaml | 2 +- tools/deployment/package-helm/templates/redis-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/redis-secret.yaml | 2 +- tools/deployment/package-helm/templates/reducer-logs-pv.yaml | 2 +- .../package-helm/templates/results-cache-data-pv.yaml | 2 +- .../package-helm/templates/results-cache-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/streams-pv.yaml | 2 +- tools/deployment/package-helm/templates/tmp-pv.yaml | 2 +- 23 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index f229201f9f..b0201e5b7d 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-api-server-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-api-server-logs" (include "clp.fullname" .)) "component" "api-server" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/archives-pv.yaml b/tools/deployment/package-helm/templates/archives-pv.yaml index bfd618723e..11072b5c9d 100644 --- a/tools/deployment/package-helm/templates/archives-pv.yaml +++ b/tools/deployment/package-helm/templates/archives-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-archives-pv" (include "clp.fullname" .)) + "name" (printf "%s-archives" (include "clp.fullname" .)) "component" "archives" "capacity" "50Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 14812f7384..6687600ebe 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-compression-scheduler-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-compression-scheduler-logs" (include "clp.fullname" .)) "component" "compression-scheduler" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index a1f5903e4f..c474d42bda 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-compression-worker-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-compression-worker-logs" (include "clp.fullname" .)) "component" "compression-worker" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index c10c71642c..6a52ad3e9c 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-staged-archives-pv" (include "clp.fullname" .)) + "name" (printf "%s-staged-archives" (include "clp.fullname" .)) "component" "staged-archives" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/database-data-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml index cf8bebe068..df4d58c1c2 100644 --- a/tools/deployment/package-helm/templates/database-data-pv.yaml +++ b/tools/deployment/package-helm/templates/database-data-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-database-data-pv" (include "clp.fullname" .)) + "name" (printf "%s-database-data" (include "clp.fullname" .)) "component" "database" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml index 92d8f548f9..0fdc598ee0 100644 --- a/tools/deployment/package-helm/templates/database-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-database-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-database-logs" (include "clp.fullname" .)) "component" "database" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/database-secret.yaml b/tools/deployment/package-helm/templates/database-secret.yaml index f2814d6877..64235ff4a7 100644 --- a/tools/deployment/package-helm/templates/database-secret.yaml +++ b/tools/deployment/package-helm/templates/database-secret.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" kind: "Secret" metadata: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database labels: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "database" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index 4b1e709b6f..eb3b1e4cc1 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-garbage-collector-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-garbage-collector-logs" (include "clp.fullname" .)) "component" "garbage-collector" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index ee6b204e65..469e292729 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-mcp-server-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-mcp-server-logs" (include "clp.fullname" .)) "component" "mcp-server" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index cf9b994d63..e4bc56060c 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-query-scheduler-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-query-scheduler-logs" (include "clp.fullname" .)) "component" "query-scheduler" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index fd52f15714..c4dc136698 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-query-worker-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-query-worker-logs" (include "clp.fullname" .)) "component" "query-worker" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index 134ec7d14c..993cb3bca1 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-staged-streams-pv" (include "clp.fullname" .)) + "name" (printf "%s-staged-streams" (include "clp.fullname" .)) "component" "staged-streams" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml index 0db0788eaa..b6ff44e134 100644 --- a/tools/deployment/package-helm/templates/queue-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-queue-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-queue-logs" (include "clp.fullname" .)) "component" "queue" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/queue-secret.yaml b/tools/deployment/package-helm/templates/queue-secret.yaml index 8b909ce22a..704e443b64 100644 --- a/tools/deployment/package-helm/templates/queue-secret.yaml +++ b/tools/deployment/package-helm/templates/queue-secret.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" kind: "Secret" metadata: - name: {{ include "clp.fullname" . }}-queue-secret + name: {{ include "clp.fullname" . }}-queue labels: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "queue" diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml index 139cfbed4e..bf170d9f8c 100644 --- a/tools/deployment/package-helm/templates/redis-data-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-redis-data-pv" (include "clp.fullname" .)) + "name" (printf "%s-redis-data" (include "clp.fullname" .)) "component" "redis" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml index d123c21d11..3a1fc890d3 100644 --- a/tools/deployment/package-helm/templates/redis-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-redis-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-redis-logs" (include "clp.fullname" .)) "component" "redis" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/redis-secret.yaml b/tools/deployment/package-helm/templates/redis-secret.yaml index 4923e0ce5d..244ccb5291 100644 --- a/tools/deployment/package-helm/templates/redis-secret.yaml +++ b/tools/deployment/package-helm/templates/redis-secret.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" kind: "Secret" metadata: - name: {{ include "clp.fullname" . }}-redis-secret + name: {{ include "clp.fullname" . }}-redis labels: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "redis" diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index 898c88d8c4..0e560d674f 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-reducer-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-reducer-logs" (include "clp.fullname" .)) "component" "reducer" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml index a74fa2cb66..52502fa827 100644 --- a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-results-cache-data-pv" (include "clp.fullname" .)) + "name" (printf "%s-results-cache-data" (include "clp.fullname" .)) "component" "results-cache" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index 89a8fbf892..61c8efa80b 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-results-cache-logs-pv" (include "clp.fullname" .)) + "name" (printf "%s-results-cache-logs" (include "clp.fullname" .)) "component" "results-cache" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/streams-pv.yaml b/tools/deployment/package-helm/templates/streams-pv.yaml index 477c23153d..f998090001 100644 --- a/tools/deployment/package-helm/templates/streams-pv.yaml +++ b/tools/deployment/package-helm/templates/streams-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-streams-pv" (include "clp.fullname" .)) + "name" (printf "%s-streams" (include "clp.fullname" .)) "component" "streams" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/tmp-pv.yaml b/tools/deployment/package-helm/templates/tmp-pv.yaml index 2ddba79f4c..43bdd2ca8c 100644 --- a/tools/deployment/package-helm/templates/tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/tmp-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-tmp-pv" (include "clp.fullname" .)) + "name" (printf "%s-tmp" (include "clp.fullname" .)) "component" "tmp" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") From 75c0101b49fa10969df79c5dd509e26086f004da Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 01:01:57 -0500 Subject: [PATCH 029/149] refactor(helm): Consolidate PVC definitions into `createPvc` and streamline component naming. --- .../package-helm/templates/_helpers.tpl | 28 +++++++++++++++++++ .../templates/api-server-deployment.yaml | 4 +-- .../templates/api-server-logs-pv.yaml | 2 +- .../templates/api-server-logs-pvc.yaml | 23 ++++----------- .../package-helm/templates/archives-pvc.yaml | 23 ++++----------- .../compression-scheduler-deployment.yaml | 4 +-- .../compression-scheduler-logs-pv.yaml | 2 +- .../compression-scheduler-logs-pvc.yaml | 23 ++++----------- .../templates/compression-worker-logs-pv.yaml | 2 +- .../compression-worker-logs-pvc.yaml | 23 ++++----------- ...ompression-worker-staged-archives-pvc.yaml | 23 ++++----------- .../templates/database-statefulset.yaml | 6 ++-- .../templates/db-table-creator-job.yaml | 4 +-- .../garbage-collector-deployment.yaml | 4 +-- .../templates/garbage-collector-logs-pv.yaml | 2 +- .../templates/garbage-collector-logs-pvc.yaml | 23 ++++----------- .../templates/mcp-server-deployment.yaml | 4 +-- .../templates/mcp-server-logs-pv.yaml | 2 +- .../templates/mcp-server-logs-pvc.yaml | 23 ++++----------- .../templates/query-scheduler-deployment.yaml | 4 +-- .../templates/query-scheduler-logs-pv.yaml | 2 +- .../templates/query-scheduler-logs-pvc.yaml | 23 ++++----------- .../templates/query-worker-archives-pvc.yaml | 17 ----------- .../templates/query-worker-logs-pv.yaml | 2 +- .../templates/query-worker-logs-pvc.yaml | 23 ++++----------- .../query-worker-staged-streams-pvc.yaml | 23 ++++----------- .../templates/queue-statefulset.yaml | 4 +-- .../templates/redis-statefulset.yaml | 2 +- .../templates/reducer-logs-pv.yaml | 2 +- .../templates/reducer-logs-pvc.yaml | 23 ++++----------- .../package-helm/templates/streams-pvc.yaml | 23 ++++----------- .../package-helm/templates/tmp-pvc.yaml | 23 ++++----------- .../templates/webui-deployment.yaml | 4 +-- 33 files changed, 134 insertions(+), 266 deletions(-) delete mode 100644 tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 249bcadaba..87fd33bc2d 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -112,4 +112,32 @@ spec: operator: "Exists" {{- end }} +{{/* +Creates a PersistentVolumeClaim. +Parameters (dict): + root: Root template context + component: Component label (used in name, labels, and selectors) + capacity: Storage capacity + accessModes: Access modes (list) +*/}} +{{- define "clp.createPvc" -}} +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" .root }}-{{ .component }} + labels: + {{- include "clp.labels" .root | nindent 4 }} + app.kubernetes.io/component: {{ .component | quote }} +spec: + accessModes: {{ .accessModes }} + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" .root | nindent 6 }} + app.kubernetes.io/component: {{ .component | quote }} + resources: + requests: + storage: {{ .capacity }} +{{- end }} + diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index a55769e02c..8bcf8f55f0 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -43,12 +43,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "CLP_LOGS_DIR" value: "/var/log/api_server" diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index b0201e5b7d..977be57d6f 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-api-server-logs" (include "clp.fullname" .)) - "component" "api-server" + "component" "api-server-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/api-server" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml index 53db492b5a..72d6f4702c 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-api-server-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "api-server" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "api-server" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "api-server-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/archives-pvc.yaml b/tools/deployment/package-helm/templates/archives-pvc.yaml index 2534d4ca78..c5b1bac44f 100644 --- a/tools/deployment/package-helm/templates/archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/archives-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-archives - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "archives" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "archives" - resources: - requests: - storage: "50Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "archives" + "capacity" "50Gi" + "accessModes" (list "ReadWriteMany") +) }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 9365b60327..510332695e 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -51,12 +51,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 6687600ebe..475a1ecc6b 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-compression-scheduler-logs" (include "clp.fullname" .)) - "component" "compression-scheduler" + "component" "compression-scheduler-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression-scheduler" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml index 9e5a5170d9..63011569c5 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-compression-scheduler-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "compression-scheduler" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "compression-scheduler" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "compression-scheduler-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index c474d42bda..7d8aeae9ba 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-compression-worker-logs" (include "clp.fullname" .)) - "component" "compression-worker" + "component" "compression-worker-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression-worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml index 617290ea74..45b11b1fad 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-compression-worker-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "compression-worker" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "compression-worker" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "compression-worker-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml index 541516ed3b..77410397a8 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-staged-archives - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "staged-archives" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "staged-archives" - resources: - requests: - storage: "20Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "staged-archives" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") +) }} diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 911adbd117..a193f83a96 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -32,17 +32,17 @@ spec: - name: "MYSQL_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "MYSQL_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "MYSQL_ROOT_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" ports: - name: "database" diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index f8f89a260a..4178adff85 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -32,12 +32,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 2369b9538a..e47bc8b8bd 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -43,12 +43,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "CLP_HOME" value: "/opt/clp" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index eb3b1e4cc1..8eaa2c9ea2 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-garbage-collector-logs" (include "clp.fullname" .)) - "component" "garbage-collector" + "component" "garbage-collector-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/garbage-collector" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml index ec16d99ae7..714e35bd94 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-garbage-collector-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "garbage-collector" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "garbage-collector" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "garbage-collector-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 436719558f..e651e7fb63 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -44,12 +44,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.mcp_server.logging_level | quote }} diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index 469e292729..da16ab4e3f 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-mcp-server-logs" (include "clp.fullname" .)) - "component" "mcp-server" + "component" "mcp-server-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/mcp-server" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml index 4ab1171bc2..f648fa9ccd 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml @@ -1,19 +1,8 @@ {{- if .Values.clpConfig.mcp_server }} -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-mcp-server-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "mcp-server" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "mcp-server" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "mcp-server-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} {{- end }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 8728244139..46a6299234 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -53,12 +53,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index e4bc56060c..8ec5ed13d4 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-query-scheduler-logs" (include "clp.fullname" .)) - "component" "query-scheduler" + "component" "query-scheduler-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query-scheduler" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml index 7b1c09aba6..bf9794ada9 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-query-scheduler-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "query-scheduler" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "query-scheduler" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "query-scheduler-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml deleted file mode 100644 index 20b4dd1c75..0000000000 --- a/tools/deployment/package-helm/templates/query-worker-archives-pvc.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-staged-streams - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "staged-streams" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "staged-streams" - resources: - requests: - storage: "20Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index c4dc136698..007505daae 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-query-worker-logs" (include "clp.fullname" .)) - "component" "query-worker" + "component" "query-worker-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query-worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml index b3948b64aa..bfbc07a9d1 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-query-worker-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "query-worker" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "query-worker" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "query-worker-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml index 20b4dd1c75..195d94319e 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-staged-streams - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "staged-streams" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "staged-streams" - resources: - requests: - storage: "20Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "staged-streams" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") +) }} diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index 7c7245a5e7..406e423e34 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -30,12 +30,12 @@ spec: - name: "RABBITMQ_DEFAULT_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-queue-secret + name: {{ include "clp.fullname" . }}-queue key: "username" - name: "RABBITMQ_DEFAULT_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-queue-secret + name: {{ include "clp.fullname" . }}-queue key: "password" - name: "RABBITMQ_LOGS" value: "/var/log/rabbitmq/rabbitmq.log" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index c9ef5b8223..b938f9bd0e 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -34,7 +34,7 @@ spec: - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-redis-secret + name: {{ include "clp.fullname" . }}-redis key: "password" ports: - name: "redis" diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index 0e560d674f..3fd67c21b5 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . "name" (printf "%s-reducer-logs" (include "clp.fullname" .)) - "component" "reducer" + "component" "reducer-logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/reducer" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml index e4502864c7..cd4eec1792 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-reducer-logs - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "reducer" -spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "reducer" - resources: - requests: - storage: "5Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "reducer-logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} diff --git a/tools/deployment/package-helm/templates/streams-pvc.yaml b/tools/deployment/package-helm/templates/streams-pvc.yaml index dfde2e91e1..bd754e6c42 100644 --- a/tools/deployment/package-helm/templates/streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/streams-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-streams - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "streams" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "streams" - resources: - requests: - storage: "20Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "streams" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") +) }} diff --git a/tools/deployment/package-helm/templates/tmp-pvc.yaml b/tools/deployment/package-helm/templates/tmp-pvc.yaml index c0e4cffad0..abdddcc887 100644 --- a/tools/deployment/package-helm/templates/tmp-pvc.yaml +++ b/tools/deployment/package-helm/templates/tmp-pvc.yaml @@ -1,17 +1,6 @@ -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" . }}-tmp - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "tmp" -spec: - accessModes: ["ReadWriteMany"] - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "tmp" - resources: - requests: - storage: "10Gi" +{{- include "clp.createPvc" (dict + "root" . + "component" "tmp" + "capacity" "10Gi" + "accessModes" (list "ReadWriteMany") +) }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 23c2b57320..6ece053202 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -43,12 +43,12 @@ spec: - name: "CLP_DB_USER" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "username" - name: "CLP_DB_PASS" valueFrom: secretKeyRef: - name: {{ include "clp.fullname" . }}-database-secret + name: {{ include "clp.fullname" . }}-database key: "password" - name: "HOST" value: "0.0.0.0" From 1476b718332e43624afde031b50af937c6dddda0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 02:18:51 -0500 Subject: [PATCH 030/149] Add user-logs PVC and PV definitions. --- .../templates/compression-scheduler-deployment.yaml | 5 +++++ tools/deployment/package-helm/templates/user-logs-pv.yaml | 8 ++++++++ .../deployment/package-helm/templates/user-logs-pvc.yaml | 6 ++++++ tools/deployment/package-helm/test.sh | 1 + 4 files changed, 20 insertions(+) create mode 100644 tools/deployment/package-helm/templates/user-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/user-logs-pvc.yaml diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 510332695e..a0af4204ca 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -73,6 +73,8 @@ spec: readOnly: true - name: "compression-scheduler-logs" mountPath: "/var/log/compression_scheduler" + - name: "user-logs" + mountPath: "/var/log/user" - name: "logs-input" mountPath: "/mnt/logs" readOnly: true @@ -88,6 +90,9 @@ spec: - name: "compression-scheduler-logs" persistentVolumeClaim: claimName: {{ include "clp.fullname" . }}-compression-scheduler-logs + - name: "user-logs" + persistentVolumeClaim: + claimName: {{ include "clp.fullname" . }}-user-logs - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" diff --git a/tools/deployment/package-helm/templates/user-logs-pv.yaml b/tools/deployment/package-helm/templates/user-logs-pv.yaml new file mode 100644 index 0000000000..b90b351a48 --- /dev/null +++ b/tools/deployment/package-helm/templates/user-logs-pv.yaml @@ -0,0 +1,8 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-user-logs" (include "clp.fullname" .)) + "component" "user-logs" + "capacity" "10Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" (printf "%s/user" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/user-logs-pvc.yaml b/tools/deployment/package-helm/templates/user-logs-pvc.yaml new file mode 100644 index 0000000000..2fb0688dc2 --- /dev/null +++ b/tools/deployment/package-helm/templates/user-logs-pvc.yaml @@ -0,0 +1,6 @@ +{{- include "clp.createPvc" (dict + "root" . + "component" "user-logs" + "capacity" "10Gi" + "accessModes" (list "ReadWriteMany") +) }} diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 33fbc61651..73927a489a 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -6,6 +6,7 @@ kind delete cluster --name clp-test sudo rm -rf /tmp/clp mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker,reducer,garbage-collector,api-server,mcp-server} mkdir -p /tmp/clp/var/data/{archives,staged-archives,staged-streams,streams} +mkdir -p /tmp/clp/var/log/user mkdir -p /tmp/clp/var/tmp mkdir -p /tmp/clp/samples From ad82e2565587073c90d73acae2025780e8b56957 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:04:31 -0500 Subject: [PATCH 031/149] fix(helm): Remove unnecessary image tag override in test script. --- tools/deployment/package-helm/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 73927a489a..d1b85b6dcc 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -40,5 +40,5 @@ cp /home/junhao/samples/postgresql.jsonl /tmp/clp/postgresql.jsonl helm uninstall test || true sleep 2 -helm install test . --set image.clpPackage.tag=nightly +helm install test . ls -l /tmp/clp/var/data/ From 29fe3c0126ec04139c4d33899f297ba5eefa3908 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:04:37 -0500 Subject: [PATCH 032/149] feat(helm): Add conditional database image selection based on configuration. --- .../package-helm/templates/database-statefulset.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index a193f83a96..65f9a21904 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -24,7 +24,12 @@ spec: fsGroup: {{ .Values.securityContext.thirdParty.gid }} containers: - name: "database" - image: "mariadb:10-jammy" + image: >- + {{- if eq .Values.clpConfig.database.type "mysql" }} + mysql:8.0.23 + {{- else }} + mariadb:10-jammy + {{- end }} imagePullPolicy: "Always" env: - name: "MYSQL_DATABASE" From c5bb582a6bb4d4d5023b1f0ed8d386f7ab4d9400 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:13:42 -0500 Subject: [PATCH 033/149] feat(helm): Add persistentVolumeClaimRetentionPolicy to statefulsets. --- .../deployment/package-helm/templates/database-statefulset.yaml | 2 ++ tools/deployment/package-helm/templates/queue-statefulset.yaml | 2 ++ tools/deployment/package-helm/templates/redis-statefulset.yaml | 2 ++ .../package-helm/templates/results-cache-statefulset.yaml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 65f9a21904..b1365ab412 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -77,6 +77,8 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" volumeClaimTemplates: - metadata: name: "database-data" diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index 406e423e34..bea559736a 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -53,6 +53,8 @@ spec: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: <<: *queue-health-check + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" volumeClaimTemplates: - metadata: name: "queue-logs" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index b938f9bd0e..245601fb68 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -68,6 +68,8 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" volumeClaimTemplates: - metadata: name: "redis-data" diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 818093fde7..441603be4e 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -60,6 +60,8 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" volumeClaimTemplates: - metadata: name: "results-cache-data" From db9cef95d80b01ced073a83a49f2845d0abf2c86 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:13:52 -0500 Subject: [PATCH 034/149] feat(helm): Add ttlSecondsAfterFinished to job templates. --- .../deployment/package-helm/templates/db-table-creator-job.yaml | 1 + .../templates/results-cache-indices-creator-job.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index 4178adff85..49f242a6e9 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -6,6 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "db-table-creator" spec: + ttlSecondsAfterFinished: 300 template: metadata: labels: diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index 4b45368f3f..5ff701fa3a 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -6,6 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "results-cache-indices-creator" spec: + ttlSecondsAfterFinished: 300 template: metadata: labels: From 1503c494bda9a53779424ca82344e018dd665e40 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:37:31 -0500 Subject: [PATCH 035/149] feat(deployment): set up initial Helm chart for CLP package: - Add database StatefulSet and db-table-creator Job. - Introduce `lint:helm` task to lint the chart using chart-testing (ct). - Update CI workflow to include Helm chart linting. --- .github/workflows/clp-lint.yaml | 4 + chart_schema.yaml | 38 +++++ .../clp_package_utils/general.py | 4 +- .../clp-py-utils/clp_py_utils/clp_config.py | 4 +- .../src/etc/credentials.template.yml | 4 +- ct.yaml | 8 + taskfiles/lint.yaml | 37 +++-- taskfiles/toolchains.yaml | 38 +++++ tools/deployment/package-helm/.helmignore | 23 +++ tools/deployment/package-helm/Chart.yaml | 17 +++ .../package-helm/templates/_helpers.tpl | 141 ++++++++++++++++++ .../package-helm/templates/configmap.yaml | 33 ++++ .../templates/database-data-pv.yaml | 8 + .../templates/database-logs-pv.yaml | 8 + .../templates/database-secret.yaml | 11 ++ .../templates/database-service.yaml | 22 +++ .../templates/database-statefulset.yaml | 106 +++++++++++++ .../templates/db-table-creator-job.yaml | 59 ++++++++ tools/deployment/package-helm/test.sh | 26 ++++ tools/deployment/package-helm/values.yaml | 47 ++++++ tools/deployment/presto-clp/scripts/init.py | 6 +- 21 files changed, 625 insertions(+), 19 deletions(-) create mode 100644 chart_schema.yaml create mode 100644 ct.yaml create mode 100644 tools/deployment/package-helm/.helmignore create mode 100644 tools/deployment/package-helm/Chart.yaml create mode 100644 tools/deployment/package-helm/templates/_helpers.tpl create mode 100644 tools/deployment/package-helm/templates/configmap.yaml create mode 100644 tools/deployment/package-helm/templates/database-data-pv.yaml create mode 100644 tools/deployment/package-helm/templates/database-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/database-secret.yaml create mode 100644 tools/deployment/package-helm/templates/database-service.yaml create mode 100644 tools/deployment/package-helm/templates/database-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/db-table-creator-job.yaml create mode 100755 tools/deployment/package-helm/test.sh create mode 100644 tools/deployment/package-helm/values.yaml diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml index 1d97737e74..933ac97325 100644 --- a/.github/workflows/clp-lint.yaml +++ b/.github/workflows/clp-lint.yaml @@ -78,3 +78,7 @@ jobs: - name: "Lint .yaml files" shell: "bash" run: "task lint:check-yaml" + + - name: "Lint Helm charts" + shell: "bash" + run: "task lint:check-helm" diff --git a/chart_schema.yaml b/chart_schema.yaml new file mode 100644 index 0000000000..4f1d69b73b --- /dev/null +++ b/chart_schema.yaml @@ -0,0 +1,38 @@ +# yamllint disable rule:document-start +name: "str()" +home: "str(required=False)" +version: "str()" +apiVersion: "str()" +appVersion: "any(str(), num(), required=False)" +description: "str(required=False)" +keywords: "list(str(), required=False)" +sources: "list(str(), required=False)" +maintainers: "list(include('maintainer'), required=False)" +dependencies: "list(include('dependency'), required=False)" +icon: "str(required=False)" +engine: "str(required=False)" +condition: "str(required=False)" +tags: "str(required=False)" +deprecated: "bool(required=False)" +kubeVersion: "str(required=False)" +annotations: "map(str(), str(), required=False)" +type: "str(required=False)" +--- +maintainer: + name: "str()" + email: "str(required=False)" + url: "str(required=False)" +--- +dependency: + name: "str()" + version: "str()" + repository: "str(required=False)" + condition: "str(required=False)" + tags: "list(str(), required=False)" + enabled: "bool(required=False)" + import-values: "list(any(str(), include('import-value')), required=False)" + alias: "str(required=False)" +--- +import-value: + child: "str()" + parent: "str()" diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a1c5fb5e15..d0ad00034c 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -453,8 +453,8 @@ def load_config_file( def generate_credentials_file(credentials_file_path: pathlib.Path): credentials = { - DB_COMPONENT_NAME: {"user": "clp-user", "password": secrets.token_urlsafe(8)}, - QUEUE_COMPONENT_NAME: {"user": "clp-user", "password": secrets.token_urlsafe(8)}, + DB_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)}, + QUEUE_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)}, REDIS_COMPONENT_NAME: {"password": secrets.token_urlsafe(16)}, } diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index c35a30dc0f..9e791754b0 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -232,7 +232,7 @@ def load_credentials_from_file(self, credentials_file_path: pathlib.Path): if config is None: raise ValueError(f"Credentials file '{credentials_file_path}' is empty.") try: - self.username = get_config_value(config, f"{DB_COMPONENT_NAME}.user") + self.username = get_config_value(config, f"{DB_COMPONENT_NAME}.username") self.password = get_config_value(config, f"{DB_COMPONENT_NAME}.password") except KeyError as ex: raise ValueError( @@ -360,7 +360,7 @@ def load_credentials_from_file(self, credentials_file_path: pathlib.Path): if config is None: raise ValueError(f"Credentials file '{credentials_file_path}' is empty.") try: - self.username = get_config_value(config, f"{QUEUE_COMPONENT_NAME}.user") + self.username = get_config_value(config, f"{QUEUE_COMPONENT_NAME}.username") self.password = get_config_value(config, f"{QUEUE_COMPONENT_NAME}.password") except KeyError as ex: raise ValueError( diff --git a/components/package-template/src/etc/credentials.template.yml b/components/package-template/src/etc/credentials.template.yml index 9a0dd73fa2..eb2f9b73f7 100644 --- a/components/package-template/src/etc/credentials.template.yml +++ b/components/package-template/src/etc/credentials.template.yml @@ -1,11 +1,11 @@ ## Database credentials #database: -# user: "user" +# username: "clp-user" # password: "pass" # ## Queue credentials #queue: -# user: "user" +# username: "clp-user" # password: "pass" # ## Redis credentials diff --git a/ct.yaml b/ct.yaml new file mode 100644 index 0000000000..159fec6d15 --- /dev/null +++ b/ct.yaml @@ -0,0 +1,8 @@ +chart-dirs: + - "tools/deployment" +validate-maintainers: true +validate-chart-schema: true +check-version-increment: true +validate-yaml: true +lint-conf: ".yamllint.yml" +chart-yaml-schema: "chart_schema.yaml" diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 29b6dcd254..74b792b91b 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -14,6 +14,7 @@ tasks: check: cmds: - task: "check-cpp-full" + - task: "check-helm" - task: "check-js" - task: "check-py" - task: "check-rust" @@ -32,6 +33,7 @@ tasks: fix: cmds: - task: "fix-cpp-full" + - task: "fix-helm" - task: "fix-js" - task: "fix-py" - task: "fix-rust" @@ -54,6 +56,18 @@ tasks: - task: "fix-rust" - task: "fix-yaml" + helm: + aliases: + - "check-helm" + - "fix-helm" + deps: + - "toolchains:helm" + - "venv" + cmds: + - |- + . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + ct lint + check-js: sources: &js_source_files - "{{.G_BUILD_DIR}}/webui-node-modules.md5" @@ -98,16 +112,19 @@ tasks: - |- . "{{.G_LINT_VENV_DIR}}/bin/activate" yamllint --strict \ - .github \ - .yamllint.yml \ - components/core/.clang-format \ - components/core/config \ - components/core/tools/ \ - components/package-template/src/etc \ - docs \ - taskfile.yaml \ - taskfiles \ - tools/deployment + ".github" \ + ".yamllint.yml" \ + "components/core/.clang-format" \ + "components/core/config" \ + "components/core/tools/" \ + "components/package-template/src/etc" \ + "chart_schema.yaml" \ + "ct.yaml" \ + "docs" \ + "taskfile.yaml" \ + "taskfiles" \ + "tools/deployment/package" \ + "tools/deployment/presto-clp" check-cpp-format: sources: &cpp_source_files diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 53a6c08f30..438e5ea23f 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -1,9 +1,12 @@ version: "3" includes: + remote: "../tools/yscope-dev-utils/exports/taskfiles/utils/remote.yaml" utils: "../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" vars: + G_HELM_TOOLCHAIN_DIR: "{{.G_BUILD_DIR}}/toolchains/helm" + G_HELM_TOOLCHAIN_ENV_FILE: "{{.G_HELM_TOOLCHAIN_DIR}}/env" G_RUST_TOOLCHAIN_DIR: "{{.G_BUILD_DIR}}/toolchains/rust" G_RUST_TOOLCHAIN_ENV_FILE: "{{.G_RUST_TOOLCHAIN_DIR}}/env" @@ -15,6 +18,41 @@ vars: G_RUST_TOOLCHAIN_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/toolchains-rust.md5" tasks: + helm: + run: "once" + vars: + CHART_TESTING_VERSION: "3.14.0" + cmds: + - "mkdir -p '{{.G_HELM_TOOLCHAIN_DIR}}'" + - task: "remote:download-and-extract-tar" + vars: + OUTPUT_DIR: "{{.G_BUILD_DIR}}/toolchains/helm" + URL: "https://github.com/helm/chart-testing/releases/download/v{{.CHART_TESTING_VERSION}}\ + /chart-testing_{{.CHART_TESTING_VERSION}}_{{OS}}_{{ARCH}}.tar.gz" + FILE_SHA256: >- + {{- if and (eq OS "linux") (eq ARCH "amd64") -}} + d16f0583616885423826241164ce1f6589c6fe5332fa74f374ebd2bd3cb3fe1f + {{- else if and (eq OS "darwin") (eq ARCH "arm64") -}} + db10dbbb42b110c7a5da5a3202908f32ad2ca6ad600d423426dc7886d09aad07 + {{- else -}} + unsupported + {{- end }} + INCLUDE_PATTERNS: ["ct"] + NUM_COMPONENTS_TO_STRIP: "0" + - |- + export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" + export HELM_INSTALL_DIR="{{.G_HELM_TOOLCHAIN_DIR}}" + curl --proto "=https" --tlsv1.2 --silent --show-error --fail \ + https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-4 | bash -s -- --no-sudo + + # Create an environment file to configure the custom Helm toolchain location. + - |- + cat < "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + #!/bin/sh + export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" + EOF + chmod +x "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + rust: # We use a label to uniquely identify this task and its checksum under `.task/checksums/`. label: "toolchains-rust" diff --git a/tools/deployment/package-helm/.helmignore b/tools/deployment/package-helm/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/tools/deployment/package-helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml new file mode 100644 index 0000000000..ba2ac6ee3d --- /dev/null +++ b/tools/deployment/package-helm/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: "v2" +name: "clp" +description: "A Helm chart for CLP (Compressed Log Processor) package deployment" +type: "application" +version: "0.1.0" +appVersion: "0.6.1-dev" +home: "https://github.com/y-scope/clp" +icon: "https://github.com/y-scope/clp/raw/main/docs/src/clp-logo.png" +sources: ["https://github.com/y-scope/clp"] +maintainers: [{name: "y-scope", email: "dev@yscope.com"}] +keywords: + - "analytics" + - "compression" + - "log-management" + - "log-parser" + - "logging" + - "search" diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl new file mode 100644 index 0000000000..658953f248 --- /dev/null +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -0,0 +1,141 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "clp.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "clp.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "clp.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "clp.labels" -}} +helm.sh/chart: {{ include "clp.chart" . }} +{{ include "clp.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "clp.selectorLabels" -}} +app.kubernetes.io/name: {{ include "clp.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Image reference for CLP Package +*/}} +{{- define "clp.image.ref" -}} +{{- $tag := .Values.image.clpPackage.tag | default .Chart.AppVersion }} +{{- printf "%s:%s" .Values.image.clpPackage.repository $tag }} +{{- end }} + +{{/* +Creates timings for readiness probes (faster checks for quicker startup). +*/}} +{{- define "clp.readinessProbeTimings" -}} +initialDelaySeconds: 1 +periodSeconds: 1 +timeoutSeconds: 1 +failureThreshold: 3 +{{- end }} + +{{/* +Creates timings for liveness probes. +*/}} +{{- define "clp.livenessProbeTimings" -}} +initialDelaySeconds: 30 +periodSeconds: 10 +timeoutSeconds: 5 +failureThreshold: 3 +{{- end }} + +{{/* +Creates a local PersistentVolume. +Parameters (dict): + root: Root template context + name: PV name (full) + component: Component label + capacity: Storage capacity + accessModes: Access modes (list) + hostPath: Absolute path on host +*/}} +{{- define "clp.createLocalPv" -}} +apiVersion: "v1" +kind: "PersistentVolume" +metadata: + name: {{ .name }} + labels: + {{- include "clp.labels" .root | nindent 4 }} + app.kubernetes.io/component: {{ .component | quote }} +spec: + capacity: + storage: {{ .capacity }} + accessModes: {{ .accessModes }} + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: {{ .hostPath }} + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: "Exists" +{{- end }} + +{{/* +Creates a PersistentVolumeClaim. +Parameters (dict): + root: Root template context + component: Component label (used in name, labels, and selectors) + capacity: Storage capacity + accessModes: Access modes (list) +*/}} +{{- define "clp.createPvc" -}} +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" .root }}-{{ .component }} + labels: + {{- include "clp.labels" .root | nindent 4 }} + app.kubernetes.io/component: {{ .component | quote }} +spec: + accessModes: {{ .accessModes }} + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" .root | nindent 6 }} + app.kubernetes.io/component: {{ .component | quote }} + resources: + requests: + storage: {{ .capacity }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml new file mode 100644 index 0000000000..ee9fc207ae --- /dev/null +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -0,0 +1,33 @@ +apiVersion: "v1" +kind: "ConfigMap" +metadata: + name: {{ include "clp.fullname" . }}-config + labels: + {{- include "clp.labels" . | nindent 4 }} +data: + clp-config.yml: | + data_directory: "/var/data" + database: + auto_commit: false + compress: true + host: {{ include "clp.fullname" . }}-database + name: {{ .Values.clpConfig.database.name | quote }} + port: 3306 + ssl_cert: null + type: {{ .Values.clpConfig.database.type | quote }} + logs_directory: "/var/log" + package: + query_engine: {{ .Values.clpConfig.package.query_engine | quote }} + storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} + + mysql-logging.cnf: | + [mysqld] + # https://dev.mysql.com/doc/refman/8.0/en/log-destinations.html + # Enable the general log + general_log=1 + # Set the destination file + general_log_file=/var/log/mysql/mysql.log + + # https://dev.mysql.com/doc/refman/8.0/en/error-log-destination-configuration.html + # Send error logs to file + log_error=/var/log/mysql/mysql-error.log diff --git a/tools/deployment/package-helm/templates/database-data-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml new file mode 100644 index 0000000000..df4d58c1c2 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-data-pv.yaml @@ -0,0 +1,8 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-database-data" (include "clp.fullname" .)) + "component" "database" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/database" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml new file mode 100644 index 0000000000..0fdc598ee0 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -0,0 +1,8 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-database-logs" (include "clp.fullname" .)) + "component" "database" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/database" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/database-secret.yaml b/tools/deployment/package-helm/templates/database-secret.yaml new file mode 100644 index 0000000000..64235ff4a7 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp.fullname" . }}-database + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +type: "Opaque" +stringData: + username: {{ .Values.credentials.database.username | quote }} + password: {{ .Values.credentials.database.password | quote }} diff --git a/tools/deployment/package-helm/templates/database-service.yaml b/tools/deployment/package-helm/templates/database-service.yaml new file mode 100644 index 0000000000..fe6f2f823e --- /dev/null +++ b/tools/deployment/package-helm/templates/database-service.yaml @@ -0,0 +1,22 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-database + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +spec: + {{- if .Values.allowSbinScripts }} + type: "NodePort" + {{- else }} + clusterIP: "None" + {{- end }} + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "database" + ports: + - port: 3306 + targetPort: "database" + {{- if .Values.allowSbinScripts }} + nodePort: {{ .Values.clpConfig.database.port }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml new file mode 100644 index 0000000000..b1365ab412 --- /dev/null +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -0,0 +1,106 @@ +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp.fullname" . }}-database + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "database" +spec: + serviceName: {{ include "clp.fullname" . }}-database + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "database" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "database" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "database" + image: >- + {{- if eq .Values.clpConfig.database.type "mysql" }} + mysql:8.0.23 + {{- else }} + mariadb:10-jammy + {{- end }} + imagePullPolicy: "Always" + env: + - name: "MYSQL_DATABASE" + value: {{ .Values.clpConfig.database.name | quote }} + - name: "MYSQL_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "username" + - name: "MYSQL_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "password" + - name: "MYSQL_ROOT_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "password" + ports: + - name: "database" + containerPort: 3306 + volumeMounts: + - name: "database-data" + mountPath: "/var/lib/mysql" + - name: "database-logs" + mountPath: "/var/log/mysql" + - name: "config" + mountPath: "/etc/mysql/conf.d/logging.cnf" + subPath: "mysql-logging.cnf" + readOnly: true + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: &database-health-check + command: [ + "mysqladmin", "ping", + "--silent", + "-h", "127.0.0.1" + ] + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: + <<: *database-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" + volumeClaimTemplates: + - metadata: + name: "database-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "database" + resources: + requests: + storage: "20Gi" + - metadata: + name: "database-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "database" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml new file mode 100644 index 0000000000..49f242a6e9 --- /dev/null +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -0,0 +1,59 @@ +apiVersion: "batch/v1" +kind: "Job" +metadata: + name: {{ include "clp.fullname" . }}-db-table-creator + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "db-table-creator" +spec: + ttlSecondsAfterFinished: 300 + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "db-table-creator" + spec: + restartPolicy: "OnFailure" + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-database" + image: "busybox:latest" + command: [ + "sh", "-c", + "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" + ] + containers: + - name: "db-table-creator" + image: "{{ include "clp.image.ref" . }}" + imagePullPolicy: "Always" + env: + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "password" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yml" + subPath: "clp-config.yml" + readOnly: true + command: [ + "python3", "-u", + "-m", "clp_py_utils.create-db-tables", + "--config", "/etc/clp-config.yml", + "--storage-engine", {{ .Values.clpConfig.package.storage_engine | quote }} + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh new file mode 100755 index 0000000000..73587b1570 --- /dev/null +++ b/tools/deployment/package-helm/test.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# TODO: to be deleted / migrated to integration tests + +kind delete cluster --name clp-test +rm -rf /tmp/clp +mkdir -p /tmp/clp/var/{data,log}/database + +cat < Date: Mon, 17 Nov 2025 03:40:05 -0500 Subject: [PATCH 036/149] chore(deployment): remove pod watch from test script. --- tools/deployment/package-helm/test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 73587b1570..1111d8d985 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -23,4 +23,3 @@ EOF helm uninstall test || true sleep 2 helm install test . -kubectl get pods -w \ No newline at end of file From 0b4c4fd54717028178eec46e7832a2a4b13dd5b1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:49:00 -0500 Subject: [PATCH 037/149] chore(deployment): update chart-testing config to set target branch. --- ct.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ct.yaml b/ct.yaml index 159fec6d15..dd82566d74 100644 --- a/ct.yaml +++ b/ct.yaml @@ -1,8 +1,9 @@ chart-dirs: - "tools/deployment" -validate-maintainers: true -validate-chart-schema: true +chart-yaml-schema: "chart_schema.yaml" check-version-increment: true -validate-yaml: true lint-conf: ".yamllint.yml" -chart-yaml-schema: "chart_schema.yaml" +target-branch: HEAD +validate-chart-schema: true +validate-maintainers: true +validate-yaml: true From f12557de3dd867210891ea5d74de4e1fd167525a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:51:45 -0500 Subject: [PATCH 038/149] quote string --- ct.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ct.yaml b/ct.yaml index dd82566d74..0333e87bc8 100644 --- a/ct.yaml +++ b/ct.yaml @@ -3,7 +3,7 @@ chart-dirs: chart-yaml-schema: "chart_schema.yaml" check-version-increment: true lint-conf: ".yamllint.yml" -target-branch: HEAD +target-branch: "HEAD" validate-chart-schema: true validate-maintainers: true validate-yaml: true From 5713eb31c0b6475aabdcfc351645553dcc56af1e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 04:00:06 -0500 Subject: [PATCH 039/149] fix missing yamale --- lint-requirements.txt | 8 +++++++- taskfiles/lint.yaml | 1 + taskfiles/toolchains.yaml | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index a5e2ef60d9..2cab04f19a 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,6 +1,12 @@ -black>=24.4.2 +# C++ clang-format>=21.1.2 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 + +# Python +black>=24.4.2 ruff>=0.4.4 + +# YAML yamllint>=1.35.1 +yamale>=6.0.0 \ No newline at end of file diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 74b792b91b..8f13303004 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -65,6 +65,7 @@ tasks: - "venv" cmds: - |- + . "{{.G_LINT_VENV_DIR}}/bin/activate" . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" ct lint diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 438e5ea23f..1df385ef74 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -47,7 +47,7 @@ tasks: # Create an environment file to configure the custom Helm toolchain location. - |- - cat < "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + cat <<"EOF" > "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" #!/bin/sh export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" EOF From fa2522eea0c170aa52354ad489ee8506b916786e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 03:51:45 -0500 Subject: [PATCH 040/149] quote string --- ct.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ct.yaml b/ct.yaml index 159fec6d15..0333e87bc8 100644 --- a/ct.yaml +++ b/ct.yaml @@ -1,8 +1,9 @@ chart-dirs: - "tools/deployment" -validate-maintainers: true -validate-chart-schema: true +chart-yaml-schema: "chart_schema.yaml" check-version-increment: true -validate-yaml: true lint-conf: ".yamllint.yml" -chart-yaml-schema: "chart_schema.yaml" +target-branch: "HEAD" +validate-chart-schema: true +validate-maintainers: true +validate-yaml: true From 8d176fecc8c6ba37e15887220fbc4278f0eb03ca Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 04:00:06 -0500 Subject: [PATCH 041/149] fix missing yamale --- taskfiles/lint.yaml | 25 +++++++++++++------------ taskfiles/toolchains.yaml | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 21597d13ec..8f13303004 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -56,6 +56,19 @@ tasks: - task: "fix-rust" - task: "fix-yaml" + helm: + aliases: + - "check-helm" + - "fix-helm" + deps: + - "toolchains:helm" + - "venv" + cmds: + - |- + . "{{.G_LINT_VENV_DIR}}/bin/activate" + . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + ct lint + check-js: sources: &js_source_files - "{{.G_BUILD_DIR}}/webui-node-modules.md5" @@ -91,18 +104,6 @@ tasks: RUFF_CHECK_FLAGS: "--fix" RUFF_FORMAT_FLAGS: "" - helm: - aliases: - - "check-helm" - - "fix-helm" - deps: - - "toolchains:helm" - - "venv" - cmds: - - |- - . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" - ct lint - yaml: aliases: - "check-yaml" diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 438e5ea23f..1df385ef74 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -47,7 +47,7 @@ tasks: # Create an environment file to configure the custom Helm toolchain location. - |- - cat < "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + cat <<"EOF" > "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" #!/bin/sh export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" EOF From e1fe43659fee74eb68e5674a90f3733596b02288 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 04:17:17 -0500 Subject: [PATCH 042/149] chore(ci): Set fetch-depth to 0 in clp-lint workflow. --- .github/workflows/clp-lint.yaml | 1 + ct.yaml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml index 933ac97325..68eb65a617 100644 --- a/.github/workflows/clp-lint.yaml +++ b/.github/workflows/clp-lint.yaml @@ -25,6 +25,7 @@ jobs: steps: - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" with: + fetch-depth: 0 submodules: "recursive" - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" diff --git a/ct.yaml b/ct.yaml index 0333e87bc8..11f80fb3ad 100644 --- a/ct.yaml +++ b/ct.yaml @@ -3,7 +3,6 @@ chart-dirs: chart-yaml-schema: "chart_schema.yaml" check-version-increment: true lint-conf: ".yamllint.yml" -target-branch: "HEAD" validate-chart-schema: true validate-maintainers: true validate-yaml: true From e4c0f2218866cdbba5a9f8d5504d84c991f4f200 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 17 Nov 2025 04:23:16 -0500 Subject: [PATCH 043/149] fix(deployment): Adjust readinessProbe timeout and failure threshold. --- tools/deployment/package-helm/templates/_helpers.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 658953f248..314b97131a 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -64,8 +64,8 @@ Creates timings for readiness probes (faster checks for quicker startup). {{- define "clp.readinessProbeTimings" -}} initialDelaySeconds: 1 periodSeconds: 1 -timeoutSeconds: 1 -failureThreshold: 3 +timeoutSeconds: 5 +failureThreshold: 10 {{- end }} {{/* From 7fe32b774262cbe8db5602e44e07b5cab05fc6c9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 05:04:48 -0500 Subject: [PATCH 044/149] chore(ci): Add comment explaining fetch-depth requirement in clp-lint workflow. --- .github/workflows/clp-lint.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml index 68eb65a617..264d5e21c1 100644 --- a/.github/workflows/clp-lint.yaml +++ b/.github/workflows/clp-lint.yaml @@ -25,9 +25,12 @@ jobs: steps: - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" with: - fetch-depth: 0 submodules: "recursive" + # Fetch all history for all branches; otherwise, `helm lint` would complain about not + # finding the `origin/main` branch. + fetch-depth: 0 + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" with: python-version: "3.11" From 6c89ed656d8238299a91315c5116bdaf7fccbb07 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 06:26:06 -0500 Subject: [PATCH 045/149] Reformat database Job initContainer command for readability. --- .../package-helm/templates/db-table-creator-job.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index 49f242a6e9..66cbace10b 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -21,10 +21,14 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo waiting for database; + sleep 2; + done containers: - name: "db-table-creator" image: "{{ include "clp.image.ref" . }}" From e40d239ba49fdc871676b293742240d26f3eb6e9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 11:50:22 -0500 Subject: [PATCH 046/149] Update database livenessProbe to include authentication parameters. --- .../templates/database-statefulset.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index b1365ab412..82df032604 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -64,11 +64,15 @@ spec: livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: &database-health-check - command: [ - "mysqladmin", "ping", - "--silent", - "-h", "127.0.0.1" - ] + command: + - "sh" + - "-c" + - >- + mysqladmin ping + --host=127.0.0.1 + --port=3306 + --user=${MYSQL_USER} + --password=${MYSQL_PASSWORD} readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: From f17264ea11a577075f71b8b356f4f7fb8f0e47f6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 11:52:24 -0500 Subject: [PATCH 047/149] alphabetize --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index 2cab04f19a..49300e3f8c 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -8,5 +8,5 @@ black>=24.4.2 ruff>=0.4.4 # YAML +yamale>=6.0.0 yamllint>=1.35.1 -yamale>=6.0.0 \ No newline at end of file From f51b97eabb0f76411ef3b03fe13ed8e2c5f0fbbf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 11:52:34 -0500 Subject: [PATCH 048/149] Adjust probe timings for readiness and liveness checks. --- tools/deployment/package-helm/templates/_helpers.tpl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 314b97131a..33687f6f97 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -62,9 +62,9 @@ Image reference for CLP Package Creates timings for readiness probes (faster checks for quicker startup). */}} {{- define "clp.readinessProbeTimings" -}} -initialDelaySeconds: 1 -periodSeconds: 1 -timeoutSeconds: 5 +initialDelaySeconds: 6 +periodSeconds: 2 +timeoutSeconds: 2 failureThreshold: 10 {{- end }} @@ -72,9 +72,9 @@ failureThreshold: 10 Creates timings for liveness probes. */}} {{- define "clp.livenessProbeTimings" -}} -initialDelaySeconds: 30 +initialDelaySeconds: 60 periodSeconds: 10 -timeoutSeconds: 5 +timeoutSeconds: 4 failureThreshold: 3 {{- end }} From 8ed32a7f020ff4e4954fc2813ae43b862cd6dd23 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 12:19:01 -0500 Subject: [PATCH 049/149] add url to maintainers - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index ba2ac6ee3d..e7dfe42d3c 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -7,7 +7,7 @@ appVersion: "0.6.1-dev" home: "https://github.com/y-scope/clp" icon: "https://github.com/y-scope/clp/raw/main/docs/src/clp-logo.png" sources: ["https://github.com/y-scope/clp"] -maintainers: [{name: "y-scope", email: "dev@yscope.com"}] +maintainers: [{name: "y-scope", email: "dev@yscope.com", url: "https://yscope.com"}] keywords: - "analytics" - "compression" From 45674e2503dac25cb65d1193d5d69cda5f9fb866 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 12:14:18 -0500 Subject: [PATCH 050/149] Standardize version field placement in Chart.yaml. --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index e7dfe42d3c..f156ec8a54 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: "v2" name: "clp" +version: "0.1.0" description: "A Helm chart for CLP (Compressed Log Processor) package deployment" type: "application" -version: "0.1.0" appVersion: "0.6.1-dev" home: "https://github.com/y-scope/clp" icon: "https://github.com/y-scope/clp/raw/main/docs/src/clp-logo.png" From 2ad9b0aa0020ccf1cd0082d3bfe3843275c1ef40 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 12:44:18 -0500 Subject: [PATCH 051/149] Adjust livenessProbe periodSeconds to 30. --- tools/deployment/package-helm/templates/_helpers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 33687f6f97..3920d4eb18 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -73,7 +73,7 @@ Creates timings for liveness probes. */}} {{- define "clp.livenessProbeTimings" -}} initialDelaySeconds: 60 -periodSeconds: 10 +periodSeconds: 30 timeoutSeconds: 4 failureThreshold: 3 {{- end }} From 5d5bd27894910920e1671945708420cfc5905905 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 21 Nov 2025 14:15:45 -0500 Subject: [PATCH 052/149] Swap liveness and readiness probe configurations in database-statefulset.yaml --- .../package-helm/templates/database-statefulset.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 82df032604..06347e5062 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -61,8 +61,8 @@ spec: mountPath: "/etc/mysql/conf.d/logging.cnf" subPath: "mysql-logging.cnf" readOnly: true - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &database-health-check command: - "sh" @@ -73,8 +73,8 @@ spec: --port=3306 --user=${MYSQL_USER} --password=${MYSQL_PASSWORD} - readinessProbe: - {{- include "clp.readinessProbeTimings" . | nindent 12 }} + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: <<: *database-health-check volumes: From fb2d28757927700a4311ceb1860da97e0130038d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 22 Nov 2025 02:28:39 -0500 Subject: [PATCH 053/149] Update all `.yml` extensions to `.yaml` for consistency --- ct.yaml | 2 +- tools/deployment/package-helm/templates/configmap.yaml | 2 +- .../package-helm/templates/db-table-creator-job.yaml | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ct.yaml b/ct.yaml index 11f80fb3ad..95717fd2b6 100644 --- a/ct.yaml +++ b/ct.yaml @@ -2,7 +2,7 @@ chart-dirs: - "tools/deployment" chart-yaml-schema: "chart_schema.yaml" check-version-increment: true -lint-conf: ".yamllint.yml" +lint-conf: ".yamllint.yaml" validate-chart-schema: true validate-maintainers: true validate-yaml: true diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index ee9fc207ae..6925e5e174 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "clp.labels" . | nindent 4 }} data: - clp-config.yml: | + clp-config.yaml: | data_directory: "/var/data" database: auto_commit: false diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index 66cbace10b..d3b6c13a9e 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -48,13 +48,13 @@ spec: value: "/opt/clp/lib/python3/site-packages" volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true command: [ "python3", "-u", "-m", "clp_py_utils.create-db-tables", - "--config", "/etc/clp-config.yml", + "--config", "/etc/clp-config.yaml", "--storage-engine", {{ .Values.clpConfig.package.storage_engine | quote }} ] volumes: From a155ba35942717dffaf8c7bad991fce522aadf91 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 02:22:26 -0500 Subject: [PATCH 054/149] Update configuration file extensions from .yml to .yaml for consistency --- .../package-helm/templates/api-server-deployment.yaml | 6 +++--- .../templates/compression-scheduler-deployment.yaml | 6 +++--- .../templates/compression-worker-deployment.yaml | 6 +++--- .../templates/garbage-collector-deployment.yaml | 6 +++--- .../package-helm/templates/mcp-server-deployment.yaml | 6 +++--- .../package-helm/templates/query-scheduler-deployment.yaml | 6 +++--- .../package-helm/templates/query-worker-deployment.yaml | 6 +++--- .../package-helm/templates/reducer-deployment.yaml | 6 +++--- 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 8bcf8f55f0..d0a91e097a 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -59,8 +59,8 @@ spec: containerPort: 3001 volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "api-server-logs" mountPath: "/var/log/api_server" @@ -68,7 +68,7 @@ spec: "/opt/clp/bin/api_server", "--host", "0.0.0.0", "--port", "3001", - "--config", "/etc/clp-config.yml" + "--config", "/etc/clp-config.yaml" ] livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index a0af4204ca..26fd11e590 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -68,8 +68,8 @@ spec: value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "compression-scheduler-logs" mountPath: "/var/log/compression_scheduler" @@ -81,7 +81,7 @@ spec: command: [ "python3", "-u", "-m", "job_orchestration.scheduler.compress.compression_scheduler", - "--config", "/etc/clp-config.yml" + "--config", "/etc/clp-config.yaml" ] volumes: - name: "config" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 6aee42b056..3baef2dbe3 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -43,7 +43,7 @@ spec: - name: "BROKER_URL" value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_CONFIG_PATH" - value: "/etc/clp-config.yml" + value: "/etc/clp-config.yaml" - name: "CLP_HOME" value: "/opt/clp" - name: "CLP_LOGGING_LEVEL" @@ -58,8 +58,8 @@ spec: value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "compression-worker-logs" mountPath: "/var/log/compression_worker" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index e47bc8b8bd..72f7d1cbf3 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -60,8 +60,8 @@ spec: value: "/opt/clp/lib/python3/site-packages" volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "garbage-collector-logs" mountPath: "/var/log/garbage_collector" @@ -72,7 +72,7 @@ spec: command: [ "python3", "-u", "-m", "job_orchestration.garbage_collector.garbage_collector", - "--config", "/etc/clp-config.yml" + "--config", "/etc/clp-config.yaml" ] volumes: - name: "config" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index e651e7fb63..89b23451d1 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -62,8 +62,8 @@ spec: containerPort: 8000 volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "mcp-server-logs" mountPath: "/var/log/mcp_server" @@ -72,7 +72,7 @@ spec: "-m", "clp_mcp_server.clp_mcp_server", "--host", "0.0.0.0", "--port", "8000", - "--config-path", "/etc/clp-config.yml" + "--config-path", "/etc/clp-config.yaml" ] livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 46a6299234..2ac200a42e 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -73,15 +73,15 @@ spec: containerPort: 7000 volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "query-scheduler-logs" mountPath: "/var/log/query_scheduler" command: [ "python3", "-u", "-m", "job_orchestration.scheduler.query.query_scheduler", - "--config", "/etc/clp-config.yml" + "--config", "/etc/clp-config.yaml" ] volumes: - name: "config" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 6f913e2a86..d91d970c4d 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -43,7 +43,7 @@ spec: - name: "BROKER_URL" value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - name: "CLP_CONFIG_PATH" - value: "/etc/clp-config.yml" + value: "/etc/clp-config.yaml" - name: "CLP_HOME" value: "/opt/clp" - name: "CLP_LOGGING_LEVEL" @@ -58,8 +58,8 @@ spec: value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "query-worker-logs" mountPath: "/var/log/query_worker" diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index edd1693849..fbfeb6ac26 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -52,15 +52,15 @@ spec: value: "/opt/clp/lib/python3/site-packages" volumeMounts: - name: "config" - mountPath: "/etc/clp-config.yml" - subPath: "clp-config.yml" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - name: "reducer-logs" mountPath: "/var/log/reducer" command: [ "python3", "-u", "-m", "job_orchestration.reducer.reducer", - "--config", "/etc/clp-config.yml", + "--config", "/etc/clp-config.yaml", "--concurrency", "{{ .Values.workerConcurrency }}", "--upsert-interval", "{{ .Values.clpConfig.reducer.upsert_interval }}" ] From be9afb3fb1a8a6b9615520f6ed14a32a522d7b0f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 02:30:41 -0500 Subject: [PATCH 055/149] Refactor liveness and readiness probes across multiple deployment files for consistency --- .../templates/api-server-deployment.yaml | 10 ++++---- .../templates/database-statefulset.yaml | 3 +-- .../templates/mcp-server-deployment.yaml | 10 ++++---- .../templates/queue-statefulset.yaml | 11 ++++----- .../templates/redis-statefulset.yaml | 24 +++++++++---------- .../templates/results-cache-statefulset.yaml | 11 ++++----- .../templates/webui-deployment.yaml | 9 ++++--- 7 files changed, 35 insertions(+), 43 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index d0a91e097a..310b74b54c 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -70,16 +70,14 @@ spec: "--port", "3001", "--config", "/etc/clp-config.yaml" ] - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} - httpGet: - path: "/health" - port: 3001 readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} - httpGet: + httpGet: &api-server-health-check path: "/health" port: 3001 + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + httpGet: *api-server-health-check volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 06347e5062..a8859e9f24 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -75,8 +75,7 @@ spec: --password=${MYSQL_PASSWORD} livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} - exec: - <<: *database-health-check + exec: *database-health-check volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 89b23451d1..46793e3e16 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -74,16 +74,14 @@ spec: "--port", "8000", "--config-path", "/etc/clp-config.yaml" ] - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} - httpGet: - path: "/health" - port: 8000 readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} - httpGet: + httpGet: &mcp-server-health-check path: "/health" port: 8000 + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + httpGet: *mcp-server-health-check volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml index bea559736a..94e8ea9bdc 100644 --- a/tools/deployment/package-helm/templates/queue-statefulset.yaml +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -45,14 +45,13 @@ spec: volumeMounts: - name: "queue-logs" mountPath: "/var/log/rabbitmq" - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} - exec: &queue-health-check - command: ["rabbitmq-diagnostics", "check_running"] readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} - exec: - <<: *queue-health-check + exec: &queue-health-check + command: [ "rabbitmq-diagnostics", "check_running" ] + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: *queue-health-check persistentVolumeClaimRetentionPolicy: whenDeleted: "Delete" volumeClaimTemplates: diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 245601fb68..4e6efd906b 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -48,20 +48,20 @@ spec: mountPath: "/etc/redis/redis.conf" subPath: "redis.conf" readOnly: true - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} - exec: &redis-health-check - command: - - "redis-cli" - - "-h" - - "127.0.0.1" - - "-p" - - "6379" - - "-a" - - "$(REDIS_PASSWORD)" - - "PING" readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: &redis-health-check + command: + - "sh" + - "-c" + - >- + redis-cli + -h 127.0.0.1 + -p 6379 + -a "${REDIS_PASSWORD}" + ping + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} exec: <<: *redis-health-check volumes: diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index 441603be4e..aea5c5b84a 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -43,8 +43,8 @@ spec: mountPath: "/etc/mongo/mongod.conf" subPath: "mongod.conf" readOnly: true - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &results-cache-health-check command: - "mongosh" @@ -52,10 +52,9 @@ spec: - "--quiet" - "--eval" - "db.runCommand('ping').ok" - readinessProbe: - {{- include "clp.readinessProbeTimings" . | nindent 12 }} - exec: - <<: *results-cache-health-check + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: *results-cache-health-check volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 6ece053202..2263dca7d0 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -78,14 +78,13 @@ spec: "/opt/clp/bin/node-22", "/opt/clp/var/www/webui/server/dist/src/main.js" ] - livenessProbe: - {{- include "clp.livenessProbeTimings" . | nindent 12 }} - tcpSocket: - port: 4000 readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} - tcpSocket: + tcpSocket: &webui-health-check port: 4000 + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + tcpSocket: *webui-health-check volumes: - name: "client-settings" configMap: From 387ad2fdaa36ab31b543ee05cc22c056a3ebb977 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 02:35:47 -0500 Subject: [PATCH 056/149] Add workerConcurrency setting to values.yaml --- tools/deployment/package-helm/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index c4c8d239a9..43d440cb7f 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -18,6 +18,8 @@ image: pullPolicy: "IfNotPresent" tag: "nightly" +workerConcurrency: 8 + clpConfig: package: storage_engine: "clp-s" From d2a43572f413a61dab27cac9c2b861bdfc3ebd0e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 02:38:21 -0500 Subject: [PATCH 057/149] Set replicas to 1 for compression and query workers --- .../package-helm/templates/compression-worker-deployment.yaml | 2 +- .../package-helm/templates/query-worker-deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 3baef2dbe3..b3540eb310 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-worker" spec: - replicas: {{ .Values.workerConcurrency }} + replicas: 1 selector: matchLabels: {{- include "clp.selectorLabels" . | nindent 6 }} diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index d91d970c4d..d96e79c62f 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-worker" spec: - replicas: {{ .Values.workerConcurrency }} + replicas: 1 selector: matchLabels: {{- include "clp.selectorLabels" . | nindent 6 }} From d4f3c81217d47ca7514587aa06e00e48de14ce73 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 20:27:16 -0500 Subject: [PATCH 058/149] upgrade yamale to 6.1.0 - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index 49300e3f8c..1f6f2d86fc 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -8,5 +8,5 @@ black>=24.4.2 ruff>=0.4.4 # YAML -yamale>=6.0.0 +yamale>=6.1.0 yamllint>=1.35.1 From 1515d0dbb713ab72e3bf73b8c2bc8c367adf633b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 20:39:38 -0500 Subject: [PATCH 059/149] Add chart-testing SHA256 checksum for Linux ARM64 architecture in toolchains.yaml --- taskfiles/toolchains.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 1df385ef74..2472f3f43d 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -32,6 +32,8 @@ tasks: FILE_SHA256: >- {{- if and (eq OS "linux") (eq ARCH "amd64") -}} d16f0583616885423826241164ce1f6589c6fe5332fa74f374ebd2bd3cb3fe1f + {{- else if and (eq OS "linux") (eq ARCH "arm64") -}} + 7b035bbbd0768fba33983e81c15445e33e9c8281d329100852a342ecfd0b16b7 {{- else if and (eq OS "darwin") (eq ARCH "arm64") -}} db10dbbb42b110c7a5da5a3202908f32ad2ca6ad600d423426dc7886d09aad07 {{- else -}} From dd35d68879a7cd81123013b983cd00ed280c665c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 25 Nov 2025 20:41:12 -0500 Subject: [PATCH 060/149] Remove redundant chart schema and lint validation check settings because they are by default true --- ct.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ct.yaml b/ct.yaml index 95717fd2b6..b3232c1b6e 100644 --- a/ct.yaml +++ b/ct.yaml @@ -1,8 +1,4 @@ chart-dirs: - "tools/deployment" chart-yaml-schema: "chart_schema.yaml" -check-version-increment: true lint-conf: ".yamllint.yaml" -validate-chart-schema: true -validate-maintainers: true -validate-yaml: true From 5296f312a9ec527c07c8b426d7e06c2c8b384efb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 07:14:52 -0500 Subject: [PATCH 061/149] Remove the unneeded `chmod u+x` from the helm env source script - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- taskfiles/toolchains.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 1a232a9dce..270943f6d7 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -53,7 +53,6 @@ tasks: #!/bin/sh export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" EOF - chmod +x "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" rust: # We use a label to uniquely identify this task and its checksum under `.task/checksums/`. From 2c412518a4d3ce8984631273053641df754b841f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 07:24:24 -0500 Subject: [PATCH 062/149] use chart-yaml-schema from the downloaded toolchain --- chart_schema.yaml | 38 -------------------------------------- ct.yaml | 2 +- taskfiles/toolchains.yaml | 2 +- 3 files changed, 2 insertions(+), 40 deletions(-) delete mode 100644 chart_schema.yaml diff --git a/chart_schema.yaml b/chart_schema.yaml deleted file mode 100644 index 4f1d69b73b..0000000000 --- a/chart_schema.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# yamllint disable rule:document-start -name: "str()" -home: "str(required=False)" -version: "str()" -apiVersion: "str()" -appVersion: "any(str(), num(), required=False)" -description: "str(required=False)" -keywords: "list(str(), required=False)" -sources: "list(str(), required=False)" -maintainers: "list(include('maintainer'), required=False)" -dependencies: "list(include('dependency'), required=False)" -icon: "str(required=False)" -engine: "str(required=False)" -condition: "str(required=False)" -tags: "str(required=False)" -deprecated: "bool(required=False)" -kubeVersion: "str(required=False)" -annotations: "map(str(), str(), required=False)" -type: "str(required=False)" ---- -maintainer: - name: "str()" - email: "str(required=False)" - url: "str(required=False)" ---- -dependency: - name: "str()" - version: "str()" - repository: "str(required=False)" - condition: "str(required=False)" - tags: "list(str(), required=False)" - enabled: "bool(required=False)" - import-values: "list(any(str(), include('import-value')), required=False)" - alias: "str(required=False)" ---- -import-value: - child: "str()" - parent: "str()" diff --git a/ct.yaml b/ct.yaml index b3232c1b6e..d50cc5563f 100644 --- a/ct.yaml +++ b/ct.yaml @@ -1,4 +1,4 @@ chart-dirs: - "tools/deployment" -chart-yaml-schema: "chart_schema.yaml" +chart-yaml-schema: "build/toolchains/helm/etc/chart_schema.yaml" lint-conf: ".yamllint.yaml" diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 270943f6d7..7e97ac55c1 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -39,7 +39,7 @@ tasks: {{- else -}} unsupported {{- end }} - INCLUDE_PATTERNS: ["ct"] + INCLUDE_PATTERNS: ["ct", "etc/chart_schema.yaml"] NUM_COMPONENTS_TO_STRIP: "0" - |- export PATH="{{.G_HELM_TOOLCHAIN_DIR}}:$PATH" From 093113611604144d0b0c91d0ca737c6a5e7a28ad Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 07:44:32 -0500 Subject: [PATCH 063/149] Remove chart_schema.yaml from lint.yaml --- taskfiles/lint.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 6187f0bdca..9654a83216 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -119,7 +119,6 @@ tasks: "components/core/config" \ "components/core/tools/" \ "components/package-template/src/etc" \ - "chart_schema.yaml" \ "ct.yaml" \ "docs" \ "taskfile.yaml" \ From dd70f5c2a819fbb0c9676e7aa5d1fb80f937abfa Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:36:45 -0500 Subject: [PATCH 064/149] Assign `nodeRole` for database persistent volumes to target control-plane nodes. Update helper template to include `nodeRole` field with fallback logic. --- tools/deployment/package-helm/templates/_helpers.tpl | 8 +++++++- .../package-helm/templates/database-data-pv.yaml | 1 + .../package-helm/templates/database-logs-pv.yaml | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 3920d4eb18..4d44c0472d 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -87,6 +87,9 @@ Parameters (dict): capacity: Storage capacity accessModes: Access modes (list) hostPath: Absolute path on host + nodeRole: Node role for affinity + Targets nodes with label "node-role.kubernetes.io/" + Always falls back to "node-role.kubernetes.io/control-plane" */}} {{- define "clp.createLocalPv" -}} apiVersion: "v1" @@ -108,7 +111,10 @@ spec: required: nodeSelectorTerms: - matchExpressions: - - key: "kubernetes.io/hostname" + - key: {{ printf "node-role.kubernetes.io/%s" .nodeRole | quote }} + operator: "Exists" + - matchExpressions: + - key: "node-role.kubernetes.io/control-plane" operator: "Exists" {{- end }} diff --git a/tools/deployment/package-helm/templates/database-data-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml index df4d58c1c2..98beaaf866 100644 --- a/tools/deployment/package-helm/templates/database-data-pv.yaml +++ b/tools/deployment/package-helm/templates/database-data-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-database-data" (include "clp.fullname" .)) "component" "database" + "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/database" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml index 0fdc598ee0..27ad28d197 100644 --- a/tools/deployment/package-helm/templates/database-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-database-logs" (include "clp.fullname" .)) "component" "database" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/database" .Values.clpConfig.logs_directory) From ea062688bf7edda734ba8b33f1e7d6213148373b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:37:33 -0500 Subject: [PATCH 065/149] Improve waiting prompts - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- .../deployment/package-helm/templates/db-table-creator-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index d3b6c13a9e..e397494f8d 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -26,7 +26,7 @@ spec: - "-c" - >- until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo waiting for database; + do echo Waiting for database to be ready...; sleep 2; done containers: From ba5d4dfe6eef2f288d6cec568335019555828874 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:37:50 -0500 Subject: [PATCH 066/149] update helm chart description - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index f156ec8a54..26b6fe1201 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v2" name: "clp" version: "0.1.0" -description: "A Helm chart for CLP (Compressed Log Processor) package deployment" +description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.6.1-dev" home: "https://github.com/y-scope/clp" From 8b44dd63e52b4dbb43c03de3f9914288d7ddabcf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:38:08 -0500 Subject: [PATCH 067/149] update docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/deployment/package-helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 1ed16596db..6b9ba6fd06 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -1,7 +1,7 @@ nameOverride: "" fullnameOverride: "" -# Whether to allow sbin script accesses to services on the node host. +# Whether to allow scripts in `sbin` access to services on the host. allowSbinScripts: true securityContext: From 9ff1611293617a3f8f1773c8573a3a31ea3ea24a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:38:36 -0500 Subject: [PATCH 068/149] Rename `allowSbinScripts` to `allowHostAccessForSbinScripts` for clearer intent in Helm values and templates. --- tools/deployment/package-helm/templates/database-service.yaml | 4 ++-- tools/deployment/package-helm/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package-helm/templates/database-service.yaml b/tools/deployment/package-helm/templates/database-service.yaml index fe6f2f823e..f940b65f86 100644 --- a/tools/deployment/package-helm/templates/database-service.yaml +++ b/tools/deployment/package-helm/templates/database-service.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "database" spec: - {{- if .Values.allowSbinScripts }} + {{- if .Values.allowHostAccessForSbinScripts }} type: "NodePort" {{- else }} clusterIP: "None" @@ -17,6 +17,6 @@ spec: ports: - port: 3306 targetPort: "database" - {{- if .Values.allowSbinScripts }} + {{- if .Values.allowHostAccessForSbinScripts }} nodePort: {{ .Values.clpConfig.database.port }} {{- end }} diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 6b9ba6fd06..20de735f43 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -2,7 +2,7 @@ nameOverride: "" fullnameOverride: "" # Whether to allow scripts in `sbin` access to services on the host. -allowSbinScripts: true +allowHostAccessForSbinScripts: true securityContext: firstParty: From 014097123366402c351b5f55b8bf772c19a49dc5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:48:32 -0500 Subject: [PATCH 069/149] update test script shebang, update TODO comment, set eu pipefail - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/deployment/package-helm/test.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 1111d8d985..c1c45cb9f7 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -1,6 +1,10 @@ -#!/bin/bash +#!/usr/bin/env bash -# TODO: to be deleted / migrated to integration tests +# TODO: Migrate into integration test + +set -o errexit +set -o nounset +set -o pipefail kind delete cluster --name clp-test rm -rf /tmp/clp From 44bd1af1ee16e3bf801323ae0580e1ffd88afc3b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:45:04 -0500 Subject: [PATCH 070/149] Make image pull policy configurable for db-table-creator job in Helm templates. --- .../deployment/package-helm/templates/db-table-creator-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index e397494f8d..a3ece1f23e 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -32,7 +32,7 @@ spec: containers: - name: "db-table-creator" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "CLP_DB_USER" valueFrom: From 72013ed41c04dd86a00369ba9e8447e15b1c5ff3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 26 Nov 2025 19:57:36 -0500 Subject: [PATCH 071/149] Add JSDoc-style comments for helper templates in Helm chart --- .../package-helm/templates/_helpers.tpl | 64 ++++++++++++------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 4d44c0472d..19bbb3e238 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -1,14 +1,18 @@ {{/* -Expand the name of the chart. +Expands the name of the chart. + +@return {string} The chart name (truncated to 63 characters) */}} {{- define "clp.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} {{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. +Creates a default fully qualified app name. We truncate at 63 chars because +some Kubernetes name fields are limited to this (by the DNS naming spec). If +release name contains chart name it will be used as a full name. + +@return {string} The fully qualified app name (truncated to 63 characters) */}} {{- define "clp.fullname" -}} {{- if .Values.fullnameOverride }} @@ -24,14 +28,18 @@ If release name contains chart name it will be used as a full name. {{- end }} {{/* -Create chart name and version as used by the chart label. +Creates chart name and version as used by the chart label. + +@return {string} Chart name and version (truncated to 63 characters) */}} {{- define "clp.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* -Common labels +Creates common labels for all resources. + +@return {string} YAML-formatted common labels */}} {{- define "clp.labels" -}} helm.sh/chart: {{ include "clp.chart" . }} @@ -43,7 +51,9 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} {{/* -Selector labels +Creates selector labels for matching resources. + +@return {string} YAML-formatted selector labels */}} {{- define "clp.selectorLabels" -}} app.kubernetes.io/name: {{ include "clp.name" . }} @@ -51,7 +61,9 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* -Image reference for CLP Package +Creates image reference for CLP Package. + +@return {string} Full image reference (repository:tag) */}} {{- define "clp.image.ref" -}} {{- $tag := .Values.image.clpPackage.tag | default .Chart.AppVersion }} @@ -60,6 +72,8 @@ Image reference for CLP Package {{/* Creates timings for readiness probes (faster checks for quicker startup). + +@return {string} YAML-formatted readiness probe timing configuration */}} {{- define "clp.readinessProbeTimings" -}} initialDelaySeconds: 6 @@ -70,6 +84,8 @@ failureThreshold: 10 {{/* Creates timings for liveness probes. + +@return {string} YAML-formatted liveness probe timing configuration */}} {{- define "clp.livenessProbeTimings" -}} initialDelaySeconds: 60 @@ -80,16 +96,17 @@ failureThreshold: 3 {{/* Creates a local PersistentVolume. -Parameters (dict): - root: Root template context - name: PV name (full) - component: Component label - capacity: Storage capacity - accessModes: Access modes (list) - hostPath: Absolute path on host - nodeRole: Node role for affinity - Targets nodes with label "node-role.kubernetes.io/" - Always falls back to "node-role.kubernetes.io/control-plane" + +@param {object} root Root template context +@param {string} name PV name +@param {string} component Component label +@param {string} capacity Storage capacity +@param {string[]} accessModes Access modes (list) +@param {string} hostPath Absolute path on host +@param {string} nodeRole Node role for affinity. Targets nodes with label + "node-role.kubernetes.io/". Always falls back to + "node-role.kubernetes.io/control-plane" +@return {string} YAML-formatted PersistentVolume resource */}} {{- define "clp.createLocalPv" -}} apiVersion: "v1" @@ -120,11 +137,12 @@ spec: {{/* Creates a PersistentVolumeClaim. -Parameters (dict): - root: Root template context - component: Component label (used in name, labels, and selectors) - capacity: Storage capacity - accessModes: Access modes (list) + +@param {object} root Root template context +@param {string} component Component label +@param {string} capacity Storage capacity +@param {string[]} accessModes +@return {string} YAML-formatted PersistentVolumeClaim resource */}} {{- define "clp.createPvc" -}} apiVersion: "v1" From 1452795e79e427c83fe288d177ba6d888d7dd7d9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 27 Nov 2025 06:41:10 -0500 Subject: [PATCH 072/149] improve docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/deployment/package-helm/templates/_helpers.tpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 19bbb3e238..51c0258c89 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -8,9 +8,9 @@ Expands the name of the chart. {{- end }} {{/* -Creates a default fully qualified app name. We truncate at 63 chars because -some Kubernetes name fields are limited to this (by the DNS naming spec). If -release name contains chart name it will be used as a full name. +Creates a default fully qualified app name. We truncate at 63 chars because some Kubernetes name +fields are limited to this (by the DNS naming spec). If release name contains chart name it will be +used as a full name. @return {string} The fully qualified app name (truncated to 63 characters) */}} @@ -61,7 +61,7 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* -Creates image reference for CLP Package. +Creates image reference for the CLP Package. @return {string} Full image reference (repository:tag) */}} From 480c14b594a7ed7071d0ae4aa73ffae688c3fec5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 27 Nov 2025 04:56:26 -0500 Subject: [PATCH 073/149] Remove unused `createPvc` helper definition from Helm templates --- .../package-helm/templates/_helpers.tpl | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 51c0258c89..92b6eae5d9 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -134,32 +134,3 @@ spec: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" {{- end }} - -{{/* -Creates a PersistentVolumeClaim. - -@param {object} root Root template context -@param {string} component Component label -@param {string} capacity Storage capacity -@param {string[]} accessModes -@return {string} YAML-formatted PersistentVolumeClaim resource -*/}} -{{- define "clp.createPvc" -}} -apiVersion: "v1" -kind: "PersistentVolumeClaim" -metadata: - name: {{ include "clp.fullname" .root }}-{{ .component }} - labels: - {{- include "clp.labels" .root | nindent 4 }} - app.kubernetes.io/component: {{ .component | quote }} -spec: - accessModes: {{ .accessModes }} - storageClassName: "local-storage" - selector: - matchLabels: - {{- include "clp.selectorLabels" .root | nindent 6 }} - app.kubernetes.io/component: {{ .component | quote }} - resources: - requests: - storage: {{ .capacity }} -{{- end }} From 41428566dbd3156cdf82d9c0efcca536b2df3494 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 27 Nov 2025 06:09:21 -0500 Subject: [PATCH 074/149] Update helm toolchain output directory to use `G_HELM_TOOLCHAIN_DIR` --- taskfiles/toolchains.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/toolchains.yaml b/taskfiles/toolchains.yaml index 7e97ac55c1..a40d1bd5d5 100644 --- a/taskfiles/toolchains.yaml +++ b/taskfiles/toolchains.yaml @@ -26,7 +26,7 @@ tasks: - "mkdir -p '{{.G_HELM_TOOLCHAIN_DIR}}'" - task: "remote:download-and-extract-tar" vars: - OUTPUT_DIR: "{{.G_BUILD_DIR}}/toolchains/helm" + OUTPUT_DIR: "{{.G_HELM_TOOLCHAIN_DIR}}" URL: "https://github.com/helm/chart-testing/releases/download/v{{.CHART_TESTING_VERSION}}\ /chart-testing_{{.CHART_TESTING_VERSION}}_{{OS}}_{{ARCH}}.tar.gz" FILE_SHA256: >- From 036222fd98b7a049410a02edbecb7028fea75709 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 27 Nov 2025 06:20:04 -0500 Subject: [PATCH 075/149] Remove `ct.yaml` and inline configuration in `lint.yaml` for clarity --- ct.yaml | 4 ---- taskfiles/lint.yaml | 6 ++++-- 2 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 ct.yaml diff --git a/ct.yaml b/ct.yaml deleted file mode 100644 index d50cc5563f..0000000000 --- a/ct.yaml +++ /dev/null @@ -1,4 +0,0 @@ -chart-dirs: - - "tools/deployment" -chart-yaml-schema: "build/toolchains/helm/etc/chart_schema.yaml" -lint-conf: ".yamllint.yaml" diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 9654a83216..fd8f58d790 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -67,7 +67,10 @@ tasks: - |- . "{{.G_LINT_VENV_DIR}}/bin/activate" . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" - ct lint + ct lint \ + --chart-dirs="tools/deployment" \ + --chart-yaml-schema="{{.G_HELM_TOOLCHAIN_DIR}}/etc/chart_schema.yaml" \ + --lint-conf=".yamllint.yaml" check-js: sources: &js_source_files @@ -119,7 +122,6 @@ tasks: "components/core/config" \ "components/core/tools/" \ "components/package-template/src/etc" \ - "ct.yaml" \ "docs" \ "taskfile.yaml" \ "taskfiles" \ From bfa2ea351d4eaceed1954d8dcb49a4bd3f60fbb7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 14:44:01 -0500 Subject: [PATCH 076/149] update values.yaml with the latest clp-config.yaml --- tools/deployment/package-helm/values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 00996b5dbb..d1e53c69cd 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -34,10 +34,12 @@ clpConfig: # this directory will be ignored. directory: "/" + bundled: ["database", "queue", "redis", "results_cache"] + database: type: "mariadb" # "mariadb" or "mysql" host: "localhost" - port: 30306 + port: 3306 name: "clp-db" compression_scheduler: From 93fcbfe841c40b611b405e1ccf6789f588b6baf9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 14:50:16 -0500 Subject: [PATCH 077/149] Assign `nodeRole` to persistent volumes for control-plane targeting. --- tools/deployment/package-helm/templates/api-server-logs-pv.yaml | 1 + tools/deployment/package-helm/templates/archives-pv.yaml | 1 + .../package-helm/templates/compression-scheduler-logs-pv.yaml | 1 + .../package-helm/templates/compression-worker-logs-pv.yaml | 1 + .../templates/compression-worker-staged-archives-pv.yaml | 1 + .../package-helm/templates/garbage-collector-logs-pv.yaml | 1 + tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml | 1 + .../package-helm/templates/query-scheduler-logs-pv.yaml | 1 + .../deployment/package-helm/templates/query-worker-logs-pv.yaml | 1 + .../package-helm/templates/query-worker-staged-streams-pv.yaml | 1 + tools/deployment/package-helm/templates/queue-logs-pv.yaml | 1 + tools/deployment/package-helm/templates/redis-data-pv.yaml | 1 + tools/deployment/package-helm/templates/redis-logs-pv.yaml | 1 + tools/deployment/package-helm/templates/reducer-logs-pv.yaml | 1 + .../deployment/package-helm/templates/results-cache-data-pv.yaml | 1 + .../deployment/package-helm/templates/results-cache-logs-pv.yaml | 1 + tools/deployment/package-helm/templates/streams-pv.yaml | 1 + tools/deployment/package-helm/templates/tmp-pv.yaml | 1 + tools/deployment/package-helm/templates/user-logs-pv.yaml | 1 + 19 files changed, 19 insertions(+) diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index 977be57d6f..74d372325c 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-api-server-logs" (include "clp.fullname" .)) "component" "api-server-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/api-server" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/archives-pv.yaml b/tools/deployment/package-helm/templates/archives-pv.yaml index 11072b5c9d..d1a043d64f 100644 --- a/tools/deployment/package-helm/templates/archives-pv.yaml +++ b/tools/deployment/package-helm/templates/archives-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-archives" (include "clp.fullname" .)) "component" "archives" + "nodeRole" "control-plane" "capacity" "50Gi" "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.archive_output.storage.directory diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 475a1ecc6b..48a46f4f8a 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-compression-scheduler-logs" (include "clp.fullname" .)) "component" "compression-scheduler-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression-scheduler" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index 7d8aeae9ba..3c451126d2 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-compression-worker-logs" (include "clp.fullname" .)) "component" "compression-worker-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression-worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index 6a52ad3e9c..11dcbd1d8a 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-staged-archives" (include "clp.fullname" .)) "component" "staged-archives" + "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") "hostPath" "/tmp/clp/var/data/staged-archives" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index 8eaa2c9ea2..75f3a6a3e5 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-garbage-collector-logs" (include "clp.fullname" .)) "component" "garbage-collector-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/garbage-collector" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index da16ab4e3f..01bbf18ab7 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-mcp-server-logs" (include "clp.fullname" .)) "component" "mcp-server-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/mcp-server" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index 8ec5ed13d4..d7b62604c4 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-query-scheduler-logs" (include "clp.fullname" .)) "component" "query-scheduler-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query-scheduler" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index 007505daae..e92ee8b03f 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-query-worker-logs" (include "clp.fullname" .)) "component" "query-worker-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query-worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index 993cb3bca1..95a641be53 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-staged-streams" (include "clp.fullname" .)) "component" "staged-streams" + "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") "hostPath" "/tmp/clp/var/data/staged-streams" diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml index b6ff44e134..cbc4fa3008 100644 --- a/tools/deployment/package-helm/templates/queue-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-queue-logs" (include "clp.fullname" .)) "component" "queue" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/queue" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml index bf170d9f8c..96fa611225 100644 --- a/tools/deployment/package-helm/templates/redis-data-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-redis-data" (include "clp.fullname" .)) "component" "redis" + "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/redis" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml index 3a1fc890d3..b3308cbfc9 100644 --- a/tools/deployment/package-helm/templates/redis-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-redis-logs" (include "clp.fullname" .)) "component" "redis" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/redis" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index 3fd67c21b5..12c67a778f 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-reducer-logs" (include "clp.fullname" .)) "component" "reducer-logs" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/reducer" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml index 52502fa827..8c67722348 100644 --- a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-results-cache-data" (include "clp.fullname" .)) "component" "results-cache" + "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/results-cache" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index 61c8efa80b..2050d3a033 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-results-cache-logs" (include "clp.fullname" .)) "component" "results-cache" + "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/results-cache" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/streams-pv.yaml b/tools/deployment/package-helm/templates/streams-pv.yaml index f998090001..df211cf22f 100644 --- a/tools/deployment/package-helm/templates/streams-pv.yaml +++ b/tools/deployment/package-helm/templates/streams-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-streams" (include "clp.fullname" .)) "component" "streams" + "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.stream_output.storage.directory diff --git a/tools/deployment/package-helm/templates/tmp-pv.yaml b/tools/deployment/package-helm/templates/tmp-pv.yaml index 43bdd2ca8c..1add0502bd 100644 --- a/tools/deployment/package-helm/templates/tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/tmp-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-tmp" (include "clp.fullname" .)) "component" "tmp" + "nodeRole" "control-plane" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.tmp_directory diff --git a/tools/deployment/package-helm/templates/user-logs-pv.yaml b/tools/deployment/package-helm/templates/user-logs-pv.yaml index b90b351a48..9c43c56e00 100644 --- a/tools/deployment/package-helm/templates/user-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/user-logs-pv.yaml @@ -2,6 +2,7 @@ "root" . "name" (printf "%s-user-logs" (include "clp.fullname" .)) "component" "user-logs" + "nodeRole" "control-plane" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") "hostPath" (printf "%s/user" .Values.clpConfig.logs_directory) From 7a1fff495d655d8ce60cf0bbe3cec8b261fc3c01 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 15:00:43 -0500 Subject: [PATCH 078/149] add back helper to create PersistentVolumeClaim templates. --- .../package-helm/templates/_helpers.tpl | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 92b6eae5d9..1a6946fbe2 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -101,7 +101,7 @@ Creates a local PersistentVolume. @param {string} name PV name @param {string} component Component label @param {string} capacity Storage capacity -@param {string[]} accessModes Access modes (list) +@param {string[]} accessModes Access modes @param {string} hostPath Absolute path on host @param {string} nodeRole Node role for affinity. Targets nodes with label "node-role.kubernetes.io/". Always falls back to @@ -134,3 +134,32 @@ spec: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" {{- end }} + +{{/* +Creates a PersistentVolumeClaim for the given component. + +@param {object} root Root template context +@param {string} component Component label +@param {string} capacity Storage capacity +@param {string[]} accessModes Access modes +@return {string} YAML-formatted PersistentVolumeClaim resource +*/}} +{{- define "clp.createPvc" -}} +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: {{ include "clp.fullname" .root }}-{{ .component }} + labels: + {{- include "clp.labels" .root | nindent 4 }} + app.kubernetes.io/component: {{ .component | quote }} +spec: + accessModes: {{ .accessModes }} + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" .root | nindent 6 }} + app.kubernetes.io/component: {{ .component | quote }} + resources: + requests: + storage: {{ .capacity }} +{{- end }} From ba6abb40d4217022507ece163ee0a751185d4cad Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 15:01:28 -0500 Subject: [PATCH 079/149] fix(helm): Rename values flag for sbin script access in results cache service. --- .../package-helm/templates/results-cache-service.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/results-cache-service.yaml b/tools/deployment/package-helm/templates/results-cache-service.yaml index 2c8e540fd8..0a6a10f4bb 100644 --- a/tools/deployment/package-helm/templates/results-cache-service.yaml +++ b/tools/deployment/package-helm/templates/results-cache-service.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "results-cache" spec: - {{- if .Values.allowSbinScripts }} + {{- if .Values.allowHostAccessForSbinScripts }} type: "NodePort" {{- else }} clusterIP: "None" @@ -17,6 +17,6 @@ spec: ports: - port: 27017 targetPort: "mongodb" - {{- if .Values.allowSbinScripts }} + {{- if .Values.allowHostAccessForSbinScripts }} nodePort: {{ .Values.clpConfig.results_cache.port }} {{- end }} From b25b2433811539ea842b16ad65aa84caed00ae52 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 15:51:08 -0500 Subject: [PATCH 080/149] refactor(helm): Update initContainer command syntax for better readability. --- .../templates/api-server-deployment.yaml | 24 ++++++++----- .../compression-scheduler-deployment.yaml | 36 ++++++++++++------- .../compression-worker-deployment.yaml | 24 ++++++++----- .../garbage-collector-deployment.yaml | 24 ++++++++----- .../templates/mcp-server-deployment.yaml | 24 ++++++++----- .../templates/query-scheduler-deployment.yaml | 36 ++++++++++++------- .../templates/query-worker-deployment.yaml | 24 ++++++++----- .../templates/reducer-deployment.yaml | 24 ++++++++----- .../results-cache-indices-creator-job.yaml | 12 ++++--- .../templates/webui-deployment.yaml | 24 ++++++++----- 10 files changed, 168 insertions(+), 84 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 310b74b54c..7b8a6870fc 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -25,16 +25,24 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo Waiting for database to be ready...; + sleep 2; + done - name: "wait-for-results-cache" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done containers: - name: "api-server" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 26fd11e590..ba21533462 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -25,22 +25,34 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo Waiting for database to be ready...; + sleep 2; + done - name: "wait-for-queue" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-queue 5672; + do echo Waiting for queue to be ready...; + sleep 2; + done - name: "wait-for-redis" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-redis 6379; + do echo Waiting for redis to be ready...; + sleep 2; + done containers: - name: "compression-scheduler" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index b3540eb310..e703c8f251 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -25,16 +25,24 @@ spec: initContainers: - name: "wait-for-queue" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-queue 5672; + do echo Waiting for queue to be ready...; + sleep 2; + done - name: "wait-for-redis" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-redis 6379; + do echo Waiting for redis to be ready...; + sleep 2; + done containers: - name: "compression-worker" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 72f7d1cbf3..8afba4cfe5 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -25,16 +25,24 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo Waiting for database to be ready...; + sleep 2; + done - name: "wait-for-results-cache" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done containers: - name: "garbage-collector" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 46793e3e16..8426f38c00 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -26,16 +26,24 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo Waiting for database to be ready...; + sleep 2; + done - name: "wait-for-results-cache" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done containers: - name: "mcp-server" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 2ac200a42e..6a4a6a6559 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -27,22 +27,34 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo Waiting for database to be ready...; + sleep 2; + done - name: "wait-for-queue" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-queue 5672; + do echo Waiting for queue to be ready...; + sleep 2; + done - name: "wait-for-redis" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-redis 6379; + do echo Waiting for redis to be ready...; + sleep 2; + done containers: - name: "query-scheduler" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index d96e79c62f..a9301c0ae1 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -25,16 +25,24 @@ spec: initContainers: - name: "wait-for-queue" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-queue 5672; do echo waiting for queue; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-queue 5672; + do echo Waiting for queue to be ready...; + sleep 2; + done - name: "wait-for-redis" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-redis 6379; do echo waiting for redis; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-redis 6379; + do echo Waiting for redis to be ready...; + sleep 2; + done containers: - name: "query-worker" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index fbfeb6ac26..3df0db7023 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -27,16 +27,24 @@ spec: initContainers: - name: "wait-for-query-scheduler" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-query-scheduler 7000; do echo waiting for query-scheduler; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-query-scheduler 7000; + do echo Waiting for query-scheduler to be ready...; + sleep 2; + done - name: "wait-for-results-cache" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done containers: - name: "reducer" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index 5ff701fa3a..7cd1075d6c 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -21,10 +21,14 @@ spec: initContainers: - name: "wait-for-results-cache" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done containers: - name: "results-cache-indices-creator" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 2263dca7d0..9ba41c4c00 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -25,16 +25,24 @@ spec: initContainers: - name: "wait-for-database" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-database 3306; do echo waiting for database; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-database 3306; + do echo Waiting for database to be ready...; + sleep 2; + done - name: "wait-for-results-cache" image: "busybox:latest" - command: [ - "sh", "-c", - "until nc -z {{ include "clp.fullname" . }}-results-cache 27017; do echo waiting for results-cache; sleep 2; done" - ] + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done containers: - name: "webui" image: "{{ include "clp.image.ref" . }}" From 4ef25b513321376309b168875c83de5faed1163e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 15:54:19 -0500 Subject: [PATCH 081/149] refactor(helm): Use configurable image pull policy for deployments. --- .../package-helm/templates/api-server-deployment.yaml | 2 +- .../templates/compression-scheduler-deployment.yaml | 2 +- .../package-helm/templates/compression-worker-deployment.yaml | 2 +- .../package-helm/templates/garbage-collector-deployment.yaml | 2 +- .../package-helm/templates/mcp-server-deployment.yaml | 2 +- .../package-helm/templates/query-scheduler-deployment.yaml | 2 +- .../package-helm/templates/query-worker-deployment.yaml | 2 +- tools/deployment/package-helm/templates/reducer-deployment.yaml | 2 +- .../templates/results-cache-indices-creator-job.yaml | 2 +- tools/deployment/package-helm/templates/webui-deployment.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 7b8a6870fc..7f8e5fb1a6 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -46,7 +46,7 @@ spec: containers: - name: "api-server" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "CLP_DB_USER" valueFrom: diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index ba21533462..1de5e362ea 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -56,7 +56,7 @@ spec: containers: - name: "compression-scheduler" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "BROKER_URL" value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index e703c8f251..92db721a17 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -46,7 +46,7 @@ spec: containers: - name: "compression-worker" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "BROKER_URL" value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 8afba4cfe5..a4fcaab0cf 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -46,7 +46,7 @@ spec: containers: - name: "garbage-collector" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "CLP_DB_USER" valueFrom: diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 8426f38c00..beb0c1e1b9 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -47,7 +47,7 @@ spec: containers: - name: "mcp-server" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "CLP_DB_USER" valueFrom: diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 6a4a6a6559..b6c6a2990e 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -58,7 +58,7 @@ spec: containers: - name: "query-scheduler" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "BROKER_URL" value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index a9301c0ae1..ba395d4d29 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -46,7 +46,7 @@ spec: containers: - name: "query-worker" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "BROKER_URL" value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index 3df0db7023..edde1c8fcd 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -48,7 +48,7 @@ spec: containers: - name: "reducer" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "CLP_HOME" value: "/opt/clp" diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index 7cd1075d6c..51c2b822db 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -32,7 +32,7 @@ spec: containers: - name: "results-cache-indices-creator" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 9ba41c4c00..06fed0405d 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -46,7 +46,7 @@ spec: containers: - name: "webui" image: "{{ include "clp.image.ref" . }}" - imagePullPolicy: "Always" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - name: "CLP_DB_USER" valueFrom: From 0ca31739c7f47c66232d3dde8dba60fb41a6b30f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 28 Nov 2025 17:59:59 -0500 Subject: [PATCH 082/149] revert db port --- tools/deployment/package-helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index d1e53c69cd..e45b76b2a8 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -39,7 +39,7 @@ clpConfig: database: type: "mariadb" # "mariadb" or "mysql" host: "localhost" - port: 3306 + port: 30306 name: "clp-db" compression_scheduler: From 399373febb6e6dee38f5c656e735708ca2b03c38 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 02:42:58 -0500 Subject: [PATCH 083/149] refactor(helm): Remove redundant host field from configuration values. --- tools/deployment/package-helm/templates/configmap.yaml | 4 ++-- tools/deployment/package-helm/values.yaml | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 71db36d4d5..e3ab999c33 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -8,7 +8,7 @@ data: clp-config.yaml: | api_server: default_max_num_query_results: {{ .Values.clpConfig.api_server.default_max_num_query_results }} - host: {{ .Values.clpConfig.api_server.host | quote }} + host: "localhost" port: 3001 query_job_polling: initial_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.initial_backoff_ms }} @@ -87,7 +87,7 @@ data: target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size }} tmp_directory: "/var/tmp" webui: - host: {{ .Values.clpConfig.webui.host | quote }} + host: "localhost" port: 4000 rate_limit: {{ .Values.clpConfig.webui.rate_limit }} results_metadata_collection_name: {{ .Values.clpConfig.webui.results_metadata_collection_name | quote }} diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index e45b76b2a8..f49fae664e 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -38,7 +38,6 @@ clpConfig: database: type: "mariadb" # "mariadb" or "mysql" - host: "localhost" port: 30306 name: "clp-db" @@ -47,30 +46,25 @@ clpConfig: logging_level: "INFO" query_scheduler: - host: "localhost" port: 7000 jobs_poll_delay: 0.1 # seconds num_archives_to_search_per_sub_job: 16 logging_level: "INFO" queue: - host: "localhost" port: 5672 redis: - host: "localhost" port: 6379 query_backend_database: 0 compression_backend_database: 1 reducer: - host: "localhost" base_port: 14009 logging_level: "INFO" upsert_interval: 100 # milliseconds results_cache: - host: "localhost" port: 30017 db_name: "clp-query-results" stream_collection_name: "stream-files" @@ -85,13 +79,11 @@ clpConfig: logging_level: "INFO" webui: - host: "localhost" port: 30400 results_metadata_collection_name: "results-metadata" rate_limit: 1000 mcp_server: null - # host: "localhost" # port: 30800 # logging_level: "INFO" @@ -145,7 +137,6 @@ clpConfig: # API server config api_server: - host: "localhost" port: 30301 default_max_num_query_results: 1000 query_job_polling: @@ -154,7 +145,6 @@ clpConfig: # Presto client config presto: null - # host: "localhost" # port: 30889 # Location where other data (besides archives) are stored. It will be created if From 0bb706bfc3559797863b4a62ab24034ff5c62301 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 02:44:07 -0500 Subject: [PATCH 084/149] remove empty line --- tools/deployment/package-helm/templates/configmap.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index e3ab999c33..6ad7cf6b5b 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -51,7 +51,6 @@ data: package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} - presto: {{ .Values.clpConfig.presto }} query_scheduler: host: {{ include "clp.fullname" . }}-query-scheduler From 7f90a853738fbfbf69ce66742d6c25c165bd04a6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 02:48:49 -0500 Subject: [PATCH 085/149] Update image tag to 'main' and pull policy to 'Always' in values.yaml. --- tools/deployment/package-helm/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index f49fae664e..532c55d828 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -15,8 +15,8 @@ securityContext: image: clpPackage: repository: "ghcr.io/y-scope/clp/clp-package" - pullPolicy: "IfNotPresent" - tag: "nightly" + pullPolicy: "Always" + tag: "main" workerConcurrency: 8 From 39fdcd604fa6b99febc767ec095cfc3088f566e4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 03:16:58 -0500 Subject: [PATCH 086/149] feat(deployment): Add redis, queue (RabbitMQ), and results cache (MongoDB) as `StatefulSet` into the CLP Package Helm chart. --- .../package-helm/templates/_helpers.tpl | 2 +- .../package-helm/templates/configmap.yaml | 30 ++++++ .../templates/database-statefulset.yaml | 3 +- .../package-helm/templates/queue-logs-pv.yaml | 9 ++ .../package-helm/templates/queue-secret.yaml | 11 +++ .../package-helm/templates/queue-service.yaml | 15 +++ .../templates/queue-statefulset.yaml | 69 +++++++++++++ .../package-helm/templates/redis-data-pv.yaml | 9 ++ .../package-helm/templates/redis-logs-pv.yaml | 9 ++ .../package-helm/templates/redis-secret.yaml | 10 ++ .../package-helm/templates/redis-service.yaml | 15 +++ .../templates/redis-statefulset.yaml | 97 +++++++++++++++++++ .../templates/results-cache-data-pv.yaml | 9 ++ .../results-cache-indices-creator-job.yaml | 44 +++++++++ .../templates/results-cache-logs-pv.yaml | 9 ++ .../templates/results-cache-service.yaml | 22 +++++ .../templates/results-cache-statefulset.yaml | 88 +++++++++++++++++ tools/deployment/package-helm/test.sh | 5 +- tools/deployment/package-helm/values.yaml | 23 ++++- 19 files changed, 472 insertions(+), 7 deletions(-) create mode 100644 tools/deployment/package-helm/templates/queue-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/queue-secret.yaml create mode 100644 tools/deployment/package-helm/templates/queue-service.yaml create mode 100644 tools/deployment/package-helm/templates/queue-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/redis-data-pv.yaml create mode 100644 tools/deployment/package-helm/templates/redis-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/redis-secret.yaml create mode 100644 tools/deployment/package-helm/templates/redis-service.yaml create mode 100644 tools/deployment/package-helm/templates/redis-statefulset.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-data-pv.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-service.yaml create mode 100644 tools/deployment/package-helm/templates/results-cache-statefulset.yaml diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 92b6eae5d9..6aa0e2b873 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -101,7 +101,7 @@ Creates a local PersistentVolume. @param {string} name PV name @param {string} component Component label @param {string} capacity Storage capacity -@param {string[]} accessModes Access modes (list) +@param {string[]} accessModes Access modes @param {string} hostPath Absolute path on host @param {string} nodeRole Node role for affinity. Targets nodes with label "node-role.kubernetes.io/". Always falls back to diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 6925e5e174..f58670e2bd 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -19,6 +19,20 @@ data: package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} + queue: + host: {{ include "clp.fullname" . }}-queue + port: 5672 + redis: + compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} + host: {{ include "clp.fullname" . }}-redis + port: 6379 + query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} + results_cache: + db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} + host: {{ include "clp.fullname" . }}-results-cache + port: 27017 + retention_period: {{ .Values.clpConfig.results_cache.retention_period }} + stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} mysql-logging.cnf: | [mysqld] @@ -31,3 +45,19 @@ data: # https://dev.mysql.com/doc/refman/8.0/en/error-log-destination-configuration.html # Send error logs to file log_error=/var/log/mysql/mysql-error.log + + redis.conf: | + # For full reference of configuration options see + # https://raw.githubusercontent.com/redis/redis/7.2/redis.conf + + databases 2 + logfile "/var/log/redis/redis.log" + + mongod.conf: | + replication: + replSetName: "rs0" + systemLog: + destination: file + path: /var/log/mongodb/mongod.log + logAppend: true + timeStampFormat: iso8601-local \ No newline at end of file diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 06347e5062..a8859e9f24 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -75,8 +75,7 @@ spec: --password=${MYSQL_PASSWORD} livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} - exec: - <<: *database-health-check + exec: *database-health-check volumes: - name: "config" configMap: diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml new file mode 100644 index 0000000000..cbc4fa3008 --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-queue-logs" (include "clp.fullname" .)) + "component" "queue" + "nodeRole" "control-plane" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/queue" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/queue-secret.yaml b/tools/deployment/package-helm/templates/queue-secret.yaml new file mode 100644 index 0000000000..704e443b64 --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp.fullname" . }}-queue + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +type: "Opaque" +stringData: + username: {{ .Values.credentials.queue.username | quote }} + password: {{ .Values.credentials.queue.password | quote }} diff --git a/tools/deployment/package-helm/templates/queue-service.yaml b/tools/deployment/package-helm/templates/queue-service.yaml new file mode 100644 index 0000000000..2971725fdb --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-queue + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "queue" + ports: + - port: 5672 + targetPort: "amqp" diff --git a/tools/deployment/package-helm/templates/queue-statefulset.yaml b/tools/deployment/package-helm/templates/queue-statefulset.yaml new file mode 100644 index 0000000000..94e8ea9bdc --- /dev/null +++ b/tools/deployment/package-helm/templates/queue-statefulset.yaml @@ -0,0 +1,69 @@ +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp.fullname" . }}-queue + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "queue" +spec: + serviceName: {{ include "clp.fullname" . }}-queue + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "queue" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "queue" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "queue" + image: "rabbitmq:3.9.8" + imagePullPolicy: "Always" + env: + - name: "RABBITMQ_DEFAULT_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-queue + key: "username" + - name: "RABBITMQ_DEFAULT_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-queue + key: "password" + - name: "RABBITMQ_LOGS" + value: "/var/log/rabbitmq/rabbitmq.log" + ports: + - name: "amqp" + containerPort: 5672 + volumeMounts: + - name: "queue-logs" + mountPath: "/var/log/rabbitmq" + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: &queue-health-check + command: [ "rabbitmq-diagnostics", "check_running" ] + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: *queue-health-check + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" + volumeClaimTemplates: + - metadata: + name: "queue-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "queue" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml new file mode 100644 index 0000000000..96fa611225 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-redis-data" (include "clp.fullname" .)) + "component" "redis" + "nodeRole" "control-plane" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/redis" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml new file mode 100644 index 0000000000..b3308cbfc9 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-redis-logs" (include "clp.fullname" .)) + "component" "redis" + "nodeRole" "control-plane" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/redis" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/redis-secret.yaml b/tools/deployment/package-helm/templates/redis-secret.yaml new file mode 100644 index 0000000000..244ccb5291 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: "v1" +kind: "Secret" +metadata: + name: {{ include "clp.fullname" . }}-redis + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +type: "Opaque" +stringData: + password: {{ .Values.credentials.redis.password | quote }} diff --git a/tools/deployment/package-helm/templates/redis-service.yaml b/tools/deployment/package-helm/templates/redis-service.yaml new file mode 100644 index 0000000000..5cd8e61386 --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-service.yaml @@ -0,0 +1,15 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-redis + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +spec: + clusterIP: "None" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "redis" + ports: + - port: 6379 + targetPort: "redis" diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml new file mode 100644 index 0000000000..4e6efd906b --- /dev/null +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -0,0 +1,97 @@ +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp.fullname" . }}-redis + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "redis" +spec: + serviceName: {{ include "clp.fullname" . }}-redis + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "redis" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "redis" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "redis" + image: "redis:7.2.4" + imagePullPolicy: "Always" + command: [ + "redis-server", "/etc/redis/redis.conf", + "--requirepass", "$(REDIS_PASSWORD)" + ] + env: + - name: "REDIS_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-redis + key: "password" + ports: + - name: "redis" + containerPort: 6379 + volumeMounts: + - name: "redis-data" + mountPath: "/data" + - name: "redis-logs" + mountPath: "/var/log/redis" + - name: "config" + mountPath: "/etc/redis/redis.conf" + subPath: "redis.conf" + readOnly: true + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: &redis-health-check + command: + - "sh" + - "-c" + - >- + redis-cli + -h 127.0.0.1 + -p 6379 + -a "${REDIS_PASSWORD}" + ping + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: + <<: *redis-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" + volumeClaimTemplates: + - metadata: + name: "redis-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "redis" + resources: + requests: + storage: "10Gi" + - metadata: + name: "redis-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "redis" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml new file mode 100644 index 0000000000..8c67722348 --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-results-cache-data" (include "clp.fullname" .)) + "component" "results-cache" + "nodeRole" "control-plane" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/results-cache" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml new file mode 100644 index 0000000000..51c2b822db --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -0,0 +1,44 @@ +apiVersion: "batch/v1" +kind: "Job" +metadata: + name: {{ include "clp.fullname" . }}-results-cache-indices-creator + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache-indices-creator" +spec: + ttlSecondsAfterFinished: 300 + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "results-cache-indices-creator" + spec: + restartPolicy: "OnFailure" + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - name: "wait-for-results-cache" + image: "busybox:latest" + command: + - "sh" + - "-c" + - >- + until nc -z {{ include "clp.fullname" . }}-results-cache 27017; + do echo Waiting for results-cache to be ready...; + sleep 2; + done + containers: + - name: "results-cache-indices-creator" + image: "{{ include "clp.image.ref" . }}" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" + env: + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + command: [ + "python3", "-u", + "-m", "clp_py_utils.initialize-results-cache", + "--uri", "mongodb://{{ include "clp.fullname" . }}-results-cache:27017/{{ .Values.clpConfig.results_cache.db_name }}", + "--stream-collection", {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} + ] diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml new file mode 100644 index 0000000000..2050d3a033 --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "name" (printf "%s-results-cache-logs" (include "clp.fullname" .)) + "component" "results-cache" + "nodeRole" "control-plane" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/results-cache" .Values.clpConfig.logs_directory) +) }} diff --git a/tools/deployment/package-helm/templates/results-cache-service.yaml b/tools/deployment/package-helm/templates/results-cache-service.yaml new file mode 100644 index 0000000000..0a6a10f4bb --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-service.yaml @@ -0,0 +1,22 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-results-cache + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + {{- if .Values.allowHostAccessForSbinScripts }} + type: "NodePort" + {{- else }} + clusterIP: "None" + {{- end }} + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" + ports: + - port: 27017 + targetPort: "mongodb" + {{- if .Values.allowHostAccessForSbinScripts }} + nodePort: {{ .Values.clpConfig.results_cache.port }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml new file mode 100644 index 0000000000..aea5c5b84a --- /dev/null +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -0,0 +1,88 @@ +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: {{ include "clp.fullname" . }}-results-cache + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "results-cache" +spec: + serviceName: {{ include "clp.fullname" . }}-results-cache + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "results-cache" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "results-cache" + spec: + securityContext: + runAsUser: {{ .Values.securityContext.thirdParty.uid }} + runAsGroup: {{ .Values.securityContext.thirdParty.gid }} + fsGroup: {{ .Values.securityContext.thirdParty.gid }} + containers: + - name: "results-cache" + image: "mongo:7.0.1" + imagePullPolicy: "Always" + command: [ + "mongod", + "--config", "/etc/mongo/mongod.conf", + "--bind_ip", "0.0.0.0" + ] + ports: + - name: "mongodb" + containerPort: 27017 + volumeMounts: + - name: "results-cache-data" + mountPath: "/data/db" + - name: "results-cache-logs" + mountPath: "/var/log/mongodb" + - name: "config" + mountPath: "/etc/mongo/mongod.conf" + subPath: "mongod.conf" + readOnly: true + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + exec: &results-cache-health-check + command: + - "mongosh" + - "127.0.0.1:27017/test" + - "--quiet" + - "--eval" + - "db.runCommand('ping').ok" + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + exec: *results-cache-health-check + volumes: + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + persistentVolumeClaimRetentionPolicy: + whenDeleted: "Delete" + volumeClaimTemplates: + - metadata: + name: "results-cache-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "results-cache" + resources: + requests: + storage: "20Gi" + - metadata: + name: "results-cache-logs" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "local-storage" + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: "results-cache" + resources: + requests: + storage: "5Gi" diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index c1c45cb9f7..f2f9664170 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -8,7 +8,7 @@ set -o pipefail kind delete cluster --name clp-test rm -rf /tmp/clp -mkdir -p /tmp/clp/var/{data,log}/database +mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache} cat < Date: Sat, 29 Nov 2025 03:25:10 -0500 Subject: [PATCH 087/149] chore(deployment): Bump Helm chart version to 0.1.1. --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 26b6fe1201..6ad8ec4576 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.1.0" +version: "0.1.1" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.6.1-dev" From 4fb6dd41ea57af4de2b9bfcff4448b5db26d855b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 03:33:03 -0500 Subject: [PATCH 088/149] add empty newline at EOF --- tools/deployment/package-helm/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index f58670e2bd..f5ae83a580 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -60,4 +60,4 @@ data: destination: file path: /var/log/mongodb/mongod.log logAppend: true - timeStampFormat: iso8601-local \ No newline at end of file + timeStampFormat: iso8601-local From 1f4df0c21098d2446ff24bcd7ab718cac46cd653 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 03:34:06 -0500 Subject: [PATCH 089/149] docs(deployment): Reorder parameters in Helm helper function documentation. --- tools/deployment/package-helm/templates/_helpers.tpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 6aa0e2b873..6577538785 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -100,12 +100,12 @@ Creates a local PersistentVolume. @param {object} root Root template context @param {string} name PV name @param {string} component Component label -@param {string} capacity Storage capacity -@param {string[]} accessModes Access modes -@param {string} hostPath Absolute path on host @param {string} nodeRole Node role for affinity. Targets nodes with label "node-role.kubernetes.io/". Always falls back to "node-role.kubernetes.io/control-plane" +@param {string} capacity Storage capacity +@param {string[]} accessModes Access modes +@param {string} hostPath Absolute path on host @return {string} YAML-formatted PersistentVolume resource */}} {{- define "clp.createLocalPv" -}} From 683f03b70b251c0e71445f898a157d9e2ed7bced Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 29 Nov 2025 03:44:04 -0500 Subject: [PATCH 090/149] fix(helm): Simplify syntax for redis livenessProbe exec configuration. --- tools/deployment/package-helm/templates/redis-statefulset.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index 4e6efd906b..ae5db90e3d 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -62,8 +62,7 @@ spec: ping livenessProbe: {{- include "clp.livenessProbeTimings" . | nindent 12 }} - exec: - <<: *redis-health-check + exec: *redis-health-check volumes: - name: "config" configMap: From 33ea69a2669d6f5ac77f4259c5b2bcc3af68f25c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 14:03:18 -0500 Subject: [PATCH 091/149] Update port names for services and deployments. --- .../package-helm/templates/api-server-deployment.yaml | 2 +- tools/deployment/package-helm/templates/api-server-service.yaml | 2 +- .../package-helm/templates/mcp-server-deployment.yaml | 2 +- tools/deployment/package-helm/templates/mcp-server-service.yaml | 2 +- tools/deployment/package-helm/templates/webui-deployment.yaml | 2 +- tools/deployment/package-helm/templates/webui-service.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 7f8e5fb1a6..7a97719902 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -63,7 +63,7 @@ spec: - name: "RUST_LOG" value: "DEBUG" ports: - - name: "http" + - name: "api-server" containerPort: 3001 volumeMounts: - name: "config" diff --git a/tools/deployment/package-helm/templates/api-server-service.yaml b/tools/deployment/package-helm/templates/api-server-service.yaml index 7c24c0f6e5..500b5cd6b6 100644 --- a/tools/deployment/package-helm/templates/api-server-service.yaml +++ b/tools/deployment/package-helm/templates/api-server-service.yaml @@ -12,5 +12,5 @@ spec: app.kubernetes.io/component: "api-server" ports: - port: 3001 - targetPort: "http" + targetPort: "api-server" nodePort: {{ .Values.clpConfig.api_server.port }} diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index beb0c1e1b9..bfe5273352 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -66,7 +66,7 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" ports: - - name: "http" + - name: "mcp-server" containerPort: 8000 volumeMounts: - name: "config" diff --git a/tools/deployment/package-helm/templates/mcp-server-service.yaml b/tools/deployment/package-helm/templates/mcp-server-service.yaml index ee2ae9fbe9..9acbe48f9e 100644 --- a/tools/deployment/package-helm/templates/mcp-server-service.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-service.yaml @@ -13,6 +13,6 @@ spec: app.kubernetes.io/component: "mcp-server" ports: - port: 8000 - targetPort: "http" + targetPort: "mcp-server" nodePort: {{ .Values.clpConfig.mcp_server.port }} {{- end }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 06fed0405d..4fc2f28e7d 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -69,7 +69,7 @@ spec: - name: "RATE_LIMIT" value: "1000" ports: - - name: "http" + - name: "webui" containerPort: 4000 volumeMounts: - name: "client-settings" diff --git a/tools/deployment/package-helm/templates/webui-service.yaml b/tools/deployment/package-helm/templates/webui-service.yaml index ebe1ea2074..aa1e885680 100644 --- a/tools/deployment/package-helm/templates/webui-service.yaml +++ b/tools/deployment/package-helm/templates/webui-service.yaml @@ -12,5 +12,5 @@ spec: app.kubernetes.io/component: "webui" ports: - port: 4000 - targetPort: "http" + targetPort: "webui" nodePort: {{ .Values.clpConfig.webui.port }} From 7a52e6c5125744520e3846f372374bced3687e54 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 14:03:59 -0500 Subject: [PATCH 092/149] chore(helm): Bump chart version to 0.1.2-dev.3 --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 9469a9b469..0a9c85ff60 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.1.2-dev.2" +version: "0.1.2-dev.3" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.7.1-dev" From abb1cf87b35b5f6b4679a7ae6aa4d7cfa8eb2c45 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 14:41:15 -0500 Subject: [PATCH 093/149] feat(helm): Add `max_concurrent_tasks_per_job` parameter to compression_scheduler configuration --- tools/deployment/package-helm/templates/configmap.yaml | 1 + tools/deployment/package-helm/values.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 6ad7cf6b5b..7cc8861103 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -27,6 +27,7 @@ data: compression_scheduler: jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} + max_concurrent_tasks_per_job: {{ .Values.clpConfig.compression_scheduler.max_concurrent_tasks_per_job }} compression_worker: logging_level: {{ .Values.clpConfig.compression_worker.logging_level | quote }} data_directory: "/var/data" diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index d0311cf337..6ee186ff1e 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -44,6 +44,7 @@ clpConfig: compression_scheduler: jobs_poll_delay: 0.1 # seconds logging_level: "INFO" + max_concurrent_tasks_per_job: 0 # A value of 0 disables the limit query_scheduler: port: 7000 From 3bf2d940781c0c0fbaf8cd97ec97a58ecd6a3482 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 14:50:36 -0500 Subject: [PATCH 094/149] Update configmap webui settings --- .../deployment/package-helm/templates/configmap.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 7cc8861103..8a732a2a8b 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -124,10 +124,11 @@ data: { "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, + "LogsInputType": {{ .Values.clpConfig.logs_input.type | quote }}, "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, - "SqlDbClpArchivesTableName": "", + "SqlDbClpArchivesTableName": "clp_archives", "SqlDbClpDatasetsTableName": "clp_datasets", - "SqlDbClpFilesTableName": "", + "SqlDbClpFilesTableName": "clp_files", "SqlDbClpTablePrefix": "clp_", "SqlDbCompressionJobsTableName": "compression_jobs" } @@ -138,6 +139,7 @@ data: "SqlDbPort": 3306, "SqlDbName": {{ .Values.clpConfig.database.name | quote }}, "SqlDbQueryJobsTableName": "query_jobs", + "SqlDbCompressionJobsTableName": "compression_jobs", "MongoDbHost": "{{ include "clp.fullname" . }}-results-cache", "MongoDbPort": 27017, "MongoDbName": {{ .Values.clpConfig.results_cache.db_name | quote }}, @@ -150,7 +152,13 @@ data: "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, + "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level }}, + "ArchiveOutputTargetArchiveSize": {{ .Values.clpConfig.archive_output.target_archive_size }}, + "ArchiveOutputTargetDictionariesSize": {{ .Values.clpConfig.archive_output.target_dictionaries_size }}, + "ArchiveOutputTargetEncodedFileSize": {{ .Values.clpConfig.archive_output.target_encoded_file_size }}, + "ArchiveOutputTargetSegmentSize": {{ .Values.clpConfig.archive_output.target_segment_size }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, + "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, "PrestoHost": null, "PrestoPort": null } From 0195c05172e364fd7d4075aff2a1fe89212599f2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 14:54:45 -0500 Subject: [PATCH 095/149] Make `StreamTargetUncompressedSize` configurable via values file --- tools/deployment/package-helm/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 8a732a2a8b..7d3a5f8463 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -148,7 +148,7 @@ data: "ClientDir": "/opt/clp/var/www/webui/client", "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer", "StreamFilesDir": "/var/data/streams", - "StreamTargetUncompressedSize": 134217728, + "StreamTargetUncompressedSize": {{ .Values.clpConfig.stream_output.target_uncompressed_size }}, "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, From af2a04b585a7ead2c5d8d8e7ce3aee35e62b0911 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 15:06:08 -0500 Subject: [PATCH 096/149] avoid encoding u64 as float --- .../package-helm/templates/configmap.yaml | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 7d3a5f8463..e204cfdb97 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -7,7 +7,8 @@ metadata: data: clp-config.yaml: | api_server: - default_max_num_query_results: {{ .Values.clpConfig.api_server.default_max_num_query_results }} + default_max_num_query_results: {{ + .Values.clpConfig.api_server.default_max_num_query_results }} host: "localhost" port: 3001 query_job_polling: @@ -19,15 +20,19 @@ data: storage: directory: "/var/data/archives" type: "fs" - target_archive_size: {{ .Values.clpConfig.archive_output.target_archive_size }} - target_dictionaries_size: {{ .Values.clpConfig.archive_output.target_dictionaries_size }} - target_encoded_file_size: {{ .Values.clpConfig.archive_output.target_encoded_file_size }} - target_segment_size: {{ .Values.clpConfig.archive_output.target_segment_size }} + target_archive_size: {{ .Values.clpConfig.archive_output.target_archive_size | int }} + target_dictionaries_size: {{ .Values.clpConfig.archive_output.target_dictionaries_size + | int }} + target_encoded_file_size: {{ .Values.clpConfig.archive_output.target_encoded_file_size + | int }} + target_segment_size: {{ .Values.clpConfig.archive_output.target_segment_size + | int }} aws_config_directory: {{ .Values.clpConfig.aws_config_directory }} compression_scheduler: jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} - max_concurrent_tasks_per_job: {{ .Values.clpConfig.compression_scheduler.max_concurrent_tasks_per_job }} + max_concurrent_tasks_per_job: {{ + .Values.clpConfig.compression_scheduler.max_concurrent_tasks_per_job }} compression_worker: logging_level: {{ .Values.clpConfig.compression_worker.logging_level | quote }} data_directory: "/var/data" @@ -57,7 +62,8 @@ data: host: {{ include "clp.fullname" . }}-query-scheduler jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} - num_archives_to_search_per_sub_job: {{ .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job }} + num_archives_to_search_per_sub_job: {{ + .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job }} port: 7000 query_worker: logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }} @@ -84,13 +90,14 @@ data: storage: directory: "/var/data/streams" type: "fs" - target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size }} + target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }} tmp_directory: "/var/tmp" webui: host: "localhost" port: 4000 rate_limit: {{ .Values.clpConfig.webui.rate_limit }} - results_metadata_collection_name: {{ .Values.clpConfig.webui.results_metadata_collection_name | quote }} + results_metadata_collection_name: {{ .Values.clpConfig.webui.results_metadata_collection_name + | quote }} mysql-logging.cnf: | [mysqld] @@ -125,7 +132,8 @@ data: "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, "LogsInputType": {{ .Values.clpConfig.logs_input.type | quote }}, - "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, + "MongoDbSearchResultsMetadataCollectionName": + {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, "SqlDbClpArchivesTableName": "clp_archives", "SqlDbClpDatasetsTableName": "clp_datasets", "SqlDbClpFilesTableName": "clp_files", @@ -143,20 +151,26 @@ data: "MongoDbHost": "{{ include "clp.fullname" . }}-results-cache", "MongoDbPort": 27017, "MongoDbName": {{ .Values.clpConfig.results_cache.db_name | quote }}, - "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, + "MongoDbSearchResultsMetadataCollectionName": + {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, "MongoDbStreamFilesCollectionName": "stream-files", "ClientDir": "/opt/clp/var/www/webui/client", "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer", "StreamFilesDir": "/var/data/streams", - "StreamTargetUncompressedSize": {{ .Values.clpConfig.stream_output.target_uncompressed_size }}, + "StreamTargetUncompressedSize": + {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }}, "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level }}, - "ArchiveOutputTargetArchiveSize": {{ .Values.clpConfig.archive_output.target_archive_size }}, - "ArchiveOutputTargetDictionariesSize": {{ .Values.clpConfig.archive_output.target_dictionaries_size }}, - "ArchiveOutputTargetEncodedFileSize": {{ .Values.clpConfig.archive_output.target_encoded_file_size }}, - "ArchiveOutputTargetSegmentSize": {{ .Values.clpConfig.archive_output.target_segment_size }}, + "ArchiveOutputTargetArchiveSize": + {{ .Values.clpConfig.archive_output.target_archive_size | int }}, + "ArchiveOutputTargetDictionariesSize": + {{ .Values.clpConfig.archive_output.target_dictionaries_size | int }}, + "ArchiveOutputTargetEncodedFileSize": + {{ .Values.clpConfig.archive_output.target_encoded_file_size | int }}, + "ArchiveOutputTargetSegmentSize": + {{ .Values.clpConfig.archive_output.target_segment_size | int }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, "PrestoHost": null, From 0a8707df1fa3c6040ad9c02f02050d31f3cf4623 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 15:19:09 -0500 Subject: [PATCH 097/149] feat(helm): Add conditional storage_engine configuration to configmap --- tools/deployment/package-helm/templates/configmap.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index e204cfdb97..8830c2098a 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -6,6 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} data: clp-config.yaml: | + {{- if eq .Values.clpConfig.package.storage_engine "clp-s" }} api_server: default_max_num_query_results: {{ .Values.clpConfig.api_server.default_max_num_query_results }} @@ -14,6 +15,9 @@ data: query_job_polling: initial_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.initial_backoff_ms }} max_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.max_backoff_ms }} + {{- else }} + api_server: null + {{- end }} archive_output: compression_level: {{ .Values.clpConfig.archive_output.compression_level }} retention_period: {{ .Values.clpConfig.archive_output.retention_period }} From eeb0fc290346454cac5479dfcf40a44415eced94 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 15:41:11 -0500 Subject: [PATCH 098/149] fix(helm): Update reducer base_port to a fixed value in configmap and reducer-service --- tools/deployment/package-helm/templates/configmap.yaml | 2 +- tools/deployment/package-helm/templates/reducer-service.yaml | 4 ++-- tools/deployment/package-helm/values.yaml | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 8830c2098a..d595a36acd 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -80,7 +80,7 @@ data: port: 6379 query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} reducer: - base_port: {{ .Values.clpConfig.reducer.base_port }} + base_port: 14009 host: {{ include "clp.fullname" . }}-reducer logging_level: {{ .Values.clpConfig.reducer.logging_level | quote }} upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval }} diff --git a/tools/deployment/package-helm/templates/reducer-service.yaml b/tools/deployment/package-helm/templates/reducer-service.yaml index 5c3fc6d7b1..8f0ee60d8d 100644 --- a/tools/deployment/package-helm/templates/reducer-service.yaml +++ b/tools/deployment/package-helm/templates/reducer-service.yaml @@ -13,6 +13,6 @@ spec: ports: {{- range $i := until (int .Values.workerConcurrency) }} - name: "reducer-{{ $i }}" - port: {{ add $.Values.clpConfig.reducer.base_port $i }} - targetPort: {{ add $.Values.clpConfig.reducer.base_port $i }} + port: {{ add 14009 $i }} + targetPort: {{ add 14009 $i }} {{- end }} diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 6ee186ff1e..8172f14d65 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -57,7 +57,6 @@ clpConfig: compression_backend_database: 1 reducer: - base_port: 14009 logging_level: "INFO" upsert_interval: 100 # milliseconds From 696ea80ee543a249808cea95549c1b205e18c234 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 15:45:36 -0500 Subject: [PATCH 099/149] fix(helm): Replace hardcoded sample file copy with dynamic download and extraction in test script --- tools/deployment/package-helm/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 7c8a9bdf7c..d5a1689a12 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -13,6 +13,8 @@ mkdir -p /tmp/clp/var/data/{archives,staged-archives,staged-streams,streams} mkdir -p /tmp/clp/var/log/user mkdir -p /tmp/clp/var/tmp mkdir -p /tmp/clp/samples +wget -O - https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1 \ + | tar xz -C /tmp/clp/samples & cat < Date: Mon, 8 Dec 2025 16:10:08 -0500 Subject: [PATCH 100/149] remove compression-scheduler-service.yaml --- .../templates/compression-scheduler-service.yaml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 tools/deployment/package-helm/templates/compression-scheduler-service.yaml diff --git a/tools/deployment/package-helm/templates/compression-scheduler-service.yaml b/tools/deployment/package-helm/templates/compression-scheduler-service.yaml deleted file mode 100644 index 605daf9055..0000000000 --- a/tools/deployment/package-helm/templates/compression-scheduler-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: "v1" -kind: "Service" -metadata: - name: {{ include "clp.fullname" . }}-compression-scheduler - labels: - {{- include "clp.labels" . | nindent 4 }} - app.kubernetes.io/component: "compression-scheduler" -spec: - clusterIP: "None" - selector: - {{- include "clp.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "compression-scheduler" - ports: - - port: 7000 - targetPort: 7000 From 7bd5558b5d915766ebe96f13c7cd993be4e8c752 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 8 Dec 2025 16:10:21 -0500 Subject: [PATCH 101/149] Download and extract sample datasets in the background during test setup --- tools/deployment/package-helm/test.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index d5a1689a12..17ec02454f 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -13,8 +13,11 @@ mkdir -p /tmp/clp/var/data/{archives,staged-archives,staged-streams,streams} mkdir -p /tmp/clp/var/log/user mkdir -p /tmp/clp/var/tmp mkdir -p /tmp/clp/samples + +# Download sample datasets in the background wget -O - https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1 \ | tar xz -C /tmp/clp/samples & +SAMPLE_DOWNLOAD_PID=$! cat < Date: Mon, 8 Dec 2025 16:21:40 -0500 Subject: [PATCH 102/149] feat(test): Add hostPath mount for /home in test setup --- tools/deployment/package-helm/test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 17ec02454f..5e9b9fbf67 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -25,6 +25,8 @@ cat < Date: Tue, 9 Dec 2025 03:48:28 -0500 Subject: [PATCH 103/149] fxi: make Web UI rate limit configurable in Helm chart --- tools/deployment/package-helm/templates/webui-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 4fc2f28e7d..97ad133ac6 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -67,7 +67,7 @@ spec: - name: "PORT" value: "4000" - name: "RATE_LIMIT" - value: "1000" + value: {{ .Values.clpConfig.webui.rate_limit | quote }} ports: - name: "webui" containerPort: 4000 From 7d6cf4fe2277c77e081ddaffffed582748a4a321 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Dec 2025 08:19:42 -0500 Subject: [PATCH 104/149] feat(helm): Add support for root database credentials in Helm templates. --- .../package-helm/templates/database-secret.yaml | 2 ++ .../package-helm/templates/database-statefulset.yaml | 2 +- .../package-helm/templates/db-table-creator-job.yaml | 10 ++++++++++ tools/deployment/package-helm/values.yaml | 2 ++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/database-secret.yaml b/tools/deployment/package-helm/templates/database-secret.yaml index 64235ff4a7..5439e66bd6 100644 --- a/tools/deployment/package-helm/templates/database-secret.yaml +++ b/tools/deployment/package-helm/templates/database-secret.yaml @@ -9,3 +9,5 @@ type: "Opaque" stringData: username: {{ .Values.credentials.database.username | quote }} password: {{ .Values.credentials.database.password | quote }} + root_username: {{ .Values.credentials.database.root_username | quote }} + root_password: {{ .Values.credentials.database.root_password | quote }} diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 39fefe81d2..47801e20f7 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -48,7 +48,7 @@ spec: valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "root_password" ports: - name: "database" containerPort: 3306 diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index a3ece1f23e..bc600db429 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -44,6 +44,16 @@ spec: secretKeyRef: name: {{ include "clp.fullname" . }}-database key: "password" + - name: "CLP_DB_ROOT_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "root_username" + - name: "CLP_DB_ROOT_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "root_password" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 8172f14d65..20803830fd 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -167,6 +167,8 @@ credentials: database: username: "clp-user" password: "pass" + root_username: "root" + root_password: "root-pass" queue: username: "clp-user" From 8f695d95705242f1a9c95f926cc0f1b4eebecdb2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 11 Dec 2025 20:22:26 -0500 Subject: [PATCH 105/149] feat(helm): Refactor init containers to use shared `clp.waitFor` helper and add job-watcher role with service account --- .../package-helm/templates/_helpers.tpl | 25 ++++++++++++++ .../templates/api-server-deployment.yaml | 23 ++----------- .../compression-scheduler-deployment.yaml | 34 +++---------------- .../compression-worker-deployment.yaml | 23 ++----------- .../templates/db-table-creator-job.yaml | 12 ++----- .../garbage-collector-deployment.yaml | 23 ++----------- .../templates/job-watcher-role.yaml | 13 +++++++ .../templates/job-watcher-rolebinding.yaml | 14 ++++++++ .../templates/job-watcher-serviceaccount.yaml | 6 ++++ .../templates/mcp-server-deployment.yaml | 23 ++----------- .../templates/query-scheduler-deployment.yaml | 34 +++---------------- .../templates/query-worker-deployment.yaml | 23 ++----------- .../templates/reducer-deployment.yaml | 23 ++----------- .../results-cache-indices-creator-job.yaml | 12 ++----- .../templates/webui-deployment.yaml | 23 ++----------- 15 files changed, 91 insertions(+), 220 deletions(-) create mode 100644 tools/deployment/package-helm/templates/job-watcher-role.yaml create mode 100644 tools/deployment/package-helm/templates/job-watcher-rolebinding.yaml create mode 100644 tools/deployment/package-helm/templates/job-watcher-serviceaccount.yaml diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index c0f7b4de12..d4de98d0e1 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -163,3 +163,28 @@ spec: requests: storage: {{ .capacity }} {{- end }} + +{{/* +Creates an initContainer that waits for a Kubernetes resource to be ready. + +@param {object} root Root template context +@param {string} type Resource type: "service" (waits for pod readiness) or "job" (waits for completion) +@param {string} name For "service": component name (used for pod label selector) + For "job": job name suffix (appended to fullname) +@return {string} YAML-formatted initContainer definition +*/}} +{{- define "clp.waitFor" -}} +- name: "wait-for-{{ .name }}" + image: "bitnami/kubectl:latest" + command: [ + "kubectl", "wait", + {{- if eq .type "service" }} + "--for=condition=ready", + "pod", "--selector", "app.kubernetes.io/component={{ .name }}", + {{- else if eq .type "job" }} + "--for=condition=complete", + "job/{{ include "clp.fullname" .root }}-{{ .name }}", + {{- end }} + "--timeout=300s" + ] +{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 7a97719902..562933dcc3 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -17,32 +17,15 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "api-server" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done - - name: "wait-for-results-cache" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-results-cache 27017; - do echo Waiting for results-cache to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} containers: - name: "api-server" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 1de5e362ea..fa71e57235 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -17,42 +17,16 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "compression-scheduler" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 300 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done - - name: "wait-for-queue" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-queue 5672; - do echo Waiting for queue to be ready...; - sleep 2; - done - - name: "wait-for-redis" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-redis 6379; - do echo Waiting for redis to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} containers: - name: "compression-scheduler" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 92db721a17..9f3f17a913 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -17,32 +17,15 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "compression-worker" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-queue" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-queue 5672; - do echo Waiting for queue to be ready...; - sleep 2; - done - - name: "wait-for-redis" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-redis 6379; - do echo Waiting for redis to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} containers: - name: "compression-worker" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index bc600db429..8772c56fd8 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -13,22 +13,14 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "db-table-creator" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher restartPolicy: "OnFailure" securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "database") | nindent 8 }} containers: - name: "db-table-creator" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index a4fcaab0cf..157bcbee0e 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -17,32 +17,15 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "garbage-collector" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done - - name: "wait-for-results-cache" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-results-cache 27017; - do echo Waiting for results-cache to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} containers: - name: "garbage-collector" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/job-watcher-role.yaml b/tools/deployment/package-helm/templates/job-watcher-role.yaml new file mode 100644 index 0000000000..7a19885ecb --- /dev/null +++ b/tools/deployment/package-helm/templates/job-watcher-role.yaml @@ -0,0 +1,13 @@ +apiVersion: "rbac.authorization.k8s.io/v1" +kind: "Role" +metadata: + name: {{ include "clp.fullname" . }}-job-reader + labels: + {{- include "clp.labels" . | nindent 4 }} +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] diff --git a/tools/deployment/package-helm/templates/job-watcher-rolebinding.yaml b/tools/deployment/package-helm/templates/job-watcher-rolebinding.yaml new file mode 100644 index 0000000000..78a61624b0 --- /dev/null +++ b/tools/deployment/package-helm/templates/job-watcher-rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: "rbac.authorization.k8s.io/v1" +kind: "RoleBinding" +metadata: + name: {{ include "clp.fullname" . }}-job-watcher-binding + labels: + {{- include "clp.labels" . | nindent 4 }} +subjects: + - kind: "ServiceAccount" + name: {{ include "clp.fullname" . }}-job-watcher + namespace: {{ .Release.Namespace }} +roleRef: + kind: "Role" + name: {{ include "clp.fullname" . }}-job-reader + apiGroup: "rbac.authorization.k8s.io" diff --git a/tools/deployment/package-helm/templates/job-watcher-serviceaccount.yaml b/tools/deployment/package-helm/templates/job-watcher-serviceaccount.yaml new file mode 100644 index 0000000000..54f305eeb3 --- /dev/null +++ b/tools/deployment/package-helm/templates/job-watcher-serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: "v1" +kind: "ServiceAccount" +metadata: + name: {{ include "clp.fullname" . }}-job-watcher + labels: + {{- include "clp.labels" . | nindent 4 }} diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index bfe5273352..2efa30a4c0 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -18,32 +18,15 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "mcp-server" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done - - name: "wait-for-results-cache" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-results-cache 27017; - do echo Waiting for results-cache to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} containers: - name: "mcp-server" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index b6c6a2990e..7ee62c3d8d 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -17,6 +17,7 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "query-scheduler" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher hostname: {{ include "clp.fullname" . }}-query-scheduler subdomain: {{ include "clp.fullname" . }}-query-scheduler terminationGracePeriodSeconds: 10 @@ -25,36 +26,9 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done - - name: "wait-for-queue" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-queue 5672; - do echo Waiting for queue to be ready...; - sleep 2; - done - - name: "wait-for-redis" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-redis 6379; - do echo Waiting for redis to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} containers: - name: "query-scheduler" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index ba395d4d29..b461996427 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -17,32 +17,15 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "query-worker" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-queue" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-queue 5672; - do echo Waiting for queue to be ready...; - sleep 2; - done - - name: "wait-for-redis" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-redis 6379; - do echo Waiting for redis to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} containers: - name: "query-worker" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index edde1c8fcd..869eb36fe4 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -17,6 +17,7 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "reducer" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher hostname: {{ include "clp.fullname" . }}-reducer subdomain: {{ include "clp.fullname" . }}-reducer terminationGracePeriodSeconds: 10 @@ -25,26 +26,8 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-query-scheduler" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-query-scheduler 7000; - do echo Waiting for query-scheduler to be ready...; - sleep 2; - done - - name: "wait-for-results-cache" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-results-cache 27017; - do echo Waiting for results-cache to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "query-scheduler") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} containers: - name: "reducer" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index 51c2b822db..f2cb81d75f 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -13,22 +13,14 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "results-cache-indices-creator" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher restartPolicy: "OnFailure" securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-results-cache" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-results-cache 27017; - do echo Waiting for results-cache to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "results-cache") | nindent 8 }} containers: - name: "results-cache-indices-creator" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 97ad133ac6..6ca09c1d38 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -17,32 +17,15 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "webui" spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - - name: "wait-for-database" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-database 3306; - do echo Waiting for database to be ready...; - sleep 2; - done - - name: "wait-for-results-cache" - image: "busybox:latest" - command: - - "sh" - - "-c" - - >- - until nc -z {{ include "clp.fullname" . }}-results-cache 27017; - do echo Waiting for results-cache to be ready...; - sleep 2; - done + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} containers: - name: "webui" image: "{{ include "clp.image.ref" . }}" From b231898d1b36eea6de8b7abd116eb933bc006bbc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 12 Dec 2025 02:29:08 -0500 Subject: [PATCH 106/149] feat(helm): Refactor PersistentVolume and PersistentVolumeClaim templates with shared helpers and consistent naming conventions --- .../package-helm/templates/_helpers.tpl | 25 ++++++++-- .../templates/api-server-deployment.yaml | 8 ++-- .../templates/api-server-logs-pv.yaml | 4 +- .../templates/api-server-logs-pvc.yaml | 3 +- .../package-helm/templates/archives-pv.yaml | 9 ---- .../compression-scheduler-deployment.yaml | 24 ++++++---- .../compression-scheduler-logs-pv.yaml | 4 +- .../compression-scheduler-logs-pvc.yaml | 3 +- ...> compression-scheduler-user-logs-pv.yaml} | 4 +- ... compression-scheduler-user-logs-pvc.yaml} | 3 +- .../compression-worker-deployment.yaml | 48 +++++++++++-------- .../templates/compression-worker-logs-pv.yaml | 4 +- .../compression-worker-logs-pvc.yaml | 3 +- ...compression-worker-staged-archives-pv.yaml | 6 +-- ...ompression-worker-staged-archives-pvc.yaml | 3 +- ...pv.yaml => compression-worker-tmp-pv.yaml} | 4 +- ...c.yaml => compression-worker-tmp-pvc.yaml} | 3 +- .../templates/database-data-pv.yaml | 2 +- .../templates/database-logs-pv.yaml | 2 +- .../garbage-collector-deployment.yaml | 28 ++++++----- .../templates/garbage-collector-logs-pv.yaml | 4 +- .../templates/garbage-collector-logs-pvc.yaml | 3 +- .../templates/mcp-server-deployment.yaml | 8 ++-- .../templates/mcp-server-logs-pv.yaml | 6 ++- .../templates/mcp-server-logs-pvc.yaml | 3 +- .../templates/query-scheduler-deployment.yaml | 8 ++-- .../templates/query-scheduler-logs-pv.yaml | 4 +- .../templates/query-scheduler-logs-pvc.yaml | 3 +- .../templates/query-worker-deployment.yaml | 40 +++++++++------- .../templates/query-worker-logs-pv.yaml | 4 +- .../templates/query-worker-logs-pvc.yaml | 3 +- .../query-worker-staged-streams-pv.yaml | 6 +-- .../query-worker-staged-streams-pvc.yaml | 3 +- .../package-helm/templates/queue-logs-pv.yaml | 2 +- .../package-helm/templates/redis-data-pv.yaml | 2 +- .../package-helm/templates/redis-logs-pv.yaml | 2 +- .../templates/reducer-deployment.yaml | 8 ++-- .../templates/reducer-logs-pv.yaml | 4 +- .../templates/reducer-logs-pvc.yaml | 3 +- .../templates/results-cache-data-pv.yaml | 2 +- .../templates/results-cache-logs-pv.yaml | 2 +- .../templates/shared-data-archives-pv.yaml | 9 ++++ ...pvc.yaml => shared-data-archives-pvc.yaml} | 3 +- .../templates/shared-data-streams-pv.yaml | 9 ++++ ...-pvc.yaml => shared-data-streams-pvc.yaml} | 3 +- .../package-helm/templates/streams-pv.yaml | 9 ---- .../templates/webui-deployment.yaml | 10 ++-- 47 files changed, 210 insertions(+), 143 deletions(-) delete mode 100644 tools/deployment/package-helm/templates/archives-pv.yaml rename tools/deployment/package-helm/templates/{user-logs-pv.yaml => compression-scheduler-user-logs-pv.yaml} (70%) rename tools/deployment/package-helm/templates/{tmp-pvc.yaml => compression-scheduler-user-logs-pvc.yaml} (64%) rename tools/deployment/package-helm/templates/{tmp-pv.yaml => compression-worker-tmp-pv.yaml} (71%) rename tools/deployment/package-helm/templates/{user-logs-pvc.yaml => compression-worker-tmp-pvc.yaml} (68%) create mode 100644 tools/deployment/package-helm/templates/shared-data-archives-pv.yaml rename tools/deployment/package-helm/templates/{archives-pvc.yaml => shared-data-archives-pvc.yaml} (69%) create mode 100644 tools/deployment/package-helm/templates/shared-data-streams-pv.yaml rename tools/deployment/package-helm/templates/{streams-pvc.yaml => shared-data-streams-pvc.yaml} (69%) delete mode 100644 tools/deployment/package-helm/templates/streams-pv.yaml diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index d4de98d0e1..b856e3a1bd 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -98,8 +98,8 @@ failureThreshold: 3 Creates a local PersistentVolume. @param {object} root Root template context -@param {string} name PV name -@param {string} component Component label +@param {string} component Component category (e.g., "shared-data", "database") +@param {string} name Storage name (e.g., "archives", "logs", "data") @param {string} nodeRole Node role for affinity. Targets nodes with label "node-role.kubernetes.io/". Always falls back to "node-role.kubernetes.io/control-plane" @@ -112,7 +112,7 @@ Creates a local PersistentVolume. apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ .name }} + name: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} labels: {{- include "clp.labels" .root | nindent 4 }} app.kubernetes.io/component: {{ .component | quote }} @@ -139,7 +139,8 @@ spec: Creates a PersistentVolumeClaim for the given component. @param {object} root Root template context -@param {string} component Component label +@param {string} component Component category (e.g., "shared-data", "database") +@param {string} name Storage name (e.g., "archives", "logs", "data") @param {string} capacity Storage capacity @param {string[]} accessModes Access modes @return {string} YAML-formatted PersistentVolumeClaim resource @@ -148,7 +149,7 @@ Creates a PersistentVolumeClaim for the given component. apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp.fullname" .root }}-{{ .component }} + name: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} labels: {{- include "clp.labels" .root | nindent 4 }} app.kubernetes.io/component: {{ .component | quote }} @@ -164,6 +165,20 @@ spec: storage: {{ .capacity }} {{- end }} +{{/* +Creates a volume definition that references a PersistentVolumeClaim. + +@param {object} root Root template context +@param {string} component Component category (e.g., "shared-data", "database") +@param {string} name Storage name (e.g., "archives", "logs", "data") +@return {string} YAML-formatted volume definition +*/}} +{{- define "clp.pvcVolume" -}} +- name: {{ printf "%s-%s" .component .name | quote }} + persistentVolumeClaim: + claimName: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} +{{- end }} + {{/* Creates an initContainer that waits for a Kubernetes resource to be ready. diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 562933dcc3..2e5f8a8809 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -70,9 +70,11 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} httpGet: *api-server-health-check volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "api-server" + "name" "logs" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "api-server-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-api-server-logs diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index 74d372325c..87b0b609e8 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-api-server-logs" (include "clp.fullname" .)) - "component" "api-server-logs" + "component" "api-server" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml index 72d6f4702c..d63e3dbb3f 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "api-server-logs" + "component" "api-server" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/archives-pv.yaml b/tools/deployment/package-helm/templates/archives-pv.yaml deleted file mode 100644 index d1a043d64f..0000000000 --- a/tools/deployment/package-helm/templates/archives-pv.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- include "clp.createLocalPv" (dict - "root" . - "name" (printf "%s-archives" (include "clp.fullname" .)) - "component" "archives" - "nodeRole" "control-plane" - "capacity" "50Gi" - "accessModes" (list "ReadWriteMany") - "hostPath" .Values.clpConfig.archive_output.storage.directory -) }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index fa71e57235..634a859582 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -53,14 +53,14 @@ spec: - name: "RESULT_BACKEND" value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: + - name: "compression-scheduler-logs" + mountPath: "/var/log/compression_scheduler" + - name: "compression-scheduler-user-logs" + mountPath: "/var/log/user" - name: "config" mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true - - name: "compression-scheduler-logs" - mountPath: "/var/log/compression_scheduler" - - name: "user-logs" - mountPath: "/var/log/user" - name: "logs-input" mountPath: "/mnt/logs" readOnly: true @@ -70,15 +70,19 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "compression-scheduler" + "name" "logs" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "compression-scheduler" + "name" "user-logs" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "compression-scheduler-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-compression-scheduler-logs - - name: "user-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-user-logs - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 48a46f4f8a..3404844121 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-compression-scheduler-logs" (include "clp.fullname" .)) - "component" "compression-scheduler-logs" + "component" "compression-scheduler" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml index 63011569c5..0d19be396a 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-scheduler-logs" + "component" "compression-scheduler" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/user-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml similarity index 70% rename from tools/deployment/package-helm/templates/user-logs-pv.yaml rename to tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml index 9c43c56e00..6653e4290d 100644 --- a/tools/deployment/package-helm/templates/user-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-user-logs" (include "clp.fullname" .)) - "component" "user-logs" + "component" "compression-scheduler" + "name" "user-logs" "nodeRole" "control-plane" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/tmp-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml similarity index 64% rename from tools/deployment/package-helm/templates/tmp-pvc.yaml rename to tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml index abdddcc887..bf56a89311 100644 --- a/tools/deployment/package-helm/templates/tmp-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "tmp" + "component" "compression-scheduler" + "name" "user-logs" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") ) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 9f3f17a913..0bfcd2954b 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -48,21 +48,21 @@ spec: - name: "RESULT_BACKEND" value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: + - name: "compression-worker-logs" + mountPath: "/var/log/compression_worker" + - name: "compression-worker-staged-archives" + mountPath: "/var/data/staged-archives" + - name: "compression-worker-tmp" + mountPath: "/var/tmp" - name: "config" mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true - - name: "compression-worker-logs" - mountPath: "/var/log/compression_worker" - - name: "archives" - mountPath: "/var/data/archives" - name: "logs-input" mountPath: "/mnt/logs" readOnly: true - - name: "staged-archives" - mountPath: "/var/data/staged-archives" - - name: "tmp" - mountPath: "/var/tmp" + - name: "shared-data-archives" + mountPath: "/var/data/archives" command: [ "python3", "-u", "/opt/clp/lib/python3/site-packages/bin/celery", @@ -75,22 +75,30 @@ spec: "-n", "compression-worker" ] volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "compression-worker" + "name" "logs" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "compression-worker" + "name" "staged-archives" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "compression-worker" + "name" "tmp" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "shared-data" + "name" "archives" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "compression-worker-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-compression-worker-logs - - name: "archives" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-archives - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" type: "Directory" - - name: "staged-archives" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-staged-archives - - name: "tmp" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-tmp diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index 3c451126d2..98e58cab64 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-compression-worker-logs" (include "clp.fullname" .)) - "component" "compression-worker-logs" + "component" "compression-worker" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml index 45b11b1fad..b6953b0393 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-worker-logs" + "component" "compression-worker" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index 11dcbd1d8a..b79ea5d0c1 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -1,9 +1,9 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-staged-archives" (include "clp.fullname" .)) - "component" "staged-archives" + "component" "compression-worker" + "name" "staged-archives" "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") - "hostPath" "/tmp/clp/var/data/staged-archives" + "hostPath" (printf "%s/staged-archives" .Values.clpConfig.data_directory) ) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml index 77410397a8..77f0324264 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "staged-archives" + "component" "compression-worker" + "name" "staged-archives" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") ) }} diff --git a/tools/deployment/package-helm/templates/tmp-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml similarity index 71% rename from tools/deployment/package-helm/templates/tmp-pv.yaml rename to tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml index 1add0502bd..18cd4a68b7 100644 --- a/tools/deployment/package-helm/templates/tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-tmp" (include "clp.fullname" .)) - "component" "tmp" + "component" "compression-worker" + "name" "tmp" "nodeRole" "control-plane" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/user-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml similarity index 68% rename from tools/deployment/package-helm/templates/user-logs-pvc.yaml rename to tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml index 2fb0688dc2..51b7bc0d60 100644 --- a/tools/deployment/package-helm/templates/user-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "user-logs" + "component" "compression-worker" + "name" "tmp" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") ) }} diff --git a/tools/deployment/package-helm/templates/database-data-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml index 98beaaf866..1515a853c8 100644 --- a/tools/deployment/package-helm/templates/database-data-pv.yaml +++ b/tools/deployment/package-helm/templates/database-data-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-database-data" (include "clp.fullname" .)) "component" "database" + "name" "data" "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml index 27ad28d197..ea454a8676 100644 --- a/tools/deployment/package-helm/templates/database-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-database-logs" (include "clp.fullname" .)) "component" "database" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 157bcbee0e..bb8022b0ed 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -56,9 +56,9 @@ spec: readOnly: true - name: "garbage-collector-logs" mountPath: "/var/log/garbage_collector" - - name: "archives" + - name: "shared-data-archives" mountPath: "/var/data/archives" - - name: "streams" + - name: "shared-data-streams" mountPath: "/var/data/streams" command: [ "python3", "-u", @@ -66,15 +66,21 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "garbage-collector" + "name" "logs" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "shared-data" + "name" "archives" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "shared-data" + "name" "streams" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "garbage-collector-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-garbage-collector-logs - - name: "archives" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-archives - - name: "streams" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-streams diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index 75f3a6a3e5..3f8ece5a41 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-garbage-collector-logs" (include "clp.fullname" .)) - "component" "garbage-collector-logs" + "component" "garbage-collector" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml index 714e35bd94..f84222d9b8 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "garbage-collector-logs" + "component" "garbage-collector" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 2efa30a4c0..fe355403f7 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -74,10 +74,12 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} httpGet: *mcp-server-health-check volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "mcp-server" + "name" "logs" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "mcp-server-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-mcp-server-logs {{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index 01bbf18ab7..fb1a6a94a5 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -1,9 +1,11 @@ +{{- if .Values.clpConfig.mcp_server }} {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-mcp-server-logs" (include "clp.fullname" .)) - "component" "mcp-server-logs" + "component" "mcp-server" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/mcp-server" .Values.clpConfig.logs_directory) ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml index f648fa9ccd..61219f973f 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml @@ -1,7 +1,8 @@ {{- if .Values.clpConfig.mcp_server }} {{- include "clp.createPvc" (dict "root" . - "component" "mcp-server-logs" + "component" "mcp-server" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 7ee62c3d8d..ed6630d9c3 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -70,9 +70,11 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "query-scheduler" + "name" "logs" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "query-scheduler-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-query-scheduler-logs diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index d7b62604c4..826a98d3ac 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-query-scheduler-logs" (include "clp.fullname" .)) - "component" "query-scheduler-logs" + "component" "query-scheduler" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml index bf9794ada9..559d3518ff 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "query-scheduler-logs" + "component" "query-scheduler" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index b461996427..150b2e65fb 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -54,11 +54,11 @@ spec: readOnly: true - name: "query-worker-logs" mountPath: "/var/log/query_worker" - - name: "archives" - mountPath: "/var/data/archives" - - name: "staged-streams" + - name: "query-worker-staged-streams" mountPath: "/var/data/staged-streams" - - name: "streams" + - name: "shared-data-archives" + mountPath: "/var/data/archives" + - name: "shared-data-streams" mountPath: "/var/data/streams" command: [ "python3", "-u", @@ -72,18 +72,26 @@ spec: "-n", "query-worker" ] volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "query-worker" + "name" "logs" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "query-worker" + "name" "staged-streams" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "shared-data" + "name" "archives" + ) | nindent 8 }} + {{- include "clp.pvcVolume" (dict + "root" . + "component" "shared-data" + "name" "streams" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "query-worker-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-query-worker-logs - - name: "archives" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-archives - - name: "staged-streams" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-staged-streams - - name: "streams" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-streams diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index e92ee8b03f..6e50f43ea1 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-query-worker-logs" (include "clp.fullname" .)) - "component" "query-worker-logs" + "component" "query-worker" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml index bfbc07a9d1..eb2f41e916 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "query-worker-logs" + "component" "query-worker" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index 95a641be53..02b95378a6 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -1,9 +1,9 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-staged-streams" (include "clp.fullname" .)) - "component" "staged-streams" + "component" "query-worker" + "name" "staged-streams" "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") - "hostPath" "/tmp/clp/var/data/staged-streams" + "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) ) }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml index 195d94319e..242fe1ff4a 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "staged-streams" + "component" "query-worker" + "name" "staged-streams" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") ) }} diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml index cbc4fa3008..a24808b632 100644 --- a/tools/deployment/package-helm/templates/queue-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-queue-logs" (include "clp.fullname" .)) "component" "queue" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml index 96fa611225..d5be52b6b1 100644 --- a/tools/deployment/package-helm/templates/redis-data-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-redis-data" (include "clp.fullname" .)) "component" "redis" + "name" "data" "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml index b3308cbfc9..755484bacd 100644 --- a/tools/deployment/package-helm/templates/redis-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-redis-logs" (include "clp.fullname" .)) "component" "redis" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index 869eb36fe4..4ed9d77e5d 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -56,9 +56,11 @@ spec: "--upsert-interval", "{{ .Values.clpConfig.reducer.upsert_interval }}" ] volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "reducer" + "name" "logs" + ) | nindent 8 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - - name: "reducer-logs" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-reducer-logs diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index 12c67a778f..1d12c20d16 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-reducer-logs" (include "clp.fullname" .)) - "component" "reducer-logs" + "component" "reducer" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml index cd4eec1792..78dbd64288 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "reducer-logs" + "component" "reducer" + "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml index 8c67722348..c55a858b39 100644 --- a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-results-cache-data" (include "clp.fullname" .)) "component" "results-cache" + "name" "data" "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index 2050d3a033..77a99383a5 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -1,7 +1,7 @@ {{- include "clp.createLocalPv" (dict "root" . - "name" (printf "%s-results-cache-logs" (include "clp.fullname" .)) "component" "results-cache" + "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml new file mode 100644 index 0000000000..9373c85fcf --- /dev/null +++ b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "component" "shared-data" + "name" "archives" + "nodeRole" "control-plane" + "capacity" "50Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" (printf "%s/archives" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/archives-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml similarity index 69% rename from tools/deployment/package-helm/templates/archives-pvc.yaml rename to tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml index c5b1bac44f..fb53ebd225 100644 --- a/tools/deployment/package-helm/templates/archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "archives" + "component" "shared-data" + "name" "archives" "capacity" "50Gi" "accessModes" (list "ReadWriteMany") ) }} diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml new file mode 100644 index 0000000000..d96a7be523 --- /dev/null +++ b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml @@ -0,0 +1,9 @@ +{{- include "clp.createLocalPv" (dict + "root" . + "component" "shared-data" + "name" "streams" + "nodeRole" "control-plane" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" (printf "%s/streams" .Values.clpConfig.data_directory) +) }} diff --git a/tools/deployment/package-helm/templates/streams-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml similarity index 69% rename from tools/deployment/package-helm/templates/streams-pvc.yaml rename to tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml index bd754e6c42..4f4e6ea9a0 100644 --- a/tools/deployment/package-helm/templates/streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml @@ -1,6 +1,7 @@ {{- include "clp.createPvc" (dict "root" . - "component" "streams" + "component" "shared-data" + "name" "streams" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") ) }} diff --git a/tools/deployment/package-helm/templates/streams-pv.yaml b/tools/deployment/package-helm/templates/streams-pv.yaml deleted file mode 100644 index df211cf22f..0000000000 --- a/tools/deployment/package-helm/templates/streams-pv.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- include "clp.createLocalPv" (dict - "root" . - "name" (printf "%s-streams" (include "clp.fullname" .)) - "component" "streams" - "nodeRole" "control-plane" - "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") - "hostPath" .Values.clpConfig.stream_output.storage.directory -) }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 6ca09c1d38..f792887afc 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -63,7 +63,7 @@ spec: mountPath: "/opt/clp/var/www/webui/server/dist/settings.json" subPath: "webui-server-settings.json" readOnly: true - - name: "streams" + - name: "shared-data-streams" mountPath: "/var/data/streams" command: [ "/opt/clp/bin/node-22", @@ -77,12 +77,14 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: *webui-health-check volumes: + {{- include "clp.pvcVolume" (dict + "root" . + "component" "shared-data" + "name" "streams" + ) | nindent 8 }} - name: "client-settings" configMap: name: {{ include "clp.fullname" . }}-config - name: "server-settings" configMap: name: {{ include "clp.fullname" . }}-config - - name: "streams" - persistentVolumeClaim: - claimName: {{ include "clp.fullname" . }}-streams From 44b0f17cb41b035d58e7d69c4eb88abeeb735ca9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 12 Dec 2025 02:45:37 -0500 Subject: [PATCH 107/149] feat(helm): Add shared helpers for Celery environment variables and improve readability of templates with consistent formatting --- .../package-helm/templates/_helpers.tpl | 28 +++++++++++++++ .../templates/api-server-deployment.yaml | 12 +++++-- .../compression-scheduler-deployment.yaml | 35 +++++++++++++------ .../compression-worker-deployment.yaml | 9 ++--- .../garbage-collector-deployment.yaml | 12 +++++-- .../templates/mcp-server-deployment.yaml | 12 +++++-- .../templates/query-scheduler-deployment.yaml | 35 +++++++++++++------ .../templates/query-worker-deployment.yaml | 9 ++--- .../templates/reducer-deployment.yaml | 12 +++++-- .../results-cache-indices-creator-job.yaml | 13 +++++-- .../templates/webui-deployment.yaml | 12 +++++-- 11 files changed, 146 insertions(+), 43 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index b856e3a1bd..988bcf65ee 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -179,6 +179,34 @@ Creates a volume definition that references a PersistentVolumeClaim. claimName: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} {{- end }} +{{/* +Creates the BROKER_URL env var for Celery workers. + +@param {object} . Root template context +@return {string} YAML-formatted env var definition +*/}} +{{- define "clp.celeryBrokerUrlEnvVar" -}} +{{- $user := .Values.credentials.queue.username -}} +{{- $pass := .Values.credentials.queue.password -}} +{{- $host := printf "%s-queue" (include "clp.fullname" .) -}} +- name: "BROKER_URL" + value: {{ printf "amqp://%s:%s@%s:5672" $user $pass $host | quote }} +{{- end }} + +{{/* +Creates the RESULT_BACKEND env var for Celery workers. + +@param {object} root Root template context +@param {string} database Redis database number from config +@return {string} YAML-formatted env var definition +*/}} +{{- define "clp.celeryResultBackendEnvVar" -}} +{{- $pass := .root.Values.credentials.redis.password -}} +{{- $host := printf "%s-redis" (include "clp.fullname" .root) -}} +- name: "RESULT_BACKEND" + value: {{ printf "redis://default:%s@%s:6379/%s" $pass $host .database | quote }} +{{- end }} + {{/* Creates an initContainer that waits for a Kubernetes resource to be ready. diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 2e5f8a8809..51a5291a1c 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -24,8 +24,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 8 }} containers: - name: "api-server" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 634a859582..fa7fd55118 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -24,34 +24,47 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "queue" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "redis" + ) | nindent 8 }} containers: - name: "compression-scheduler" image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - - name: "CLP_DB_USER" + {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} + {{- include "clp.celeryResultBackendEnvVar" (dict + "root" . + "database" .Values.clpConfig.redis.compression_backend_database + ) | nindent 12 }} + - name: "CLP_DB_PASS" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "username" - - name: "CLP_DB_PASS" + key: "password" + - name: "CLP_DB_USER" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "username" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} - name: "CLP_LOGS_DIR" value: "/var/log/compression_scheduler" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: - name: "compression-scheduler-logs" mountPath: "/var/log/compression_scheduler" diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 0bfcd2954b..7d9a21524b 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -31,8 +31,11 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" + {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} + {{- include "clp.celeryResultBackendEnvVar" (dict + "root" . + "database" .Values.clpConfig.redis.compression_backend_database + ) | nindent 12 }} - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yaml" - name: "CLP_HOME" @@ -45,8 +48,6 @@ spec: value: "/var/log/compression_worker/worker.log" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.compression_backend_database }}" volumeMounts: - name: "compression-worker-logs" mountPath: "/var/log/compression_worker" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index bb8022b0ed..b5e22ebff2 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -24,8 +24,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 8 }} containers: - name: "garbage-collector" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index fe355403f7..13863f2f60 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -25,8 +25,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 8 }} containers: - name: "mcp-server" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index ed6630d9c3..94d79a28d3 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -26,34 +26,47 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "queue" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "redis" + ) | nindent 8 }} containers: - name: "query-scheduler" image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" - - name: "CLP_DB_USER" + {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} + {{- include "clp.celeryResultBackendEnvVar" (dict + "root" . + "database" .Values.clpConfig.redis.query_backend_database + ) | nindent 12 }} + - name: "CLP_DB_PASS" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "username" - - name: "CLP_DB_PASS" + key: "password" + - name: "CLP_DB_USER" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "username" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} - name: "CLP_LOGS_DIR" value: "/var/log/query_scheduler" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" ports: - name: "query-scheduler" containerPort: 7000 diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 150b2e65fb..5329f3b745 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -31,8 +31,11 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "BROKER_URL" - value: "amqp://{{ .Values.credentials.queue.username }}:{{ .Values.credentials.queue.password }}@{{ include "clp.fullname" . }}-queue:5672" + {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} + {{- include "clp.celeryResultBackendEnvVar" (dict + "root" . + "database" .Values.clpConfig.redis.query_backend_database + ) | nindent 12 }} - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yaml" - name: "CLP_HOME" @@ -45,8 +48,6 @@ spec: value: "/var/log/query_worker/worker.log" - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" - - name: "RESULT_BACKEND" - value: "redis://default:{{ .Values.credentials.redis.password }}@{{ include "clp.fullname" . }}-redis:6379/{{ .Values.clpConfig.redis.query_backend_database }}" volumeMounts: - name: "config" mountPath: "/etc/clp-config.yaml" diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index 4ed9d77e5d..f6ee050a35 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -26,8 +26,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "query-scheduler") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "query-scheduler" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 8 }} containers: - name: "reducer" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index f2cb81d75f..0dcf9ee646 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -20,7 +20,11 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "results-cache") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "results-cache" + ) | nindent 8 }} containers: - name: "results-cache-indices-creator" image: "{{ include "clp.image.ref" . }}" @@ -31,6 +35,9 @@ spec: command: [ "python3", "-u", "-m", "clp_py_utils.initialize-results-cache", - "--uri", "mongodb://{{ include "clp.fullname" . }}-results-cache:27017/{{ .Values.clpConfig.results_cache.db_name }}", - "--stream-collection", {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} + "--uri", + "mongodb://{{ include "clp.fullname" . }}-results-cache:27017/{{ + .Values.clpConfig.results_cache.db_name }}", + "--stream-collection", {{ .Values.clpConfig.results_cache.stream_collection_name + | quote }} ] diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index f792887afc..4f0415556f 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -24,8 +24,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "db-table-creator") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "job" "name" "results-cache-indices-creator") | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 8 }} + {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 8 }} containers: - name: "webui" image: "{{ include "clp.image.ref" . }}" From a9e6d09d5c62a5822a1f9f40dd0813714cfacbc2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 12 Dec 2025 02:54:34 -0500 Subject: [PATCH 108/149] feat(helm): Update hostPath in PersistentVolume templates for consistency --- tools/deployment/package-helm/templates/api-server-logs-pv.yaml | 2 +- .../package-helm/templates/compression-scheduler-logs-pv.yaml | 2 +- .../package-helm/templates/compression-worker-logs-pv.yaml | 2 +- .../package-helm/templates/garbage-collector-logs-pv.yaml | 2 +- tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml | 2 +- .../package-helm/templates/query-scheduler-logs-pv.yaml | 2 +- .../deployment/package-helm/templates/query-worker-logs-pv.yaml | 2 +- .../package-helm/templates/results-cache-data-pv.yaml | 2 +- .../package-helm/templates/results-cache-logs-pv.yaml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index 87b0b609e8..84b44ad906 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/api-server" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/api_server" .Values.clpConfig.logs_directory) ) }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 3404844121..565fc541f1 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/compression-scheduler" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/compression_scheduler" .Values.clpConfig.logs_directory) ) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index 98e58cab64..aa1f3d165e 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/compression-worker" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/compression_worker" .Values.clpConfig.logs_directory) ) }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index 3f8ece5a41..d986b3cf7a 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/garbage-collector" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/garbage_collector" .Values.clpConfig.logs_directory) ) }} diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index fb1a6a94a5..5af6d1c6ed 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -6,6 +6,6 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/mcp-server" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/mcp_server" .Values.clpConfig.logs_directory) ) }} {{- end }} diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index 826a98d3ac..346b250c2c 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/query-scheduler" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/query_scheduler" .Values.clpConfig.logs_directory) ) }} diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index 6e50f43ea1..65f9ff3eed 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/query-worker" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/query_worker" .Values.clpConfig.logs_directory) ) }} diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml index c55a858b39..a4a32223b5 100644 --- a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/results-cache" .Values.clpConfig.data_directory) + "hostPath" (printf "%s/results_cache" .Values.clpConfig.data_directory) ) }} diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index 77a99383a5..9498d0ad65 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -5,5 +5,5 @@ "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/results-cache" .Values.clpConfig.logs_directory) + "hostPath" (printf "%s/results_cache" .Values.clpConfig.logs_directory) ) }} From 92790326fcff6874e78fe6504ebb1069906e1940 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 12 Dec 2025 03:13:39 -0500 Subject: [PATCH 109/149] fix(test): Correct temp directory structure and ensure Redis database value is cast to integer in Helm templates --- tools/deployment/package-helm/templates/_helpers.tpl | 2 +- tools/deployment/package-helm/test.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 988bcf65ee..59d0a84b71 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -204,7 +204,7 @@ Creates the RESULT_BACKEND env var for Celery workers. {{- $pass := .root.Values.credentials.redis.password -}} {{- $host := printf "%s-redis" (include "clp.fullname" .root) -}} - name: "RESULT_BACKEND" - value: {{ printf "redis://default:%s@%s:6379/%s" $pass $host .database | quote }} + value: {{ printf "redis://default:%s@%s:6379/%d" $pass $host (int .database) | quote }} {{- end }} {{/* diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 5e9b9fbf67..8322934ce6 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -8,7 +8,8 @@ set -o pipefail kind delete cluster --name clp-test rm -rf /tmp/clp -mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results-cache,compression-scheduler,compression-worker,query-scheduler,query-worker,reducer,garbage-collector,api-server,mcp-server} +mkdir -p /tmp/clp/var/{data,log}/{database,queue,redis,results_cache} +mkdir -p /tmp/clp/var/log/{compression_scheduler,compression_worker,query_scheduler,query_worker,reducer,garbage_collector,api_server,mcp_server} mkdir -p /tmp/clp/var/data/{archives,staged-archives,staged-streams,streams} mkdir -p /tmp/clp/var/log/user mkdir -p /tmp/clp/var/tmp From 8580c113a0a6461b39532e223627ef9b0aa58cd0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 12 Dec 2025 10:17:36 -0500 Subject: [PATCH 110/149] refactor(helm): Reorder volumeMounts in StatefulSet templates for consistency across components --- .../package-helm/templates/api-server-deployment.yaml | 4 ++-- .../package-helm/templates/database-statefulset.yaml | 8 ++++---- .../package-helm/templates/redis-statefulset.yaml | 8 ++++---- .../package-helm/templates/results-cache-statefulset.yaml | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 51a5291a1c..6571f84b9f 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -57,12 +57,12 @@ spec: - name: "api-server" containerPort: 3001 volumeMounts: + - name: "api-server-logs" + mountPath: "/var/log/api_server" - name: "config" mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true - - name: "api-server-logs" - mountPath: "/var/log/api_server" command: [ "/opt/clp/bin/api_server", "--host", "0.0.0.0", diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 47801e20f7..208dd6295d 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -53,14 +53,14 @@ spec: - name: "database" containerPort: 3306 volumeMounts: - - name: "database-data" - mountPath: "/var/lib/mysql" - - name: "database-logs" - mountPath: "/var/log/mysql" - name: "config" mountPath: "/etc/mysql/conf.d/logging.cnf" subPath: "mysql-logging.cnf" readOnly: true + - name: "database-data" + mountPath: "/var/lib/mysql" + - name: "database-logs" + mountPath: "/var/log/mysql" readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &database-health-check diff --git a/tools/deployment/package-helm/templates/redis-statefulset.yaml b/tools/deployment/package-helm/templates/redis-statefulset.yaml index d7895ee04a..f0c6b760da 100644 --- a/tools/deployment/package-helm/templates/redis-statefulset.yaml +++ b/tools/deployment/package-helm/templates/redis-statefulset.yaml @@ -41,14 +41,14 @@ spec: - name: "redis" containerPort: 6379 volumeMounts: - - name: "redis-data" - mountPath: "/data" - - name: "redis-logs" - mountPath: "/var/log/redis" - name: "config" mountPath: "/etc/redis/redis.conf" subPath: "redis.conf" readOnly: true + - name: "redis-data" + mountPath: "/data" + - name: "redis-logs" + mountPath: "/var/log/redis" readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &redis-health-check diff --git a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml index adbe0cae9b..3245780e9b 100644 --- a/tools/deployment/package-helm/templates/results-cache-statefulset.yaml +++ b/tools/deployment/package-helm/templates/results-cache-statefulset.yaml @@ -34,14 +34,14 @@ spec: - name: "results-cache" containerPort: 27017 volumeMounts: - - name: "results-cache-data" - mountPath: "/data/db" - - name: "results-cache-logs" - mountPath: "/var/log/mongodb" - name: "config" mountPath: "/etc/mongo/mongod.conf" subPath: "mongod.conf" readOnly: true + - name: "results-cache-data" + mountPath: "/data/db" + - name: "results-cache-logs" + mountPath: "/var/log/mongodb" readinessProbe: {{- include "clp.readinessProbeTimings" . | nindent 12 }} exec: &results-cache-health-check From 1c2669281345b6280386a3f468165bf39f0c816e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 14 Dec 2025 18:31:08 -0500 Subject: [PATCH 111/149] refactor(helm): Update templates to replace `component` with `component_category` for improved clarity and consistency --- .../package-helm/templates/_helpers.tpl | 32 +++++++++---------- .../templates/api-server-deployment.yaml | 2 +- .../templates/api-server-logs-pv.yaml | 2 +- .../templates/api-server-logs-pvc.yaml | 2 +- .../compression-scheduler-deployment.yaml | 4 +-- .../compression-scheduler-logs-pv.yaml | 2 +- .../compression-scheduler-logs-pvc.yaml | 2 +- .../compression-scheduler-user-logs-pv.yaml | 2 +- .../compression-scheduler-user-logs-pvc.yaml | 2 +- .../compression-worker-deployment.yaml | 8 ++--- .../templates/compression-worker-logs-pv.yaml | 2 +- .../compression-worker-logs-pvc.yaml | 2 +- ...compression-worker-staged-archives-pv.yaml | 2 +- ...ompression-worker-staged-archives-pvc.yaml | 2 +- .../templates/compression-worker-tmp-pv.yaml | 2 +- .../templates/compression-worker-tmp-pvc.yaml | 2 +- .../templates/database-data-pv.yaml | 2 +- .../templates/database-logs-pv.yaml | 2 +- .../garbage-collector-deployment.yaml | 6 ++-- .../templates/garbage-collector-logs-pv.yaml | 2 +- .../templates/garbage-collector-logs-pvc.yaml | 2 +- .../templates/mcp-server-deployment.yaml | 2 +- .../templates/mcp-server-logs-pv.yaml | 2 +- .../templates/mcp-server-logs-pvc.yaml | 2 +- .../templates/query-scheduler-deployment.yaml | 2 +- .../templates/query-scheduler-logs-pv.yaml | 2 +- .../templates/query-scheduler-logs-pvc.yaml | 2 +- .../templates/query-worker-deployment.yaml | 8 ++--- .../templates/query-worker-logs-pv.yaml | 2 +- .../templates/query-worker-logs-pvc.yaml | 2 +- .../query-worker-staged-streams-pv.yaml | 2 +- .../query-worker-staged-streams-pvc.yaml | 2 +- .../package-helm/templates/queue-logs-pv.yaml | 2 +- .../package-helm/templates/redis-data-pv.yaml | 2 +- .../package-helm/templates/redis-logs-pv.yaml | 2 +- .../templates/reducer-deployment.yaml | 2 +- .../templates/reducer-logs-pv.yaml | 2 +- .../templates/reducer-logs-pvc.yaml | 2 +- .../templates/results-cache-data-pv.yaml | 2 +- .../templates/results-cache-logs-pv.yaml | 2 +- .../templates/shared-data-archives-pv.yaml | 2 +- .../templates/shared-data-archives-pvc.yaml | 2 +- .../templates/shared-data-streams-pv.yaml | 2 +- .../templates/shared-data-streams-pvc.yaml | 2 +- .../templates/webui-deployment.yaml | 2 +- 45 files changed, 69 insertions(+), 69 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 59d0a84b71..90c444f565 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -98,8 +98,8 @@ failureThreshold: 3 Creates a local PersistentVolume. @param {object} root Root template context -@param {string} component Component category (e.g., "shared-data", "database") -@param {string} name Storage name (e.g., "archives", "logs", "data") +@param {string} component_category (e.g., "shared-data", "database") +@param {string} name (e.g., "archives", "logs", "data") @param {string} nodeRole Node role for affinity. Targets nodes with label "node-role.kubernetes.io/". Always falls back to "node-role.kubernetes.io/control-plane" @@ -112,10 +112,10 @@ Creates a local PersistentVolume. apiVersion: "v1" kind: "PersistentVolume" metadata: - name: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} + name: {{ include "clp.fullname" .root }}-{{ .component_category }}-{{ .name }} labels: {{- include "clp.labels" .root | nindent 4 }} - app.kubernetes.io/component: {{ .component | quote }} + app.kubernetes.io/component: {{ .component_category | quote }} spec: capacity: storage: {{ .capacity }} @@ -139,8 +139,8 @@ spec: Creates a PersistentVolumeClaim for the given component. @param {object} root Root template context -@param {string} component Component category (e.g., "shared-data", "database") -@param {string} name Storage name (e.g., "archives", "logs", "data") +@param {string} component_category (e.g., "shared-data", "database") +@param {string} name (e.g., "archives", "logs", "data") @param {string} capacity Storage capacity @param {string[]} accessModes Access modes @return {string} YAML-formatted PersistentVolumeClaim resource @@ -149,17 +149,17 @@ Creates a PersistentVolumeClaim for the given component. apiVersion: "v1" kind: "PersistentVolumeClaim" metadata: - name: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} + name: {{ include "clp.fullname" .root }}-{{ .component_category }}-{{ .name }} labels: {{- include "clp.labels" .root | nindent 4 }} - app.kubernetes.io/component: {{ .component | quote }} + app.kubernetes.io/component: {{ .component_category | quote }} spec: accessModes: {{ .accessModes }} storageClassName: "local-storage" selector: matchLabels: {{- include "clp.selectorLabels" .root | nindent 6 }} - app.kubernetes.io/component: {{ .component | quote }} + app.kubernetes.io/component: {{ .component_category | quote }} resources: requests: storage: {{ .capacity }} @@ -169,14 +169,14 @@ spec: Creates a volume definition that references a PersistentVolumeClaim. @param {object} root Root template context -@param {string} component Component category (e.g., "shared-data", "database") -@param {string} name Storage name (e.g., "archives", "logs", "data") +@param {string} component_category (e.g., "shared-data", "database") +@param {string} name (e.g., "archives", "logs", "data") @return {string} YAML-formatted volume definition */}} {{- define "clp.pvcVolume" -}} -- name: {{ printf "%s-%s" .component .name | quote }} +- name: {{ printf "%s-%s" .component_category .name | quote }} persistentVolumeClaim: - claimName: {{ include "clp.fullname" .root }}-{{ .component }}-{{ .name }} + claimName: {{ include "clp.fullname" .root }}-{{ .component_category }}-{{ .name }} {{- end }} {{/* @@ -211,9 +211,9 @@ Creates the RESULT_BACKEND env var for Celery workers. Creates an initContainer that waits for a Kubernetes resource to be ready. @param {object} root Root template context -@param {string} type Resource type: "service" (waits for pod readiness) or "job" (waits for completion) -@param {string} name For "service": component name (used for pod label selector) - For "job": job name suffix (appended to fullname) +@param {string} type "service" (waits for pod readiness) or "job" (waits for completion) +@param {string} name For "service": component name + For "job": job name suffix @return {string} YAML-formatted initContainer definition */}} {{- define "clp.waitFor" -}} diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 6571f84b9f..8756623df3 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -80,7 +80,7 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "api-server" + "component_category" "api-server" "name" "logs" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index 84b44ad906..8d51f753fc 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "api-server" + "component_category" "api-server" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml index d63e3dbb3f..d290ac58cf 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "api-server" + "component_category" "api-server" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index fa7fd55118..fee1ba1d85 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -85,12 +85,12 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "compression-scheduler" + "component_category" "compression-scheduler" "name" "logs" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "compression-scheduler" + "component_category" "compression-scheduler" "name" "user-logs" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 565fc541f1..565f5dc40b 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "compression-scheduler" + "component_category" "compression-scheduler" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml index 0d19be396a..37d53a0105 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-scheduler" + "component_category" "compression-scheduler" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml index 6653e4290d..c8667c4597 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "compression-scheduler" + "component_category" "compression-scheduler" "name" "user-logs" "nodeRole" "control-plane" "capacity" "10Gi" diff --git a/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml index bf56a89311..5e0637d2eb 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-scheduler" + "component_category" "compression-scheduler" "name" "user-logs" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 7d9a21524b..8a4e31a972 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -78,22 +78,22 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "logs" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "staged-archives" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "tmp" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "archives" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index aa1f3d165e..4b6d55466b 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml index b6953b0393..136ffbd3ae 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index b79ea5d0c1..924fe4639b 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "staged-archives" "nodeRole" "control-plane" "capacity" "20Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml index 77f0324264..d96f2a62e1 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "staged-archives" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml index 18cd4a68b7..b091cc1634 100644 --- a/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "tmp" "nodeRole" "control-plane" "capacity" "10Gi" diff --git a/tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml index 51b7bc0d60..f1f500e60c 100644 --- a/tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-tmp-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "compression-worker" + "component_category" "compression-worker" "name" "tmp" "capacity" "10Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/database-data-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml index 1515a853c8..3bd6ea2b9a 100644 --- a/tools/deployment/package-helm/templates/database-data-pv.yaml +++ b/tools/deployment/package-helm/templates/database-data-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "database" + "component_category" "database" "name" "data" "nodeRole" "control-plane" "capacity" "20Gi" diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml index ea454a8676..794215cf3a 100644 --- a/tools/deployment/package-helm/templates/database-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "database" + "component_category" "database" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index b5e22ebff2..1cac183243 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -76,17 +76,17 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "garbage-collector" + "component_category" "garbage-collector" "name" "logs" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "archives" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "streams" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index d986b3cf7a..e03597028c 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "garbage-collector" + "component_category" "garbage-collector" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml index f84222d9b8..45e39243b6 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "garbage-collector" + "component_category" "garbage-collector" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 13863f2f60..5f5a7d939d 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -84,7 +84,7 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "mcp-server" + "component_category" "mcp-server" "name" "logs" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index 5af6d1c6ed..9c53c9c7f1 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -1,7 +1,7 @@ {{- if .Values.clpConfig.mcp_server }} {{- include "clp.createLocalPv" (dict "root" . - "component" "mcp-server" + "component_category" "mcp-server" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml index 61219f973f..7ab19f231d 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pvc.yaml @@ -1,7 +1,7 @@ {{- if .Values.clpConfig.mcp_server }} {{- include "clp.createPvc" (dict "root" . - "component" "mcp-server" + "component_category" "mcp-server" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 94d79a28d3..76fc1c60fb 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -85,7 +85,7 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "query-scheduler" + "component_category" "query-scheduler" "name" "logs" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index 346b250c2c..de7633da55 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "query-scheduler" + "component_category" "query-scheduler" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml index 559d3518ff..fa56f7af7e 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "query-scheduler" + "component_category" "query-scheduler" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 5329f3b745..729f7becc8 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -75,22 +75,22 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "query-worker" + "component_category" "query-worker" "name" "logs" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "query-worker" + "component_category" "query-worker" "name" "staged-streams" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "archives" ) | nindent 8 }} {{- include "clp.pvcVolume" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "streams" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index 65f9ff3eed..4f602a8c74 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "query-worker" + "component_category" "query-worker" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml index eb2f41e916..84d478b40a 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "query-worker" + "component_category" "query-worker" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index 02b95378a6..157c203f65 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "query-worker" + "component_category" "query-worker" "name" "staged-streams" "nodeRole" "control-plane" "capacity" "20Gi" diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml index 242fe1ff4a..cf9bb28eaa 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "query-worker" + "component_category" "query-worker" "name" "staged-streams" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml index a24808b632..c9315630f5 100644 --- a/tools/deployment/package-helm/templates/queue-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "queue" + "component_category" "queue" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml index d5be52b6b1..56efc9d19c 100644 --- a/tools/deployment/package-helm/templates/redis-data-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "redis" + "component_category" "redis" "name" "data" "nodeRole" "control-plane" "capacity" "20Gi" diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml index 755484bacd..7f03c8cdad 100644 --- a/tools/deployment/package-helm/templates/redis-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "redis" + "component_category" "redis" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index f6ee050a35..a9af48f0be 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -66,7 +66,7 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "reducer" + "component_category" "reducer" "name" "logs" ) | nindent 8 }} - name: "config" diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index 1d12c20d16..0aab54233a 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "reducer" + "component_category" "reducer" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml index 78dbd64288..40a3affaea 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "reducer" + "component_category" "reducer" "name" "logs" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml index a4a32223b5..8410734b2e 100644 --- a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "results-cache" + "component_category" "results-cache" "name" "data" "nodeRole" "control-plane" "capacity" "20Gi" diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index 9498d0ad65..e76aaf90af 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "results-cache" + "component_category" "results-cache" "name" "logs" "nodeRole" "control-plane" "capacity" "5Gi" diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml index 9373c85fcf..d35eb98dbb 100644 --- a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "archives" "nodeRole" "control-plane" "capacity" "50Gi" diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml index fb53ebd225..1cbc9c3f4d 100644 --- a/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "archives" "capacity" "50Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml index d96a7be523..bba9a8d5b5 100644 --- a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml @@ -1,6 +1,6 @@ {{- include "clp.createLocalPv" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "streams" "nodeRole" "control-plane" "capacity" "20Gi" diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml index 4f4e6ea9a0..3477876d73 100644 --- a/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml @@ -1,6 +1,6 @@ {{- include "clp.createPvc" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "streams" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 4f0415556f..627a44abed 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -87,7 +87,7 @@ spec: volumes: {{- include "clp.pvcVolume" (dict "root" . - "component" "shared-data" + "component_category" "shared-data" "name" "streams" ) | nindent 8 }} - name: "client-settings" From a0cbb98dee06b7bea818244570df0f6a0c30bd13 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 14 Dec 2025 18:58:08 -0500 Subject: [PATCH 112/149] refactor(helm): Standardize indentation and list formatting in templates for consistent readability --- .../package-helm/templates/_helpers.tpl | 40 ++++++------- .../templates/api-server-deployment.yaml | 30 +++++----- .../compression-scheduler-deployment.yaml | 60 +++++++++---------- .../compression-worker-deployment.yaml | 50 ++++++++-------- .../templates/db-table-creator-job.yaml | 2 +- .../garbage-collector-deployment.yaml | 50 ++++++++-------- .../templates/mcp-server-deployment.yaml | 30 +++++----- .../templates/query-scheduler-deployment.yaml | 46 +++++++------- .../templates/query-worker-deployment.yaml | 50 ++++++++-------- .../templates/reducer-deployment.yaml | 30 +++++----- .../results-cache-indices-creator-job.yaml | 10 ++-- .../templates/webui-deployment.yaml | 30 +++++----- 12 files changed, 214 insertions(+), 214 deletions(-) diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 90c444f565..0c5832bcb2 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -174,9 +174,9 @@ Creates a volume definition that references a PersistentVolumeClaim. @return {string} YAML-formatted volume definition */}} {{- define "clp.pvcVolume" -}} -- name: {{ printf "%s-%s" .component_category .name | quote }} - persistentVolumeClaim: - claimName: {{ include "clp.fullname" .root }}-{{ .component_category }}-{{ .name }} +name: {{ printf "%s-%s" .component_category .name | quote }} +persistentVolumeClaim: + claimName: {{ include "clp.fullname" .root }}-{{ .component_category }}-{{ .name }} {{- end }} {{/* @@ -189,8 +189,8 @@ Creates the BROKER_URL env var for Celery workers. {{- $user := .Values.credentials.queue.username -}} {{- $pass := .Values.credentials.queue.password -}} {{- $host := printf "%s-queue" (include "clp.fullname" .) -}} -- name: "BROKER_URL" - value: {{ printf "amqp://%s:%s@%s:5672" $user $pass $host | quote }} +name: "BROKER_URL" +value: {{ printf "amqp://%s:%s@%s:5672" $user $pass $host | quote }} {{- end }} {{/* @@ -203,8 +203,8 @@ Creates the RESULT_BACKEND env var for Celery workers. {{- define "clp.celeryResultBackendEnvVar" -}} {{- $pass := .root.Values.credentials.redis.password -}} {{- $host := printf "%s-redis" (include "clp.fullname" .root) -}} -- name: "RESULT_BACKEND" - value: {{ printf "redis://default:%s@%s:6379/%d" $pass $host (int .database) | quote }} +name: "RESULT_BACKEND" +value: {{ printf "redis://default:%s@%s:6379/%d" $pass $host (int .database) | quote }} {{- end }} {{/* @@ -217,17 +217,17 @@ Creates an initContainer that waits for a Kubernetes resource to be ready. @return {string} YAML-formatted initContainer definition */}} {{- define "clp.waitFor" -}} -- name: "wait-for-{{ .name }}" - image: "bitnami/kubectl:latest" - command: [ - "kubectl", "wait", - {{- if eq .type "service" }} - "--for=condition=ready", - "pod", "--selector", "app.kubernetes.io/component={{ .name }}", - {{- else if eq .type "job" }} - "--for=condition=complete", - "job/{{ include "clp.fullname" .root }}-{{ .name }}", - {{- end }} - "--timeout=300s" - ] +name: "wait-for-{{ .name }}" +image: "bitnami/kubectl:latest" +command: [ + "kubectl", "wait", + {{- if eq .type "service" }} + "--for=condition=ready", + "pod", "--selector", "app.kubernetes.io/component={{ .name }}", + {{- else if eq .type "job" }} + "--for=condition=complete", + "job/{{ include "clp.fullname" .root }}-{{ .name }}", + {{- end }} + "--timeout=300s" +] {{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 8756623df3..f40eec24f4 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -24,16 +24,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "db-table-creator" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "results-cache-indices-creator" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 10 }} containers: - name: "api-server" image: "{{ include "clp.image.ref" . }}" @@ -78,11 +78,11 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} httpGet: *api-server-health-check volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "api-server" - "name" "logs" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "api-server" + "name" "logs" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index fee1ba1d85..bfff811db9 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -24,31 +24,31 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "db-table-creator" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "service" - "name" "queue" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "service" - "name" "redis" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "queue" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "redis" + ) | nindent 10 }} containers: - name: "compression-scheduler" image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} - {{- include "clp.celeryResultBackendEnvVar" (dict - "root" . - "database" .Values.clpConfig.redis.compression_backend_database - ) | nindent 12 }} + - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 14 }} + - {{- include "clp.celeryResultBackendEnvVar" (dict + "root" . + "database" .Values.clpConfig.redis.compression_backend_database + ) | nindent 14 }} - name: "CLP_DB_PASS" valueFrom: secretKeyRef: @@ -83,16 +83,16 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "compression-scheduler" - "name" "logs" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "compression-scheduler" - "name" "user-logs" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-scheduler" + "name" "logs" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-scheduler" + "name" "user-logs" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 8a4e31a972..f8f8d1b5cd 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -24,18 +24,18 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} + - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 10 }} + - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 10 }} containers: - name: "compression-worker" image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} - {{- include "clp.celeryResultBackendEnvVar" (dict + - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 14 }} + - {{- include "clp.celeryResultBackendEnvVar" (dict "root" . "database" .Values.clpConfig.redis.compression_backend_database - ) | nindent 12 }} + ) | nindent 14 }} - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yaml" - name: "CLP_HOME" @@ -76,26 +76,26 @@ spec: "-n", "compression-worker" ] volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "compression-worker" - "name" "logs" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "compression-worker" - "name" "staged-archives" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "compression-worker" - "name" "tmp" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "archives" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-worker" + "name" "logs" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-worker" + "name" "staged-archives" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-worker" + "name" "tmp" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "archives" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/db-table-creator-job.yaml b/tools/deployment/package-helm/templates/db-table-creator-job.yaml index 8772c56fd8..f5e5d0698d 100644 --- a/tools/deployment/package-helm/templates/db-table-creator-job.yaml +++ b/tools/deployment/package-helm/templates/db-table-creator-job.yaml @@ -20,7 +20,7 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "database") | nindent 8 }} + - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "database") | nindent 10 }} containers: - name: "db-table-creator" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 1cac183243..301bec93f7 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -24,16 +24,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "db-table-creator" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "results-cache-indices-creator" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 10 }} containers: - name: "garbage-collector" image: "{{ include "clp.image.ref" . }}" @@ -74,21 +74,21 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "garbage-collector" - "name" "logs" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "archives" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "streams" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "garbage-collector" + "name" "logs" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "archives" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "streams" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 5f5a7d939d..c66ab63747 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -25,16 +25,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "db-table-creator" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "results-cache-indices-creator" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 10 }} containers: - name: "mcp-server" image: "{{ include "clp.image.ref" . }}" @@ -82,11 +82,11 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} httpGet: *mcp-server-health-check volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "mcp-server" - "name" "logs" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "mcp-server" + "name" "logs" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 76fc1c60fb..ba31a85aa6 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -26,31 +26,31 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "db-table-creator" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "service" - "name" "queue" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "service" - "name" "redis" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "queue" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "redis" + ) | nindent 10 }} containers: - name: "query-scheduler" image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} - {{- include "clp.celeryResultBackendEnvVar" (dict + - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 14 }} + - {{- include "clp.celeryResultBackendEnvVar" (dict "root" . "database" .Values.clpConfig.redis.query_backend_database - ) | nindent 12 }} + ) | nindent 14 }} - name: "CLP_DB_PASS" valueFrom: secretKeyRef: @@ -83,11 +83,11 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "query-scheduler" - "name" "logs" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "query-scheduler" + "name" "logs" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 729f7becc8..93a58bbca2 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -24,18 +24,18 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 8 }} - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 8 }} + - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "queue") | nindent 10 }} + - {{- include "clp.waitFor" (dict "root" . "type" "service" "name" "redis") | nindent 10 }} containers: - name: "query-worker" image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 12 }} - {{- include "clp.celeryResultBackendEnvVar" (dict + - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 14 }} + - {{- include "clp.celeryResultBackendEnvVar" (dict "root" . "database" .Values.clpConfig.redis.query_backend_database - ) | nindent 12 }} + ) | nindent 14 }} - name: "CLP_CONFIG_PATH" value: "/etc/clp-config.yaml" - name: "CLP_HOME" @@ -73,26 +73,26 @@ spec: "-n", "query-worker" ] volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "query-worker" - "name" "logs" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "query-worker" - "name" "staged-streams" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "archives" - ) | nindent 8 }} - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "streams" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "query-worker" + "name" "logs" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "query-worker" + "name" "staged-streams" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "archives" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "streams" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index a9af48f0be..97fc4d55df 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -26,16 +26,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "service" - "name" "query-scheduler" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "results-cache-indices-creator" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "query-scheduler" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 10 }} containers: - name: "reducer" image: "{{ include "clp.image.ref" . }}" @@ -64,11 +64,11 @@ spec: "--upsert-interval", "{{ .Values.clpConfig.reducer.upsert_interval }}" ] volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "reducer" - "name" "logs" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "reducer" + "name" "logs" + ) | nindent 10 }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml index 0dcf9ee646..170707562c 100644 --- a/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml +++ b/tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml @@ -20,11 +20,11 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "service" - "name" "results-cache" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "results-cache" + ) | nindent 10 }} containers: - name: "results-cache-indices-creator" image: "{{ include "clp.image.ref" . }}" diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 627a44abed..86198f55df 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -24,16 +24,16 @@ spec: runAsGroup: {{ .Values.securityContext.firstParty.gid }} fsGroup: {{ .Values.securityContext.firstParty.gid }} initContainers: - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "db-table-creator" - ) | nindent 8 }} - {{- include "clp.waitFor" (dict - "root" . - "type" "job" - "name" "results-cache-indices-creator" - ) | nindent 8 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 10 }} containers: - name: "webui" image: "{{ include "clp.image.ref" . }}" @@ -85,11 +85,11 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: *webui-health-check volumes: - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "streams" - ) | nindent 8 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "streams" + ) | nindent 10 }} - name: "client-settings" configMap: name: {{ include "clp.fullname" . }}-config From 8beb8e073985245ffe65776f96c664bea961ac97 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 11:51:31 -0500 Subject: [PATCH 113/149] feat(helm): Add S3 support for storage configurations and AWS authentication options. --- .../templates/api-server-deployment.yaml | 20 +++++ .../api-server-staged-streams-pv.yaml | 11 +++ .../api-server-staged-streams-pvc.yaml | 9 +++ .../compression-scheduler-deployment.yaml | 15 ++++ .../compression-worker-deployment.yaml | 19 +++++ .../package-helm/templates/configmap.yaml | 77 +++++++++++++++++++ .../garbage-collector-deployment.yaml | 19 +++++ .../templates/query-worker-deployment.yaml | 20 +++++ .../templates/webui-deployment.yaml | 22 ++++++ 9 files changed, 212 insertions(+) create mode 100644 tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml create mode 100644 tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index f40eec24f4..e54d126f51 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -63,6 +63,13 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + - name: "api-server-staged-streams" + mountPath: "/var/data/staged-streams" + {{- else }} + - name: "shared-data-streams" + mountPath: "/var/data/streams" + {{- end }} command: [ "/opt/clp/bin/api_server", "--host", "0.0.0.0", @@ -83,6 +90,19 @@ spec: "component_category" "api-server" "name" "logs" ) | nindent 10 }} + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "api-server" + "name" "staged-streams" + ) | nindent 10 }} + {{- else }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "streams" + ) | nindent 10 }} + {{- end }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml new file mode 100644 index 0000000000..6187dd822a --- /dev/null +++ b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml @@ -0,0 +1,11 @@ +{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} +{{- include "clp.createLocalPv" (dict + "root" . + "component_category" "api-server" + "name" "staged-streams" + "nodeRole" "control-plane" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") + "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml new file mode 100644 index 0000000000..79ef5617c8 --- /dev/null +++ b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml @@ -0,0 +1,9 @@ +{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} +{{- include "clp.createPvc" (dict + "root" . + "component_category" "api-server" + "name" "staged-streams" + "capacity" "20Gi" + "accessModes" (list "ReadWriteMany") +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index bfff811db9..91d76adbd9 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -74,9 +74,16 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - name: "logs-input" mountPath: "/mnt/logs" readOnly: true + {{- end }} command: [ "python3", "-u", "-m", "job_orchestration.scheduler.compress.compression_scheduler", @@ -96,7 +103,15 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: "{{ .Values.clpConfig.aws_config_directory }}" + type: "Directory" + {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" type: "Directory" + {{- end }} diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index f8f8d1b5cd..508f0388ae 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -59,11 +59,20 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - name: "logs-input" mountPath: "/mnt/logs" readOnly: true + {{- end }} + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: "shared-data-archives" mountPath: "/var/data/archives" + {{- end }} command: [ "python3", "-u", "/opt/clp/lib/python3/site-packages/bin/celery", @@ -91,15 +100,25 @@ spec: "component_category" "compression-worker" "name" "tmp" ) | nindent 10 }} + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "archives" ) | nindent 10 }} + {{- end }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - name: "logs-input" hostPath: path: "{{ .Values.clpConfig.logs_input.directory }}" type: "Directory" + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: "{{ .Values.clpConfig.aws_config_directory }}" + type: "Directory" + {{- end }} diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index d595a36acd..27276e9669 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -22,8 +22,28 @@ data: compression_level: {{ .Values.clpConfig.archive_output.compression_level }} retention_period: {{ .Values.clpConfig.archive_output.retention_period }} storage: + {{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} + type: "s3" + staging_directory: "/var/data/staged-archives" + s3_config: + endpoint_url: {{ .Values.clpConfig.archive_output.storage.s3_config.endpoint_url | default "null" }} + region_code: {{ .Values.clpConfig.archive_output.storage.s3_config.region_code | quote }} + bucket: {{ .Values.clpConfig.archive_output.storage.s3_config.bucket | quote }} + key_prefix: {{ .Values.clpConfig.archive_output.storage.s3_config.key_prefix | quote }} + aws_authentication: + type: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.type | quote }} + {{- if eq .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.type "credentials" }} + credentials: + access_key_id: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} + secret_access_key: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + {{- end }} + {{- if eq .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.type "profile" }} + profile: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.profile | quote }} + {{- end }} + {{- else }} directory: "/var/data/archives" type: "fs" + {{- end }} target_archive_size: {{ .Values.clpConfig.archive_output.target_archive_size | int }} target_dictionaries_size: {{ .Values.clpConfig.archive_output.target_dictionaries_size | int }} @@ -55,8 +75,22 @@ data: search_result: {{ .Values.clpConfig.garbage_collector.sweep_interval.search_result }} logs_directory: "/var/log" logs_input: + {{- if eq .Values.clpConfig.logs_input.type "s3" }} + type: "s3" + aws_authentication: + type: {{ .Values.clpConfig.logs_input.aws_authentication.type | quote }} + {{- if eq .Values.clpConfig.logs_input.aws_authentication.type "credentials" }} + credentials: + access_key_id: {{ .Values.clpConfig.logs_input.aws_authentication.credentials.access_key_id | quote }} + secret_access_key: {{ .Values.clpConfig.logs_input.aws_authentication.credentials.secret_access_key | quote }} + {{- end }} + {{- if eq .Values.clpConfig.logs_input.aws_authentication.type "profile" }} + profile: {{ .Values.clpConfig.logs_input.aws_authentication.profile | quote }} + {{- end }} + {{- else }} directory: "/mnt/logs" type: "fs" + {{- end }} mcp_server: {{ .Values.clpConfig.mcp_server }} package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} @@ -92,8 +126,28 @@ data: stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} stream_output: storage: + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + type: "s3" + staging_directory: "/var/data/staged-streams" + s3_config: + endpoint_url: {{ .Values.clpConfig.stream_output.storage.s3_config.endpoint_url | default "null" }} + region_code: {{ .Values.clpConfig.stream_output.storage.s3_config.region_code | quote }} + bucket: {{ .Values.clpConfig.stream_output.storage.s3_config.bucket | quote }} + key_prefix: {{ .Values.clpConfig.stream_output.storage.s3_config.key_prefix | quote }} + aws_authentication: + type: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type | quote }} + {{- if eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "credentials" }} + credentials: + access_key_id: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} + secret_access_key: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "profile" }} + profile: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.profile | quote }} + {{- end }} + {{- else }} directory: "/var/data/streams" type: "fs" + {{- end }} target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }} tmp_directory: "/var/tmp" webui: @@ -136,6 +190,11 @@ data: "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, "LogsInputType": {{ .Values.clpConfig.logs_input.type | quote }}, + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + "LogsInputRootDir": "/mnt/logs", + {{- else }} + "LogsInputRootDir": null, + {{- end }} "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, "SqlDbClpArchivesTableName": "clp_archives", @@ -160,12 +219,30 @@ data: "MongoDbStreamFilesCollectionName": "stream-files", "ClientDir": "/opt/clp/var/www/webui/client", "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer", + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + "LogsInputRootDir": "/mnt/logs", + {{- else }} + "LogsInputRootDir": null, + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + "StreamFilesDir": null, + "StreamTargetUncompressedSize": + {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }}, + "StreamFilesS3Region": {{ .Values.clpConfig.stream_output.storage.s3_config.region_code | quote }}, + "StreamFilesS3PathPrefix": "{{ .Values.clpConfig.stream_output.storage.s3_config.bucket }}/{{ .Values.clpConfig.stream_output.storage.s3_config.key_prefix }}", + {{- if eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "profile" }} + "StreamFilesS3Profile": {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.profile | quote }}, + {{- else }} + "StreamFilesS3Profile": null, + {{- end }} + {{- else }} "StreamFilesDir": "/var/data/streams", "StreamTargetUncompressedSize": {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }}, "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, + {{- end }} "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level }}, "ArchiveOutputTargetArchiveSize": {{ .Values.clpConfig.archive_output.target_archive_size | int }}, diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 301bec93f7..08f5b7bab3 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -64,10 +64,19 @@ spec: readOnly: true - name: "garbage-collector-logs" mountPath: "/var/log/garbage_collector" + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: "shared-data-archives" mountPath: "/var/data/archives" + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: "shared-data-streams" mountPath: "/var/data/streams" + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} command: [ "python3", "-u", "-m", "job_orchestration.garbage_collector.garbage_collector", @@ -79,16 +88,26 @@ spec: "component_category" "garbage-collector" "name" "logs" ) | nindent 10 }} + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "archives" ) | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "streams" ) | nindent 10 }} + {{- end }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: "{{ .Values.clpConfig.aws_config_directory }}" + type: "Directory" + {{- end }} diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 93a58bbca2..4d54971e75 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -57,10 +57,20 @@ spec: mountPath: "/var/log/query_worker" - name: "query-worker-staged-streams" mountPath: "/var/data/staged-streams" + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: "shared-data-archives" mountPath: "/var/data/archives" + readOnly: true + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: "shared-data-streams" mountPath: "/var/data/streams" + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} command: [ "python3", "-u", "/opt/clp/lib/python3/site-packages/bin/celery", @@ -83,16 +93,26 @@ spec: "component_category" "query-worker" "name" "staged-streams" ) | nindent 10 }} + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "archives" ) | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "streams" ) | nindent 10 }} + {{- end }} - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: "{{ .Values.clpConfig.aws_config_directory }}" + type: "Directory" + {{- end }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 86198f55df..acc2ac51b7 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -39,6 +39,12 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: + {{- if and (eq .Values.clpConfig.stream_output.storage.type "s3") (eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "credentials") }} + - name: "AWS_ACCESS_KEY_ID" + value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} + - name: "AWS_SECRET_ACCESS_KEY" + value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + {{- end }} - name: "CLP_DB_USER" valueFrom: secretKeyRef: @@ -71,8 +77,16 @@ spec: mountPath: "/opt/clp/var/www/webui/server/dist/settings.json" subPath: "webui-server-settings.json" readOnly: true + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: "shared-data-streams" mountPath: "/var/data/streams" + readOnly: true + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} command: [ "/opt/clp/bin/node-22", "/opt/clp/var/www/webui/server/dist/src/main.js" @@ -85,14 +99,22 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: *webui-health-check volumes: + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "streams" ) | nindent 10 }} + {{- end }} - name: "client-settings" configMap: name: {{ include "clp.fullname" . }}-config - name: "server-settings" configMap: name: {{ include "clp.fullname" . }}-config + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: "{{ .Values.clpConfig.aws_config_directory }}" + type: "Directory" + {{- end }} From 41f0348265c187d0360644ac3b8d624dea83044e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:15:55 -0500 Subject: [PATCH 114/149] refactor(helm): Reorganize volumeMounts and environment variables for clarity and alignment across templates --- .../templates/api-server-deployment.yaml | 25 +++++++++----- .../compression-scheduler-deployment.yaml | 6 ++-- .../compression-worker-deployment.yaml | 27 ++++++++------- .../garbage-collector-deployment.yaml | 33 ++++++++++++------- .../templates/mcp-server-deployment.yaml | 13 +++++--- .../templates/webui-deployment.yaml | 24 +++++++------- 6 files changed, 76 insertions(+), 52 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index e54d126f51..2b46eb5e66 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -39,16 +39,16 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "CLP_DB_USER" + - name: "CLP_DB_PASS" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "username" - - name: "CLP_DB_PASS" + key: "password" + - name: "CLP_DB_USER" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "username" - name: "CLP_LOGS_DIR" value: "/var/log/api_server" - name: "RUST_LOG" @@ -57,17 +57,26 @@ spec: - name: "api-server" containerPort: 3001 volumeMounts: - - name: "api-server-logs" - mountPath: "/var/log/api_server" - name: "config" mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true + - name: {{ include "clp.volumeName" (dict + "component_category" "api-server" + "name" "logs" + ) | quote }} + mountPath: "/var/log/api_server" {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - - name: "api-server-staged-streams" + - name: {{ include "clp.volumeName" (dict + "component_category" "api-server" + "name" "staged-streams" + ) | quote }} mountPath: "/var/data/staged-streams" {{- else }} - - name: "shared-data-streams" + - name: {{ include "clp.volumeName" (dict + "component_category" "shared-data" + "name" "streams" + ) | quote }} mountPath: "/var/data/streams" {{- end }} command: [ diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 06bc69c31f..97f9c484b8 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -66,9 +66,6 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - {{- if eq .Values.clpConfig.logs_input.type "fs" }} - - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} - {{- end }} - name: "config" mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" @@ -88,6 +85,9 @@ spec: mountPath: "/opt/clp/.aws" readOnly: true {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} + {{- end }} command: [ "python3", "-u", "-m", "job_orchestration.scheduler.compress.compression_scheduler", diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index aa3fdd0174..b48eebf08b 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -45,23 +45,30 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - {{- if eq .Values.clpConfig.logs_input.type "fs" }} - - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} - {{- end }} + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "compression-worker" "name" "logs" ) | quote }} mountPath: "/var/log/compression_worker" + - name: {{ include "clp.volumeName" (dict + "component_category" "compression-worker" + "name" "staged-archives" + ) | quote }} + mountPath: "/var/data/staged-archives" - name: {{ include "clp.volumeName" (dict "component_category" "compression-worker" "name" "tmp" ) | quote }} mountPath: "/var/tmp" - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" readOnly: true + {{- end }} {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -69,12 +76,8 @@ spec: ) | quote }} mountPath: "/var/data/archives" {{- end }} - - name: "compression-worker-staged-archives" - mountPath: "/var/data/staged-archives" - {{- if .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - mountPath: "/opt/clp/.aws" - readOnly: true + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} {{- end }} command: [ "python3", "-u", diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 08f5b7bab3..ab2a3e770e 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -39,16 +39,16 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "CLP_DB_USER" + - name: "CLP_DB_PASS" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "username" - - name: "CLP_DB_PASS" + key: "password" + - name: "CLP_DB_USER" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "username" - name: "CLP_HOME" value: "/opt/clp" - name: "CLP_LOGGING_LEVEL" @@ -62,21 +62,30 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true - - name: "garbage-collector-logs" + - name: {{ include "clp.volumeName" (dict + "component_category" "garbage-collector" + "name" "logs" + ) | quote }} mountPath: "/var/log/garbage_collector" + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - - name: "shared-data-archives" + - name: {{ include "clp.volumeName" (dict + "component_category" "shared-data" + "name" "archives" + ) | quote }} mountPath: "/var/data/archives" {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - - name: "shared-data-streams" + - name: {{ include "clp.volumeName" (dict + "component_category" "shared-data" + "name" "streams" + ) | quote }} mountPath: "/var/data/streams" {{- end }} - {{- if .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - mountPath: "/opt/clp/.aws" - readOnly: true - {{- end }} command: [ "python3", "-u", "-m", "job_orchestration.garbage_collector.garbage_collector", diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index c66ab63747..1238602ba3 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -40,16 +40,16 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - - name: "CLP_DB_USER" + - name: "CLP_DB_PASS" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "username" - - name: "CLP_DB_PASS" + key: "password" + - name: "CLP_DB_USER" valueFrom: secretKeyRef: name: {{ include "clp.fullname" . }}-database - key: "password" + key: "username" - name: "CLP_LOGGING_LEVEL" value: {{ .Values.clpConfig.mcp_server.logging_level | quote }} - name: "CLP_LOGS_DIR" @@ -64,7 +64,10 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true - - name: "mcp-server-logs" + - name: {{ include "clp.volumeName" (dict + "component_category" "mcp-server" + "name" "logs" + ) | quote }} mountPath: "/var/log/mcp_server" command: [ "python3", "-u", diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 5a1a6d50b6..b418874783 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -39,12 +39,6 @@ spec: image: "{{ include "clp.image.ref" . }}" imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" env: - {{- if and (eq .Values.clpConfig.stream_output.storage.type "s3") (eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "credentials") }} - - name: "AWS_ACCESS_KEY_ID" - value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} - - name: "AWS_SECRET_ACCESS_KEY" - value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} - {{- end }} - name: "CLP_DB_PASS" valueFrom: secretKeyRef: @@ -65,11 +59,16 @@ spec: value: "4000" - name: "RATE_LIMIT" value: {{ .Values.clpConfig.webui.rate_limit | quote }} + {{- if and (eq .Values.clpConfig.stream_output.storage.type "s3") (eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "credentials") }} + - name: "AWS_ACCESS_KEY_ID" + value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} + - name: "AWS_SECRET_ACCESS_KEY" + value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + {{- end }} ports: - name: "webui" containerPort: 4000 volumeMounts: - - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} - name: "client-settings" mountPath: "/opt/clp/var/www/webui/client/settings.json" subPath: "webui-client-settings.json" @@ -78,6 +77,12 @@ spec: mountPath: "/opt/clp/var/www/webui/server/dist/settings.json" subPath: "webui-server-settings.json" readOnly: true + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} + {{- include "clp.logsInputVolumeMount" . | nindent 12 }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -85,11 +90,6 @@ spec: ) | quote }} mountPath: "/var/data/streams" {{- end }} - {{- if .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - mountPath: "/opt/clp/.aws" - readOnly: true - {{- end }} command: [ "/opt/clp/bin/node-22", "/opt/clp/var/www/webui/server/dist/src/main.js" From 3363229cf2c22170d24ba8ac5797d2f17272f2f1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:37:50 -0500 Subject: [PATCH 115/149] feat(helm): Add log_ingestor configuration for S3 log input support --- .../package-helm/templates/configmap.yaml | 9 +++++++++ tools/deployment/package-helm/values.yaml | 13 +++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index d1f3b7a52d..b54e31a4a8 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -91,6 +91,15 @@ data: type: "fs" directory: "/mnt/logs" {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "s3" }} + log_ingestor: + buffer_flush_threshold: {{ .Values.clpConfig.log_ingestor.buffer_flush_threshold }} + buffer_flush_timeout: {{ .Values.clpConfig.log_ingestor.buffer_flush_timeout }} + channel_capacity: {{ .Values.clpConfig.log_ingestor.channel_capacity }} + host: "localhost" + logging_level: {{ .Values.clpConfig.log_ingestor.logging_level | quote }} + port: 3002 + {{- end }} package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index ae1029ff51..3320175ebd 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -139,6 +139,19 @@ clpConfig: initial_backoff_ms: 100 max_backoff_ms: 5000 + # log-ingestor config. Currently, the config is applicable only if `logs_input.type` is "s3". + log_ingestor: + port: 30302 + # The timeout (in seconds) after which the log buffer is flushed for compression if no new + # input arrives. + buffer_flush_timeout: 300 + # The log buffer size (in bytes) that triggers a flush for compression. + buffer_flush_threshold: 268435456 # 256 MiB + # The capacity of the internal channel used for communication between an ingestion job and the + # log buffer. + channel_capacity: 10 + logging_level: "INFO" + # Presto client config presto: null # port: 30889 From 17e2fc235d488d6dc3ea17b0b1bfc3ebb539ca28 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:39:17 -0500 Subject: [PATCH 116/149] feat(helm): Add `type` configuration to compression_scheduler for task execution mode selection --- tools/deployment/package-helm/templates/configmap.yaml | 1 + tools/deployment/package-helm/values.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index b54e31a4a8..f2aee9a1d0 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -57,6 +57,7 @@ data: logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} max_concurrent_tasks_per_job: {{ .Values.clpConfig.compression_scheduler.max_concurrent_tasks_per_job }} + type: {{ .Values.clpConfig.compression_scheduler.type | quote }} compression_worker: logging_level: {{ .Values.clpConfig.compression_worker.logging_level | quote }} data_directory: "/var/data" diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 3320175ebd..a65c38df6a 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -45,6 +45,7 @@ clpConfig: jobs_poll_delay: 0.1 # seconds logging_level: "INFO" max_concurrent_tasks_per_job: 0 # A value of 0 disables the limit + type: "celery" # "celery" or "spider" query_scheduler: port: 7000 From 2300cc5689b26fc72c656c547be8032e4b0b06f4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:41:53 -0500 Subject: [PATCH 117/149] feat(helm): Add log-ingestor deployment, service, and volume configurations for S3 log input handling --- .../templates/log-ingestor-deployment.yaml | 88 +++++++++++++++++++ .../templates/log-ingestor-logs-pv.yaml | 11 +++ .../templates/log-ingestor-logs-pvc.yaml | 9 ++ .../templates/log-ingestor-service.yaml | 18 ++++ 4 files changed, 126 insertions(+) create mode 100644 tools/deployment/package-helm/templates/log-ingestor-deployment.yaml create mode 100644 tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/log-ingestor-logs-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/log-ingestor-service.yaml diff --git a/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml b/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml new file mode 100644 index 0000000000..4c4946a892 --- /dev/null +++ b/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml @@ -0,0 +1,88 @@ +{{- if eq .Values.clpConfig.logs_input.type "s3" }} +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp.fullname" . }}-log-ingestor + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "log-ingestor" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "log-ingestor" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "log-ingestor" + spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher + terminationGracePeriodSeconds: 60 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + containers: + - name: "log-ingestor" + image: "{{ include "clp.image.ref" . }}" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" + env: + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "password" + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "username" + - name: "CLP_LOGS_DIR" + value: "/var/log/log_ingestor" + - name: "RUST_LOG" + value: {{ .Values.clpConfig.log_ingestor.logging_level | quote }} + ports: + - name: "log-ingestor" + containerPort: 3002 + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true + - name: {{ include "clp.volumeName" (dict + "component_category" "log-ingestor" + "name" "logs" + ) | quote }} + mountPath: "/var/log/log_ingestor" + command: [ + "/opt/clp/bin/log-ingestor", + "--config", "/etc/clp-config.yaml", + "--host", "0.0.0.0", + "--port", "3002" + ] + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + httpGet: &log-ingestor-health-check + path: "/health" + port: 3002 + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + httpGet: *log-ingestor-health-check + volumes: + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "log-ingestor" + "name" "logs" + ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config +{{- end }} diff --git a/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml b/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml new file mode 100644 index 0000000000..1abb5d03c4 --- /dev/null +++ b/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml @@ -0,0 +1,11 @@ +{{- if eq .Values.clpConfig.logs_input.type "s3" }} +{{- include "clp.createLocalPv" (dict + "root" . + "component_category" "log-ingestor" + "name" "logs" + "nodeRole" "control-plane" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/log_ingestor" .Values.clpConfig.logs_directory) +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/log-ingestor-logs-pvc.yaml b/tools/deployment/package-helm/templates/log-ingestor-logs-pvc.yaml new file mode 100644 index 0000000000..62fc3c9740 --- /dev/null +++ b/tools/deployment/package-helm/templates/log-ingestor-logs-pvc.yaml @@ -0,0 +1,9 @@ +{{- if eq .Values.clpConfig.logs_input.type "s3" }} +{{- include "clp.createPvc" (dict + "root" . + "component_category" "log-ingestor" + "name" "logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/log-ingestor-service.yaml b/tools/deployment/package-helm/templates/log-ingestor-service.yaml new file mode 100644 index 0000000000..f79a324e61 --- /dev/null +++ b/tools/deployment/package-helm/templates/log-ingestor-service.yaml @@ -0,0 +1,18 @@ +{{- if eq .Values.clpConfig.logs_input.type "s3" }} +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-log-ingestor + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "log-ingestor" +spec: + type: "NodePort" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "log-ingestor" + ports: + - port: 3002 + targetPort: "log-ingestor" + nodePort: {{ .Values.clpConfig.log_ingestor.port }} +{{- end }} From 0e34c789f17f628b27e7b4a72e96b668728d9908 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:42:04 -0500 Subject: [PATCH 118/149] refactor(helm): Replace `database.name` with `database.names.clp` --- tools/deployment/package-helm/templates/configmap.yaml | 5 +++-- .../package-helm/templates/database-statefulset.yaml | 2 +- tools/deployment/package-helm/values.yaml | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index f2aee9a1d0..55a8652f5b 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -65,7 +65,8 @@ data: auto_commit: false compress: true host: {{ include "clp.fullname" . }}-database - name: {{ .Values.clpConfig.database.name | quote }} + names: + clp: {{ .Values.clpConfig.database.names.clp | quote }} port: 3306 ssl_cert: null type: {{ .Values.clpConfig.database.type | quote }} @@ -217,7 +218,7 @@ data: { "SqlDbHost": "{{ include "clp.fullname" . }}-database", "SqlDbPort": 3306, - "SqlDbName": {{ .Values.clpConfig.database.name | quote }}, + "SqlDbName": {{ .Values.clpConfig.database.names.clp | quote }}, "SqlDbQueryJobsTableName": "query_jobs", "SqlDbCompressionJobsTableName": "compression_jobs", "MongoDbHost": "{{ include "clp.fullname" . }}-results-cache", diff --git a/tools/deployment/package-helm/templates/database-statefulset.yaml b/tools/deployment/package-helm/templates/database-statefulset.yaml index 8e67cee934..4c510c57c8 100644 --- a/tools/deployment/package-helm/templates/database-statefulset.yaml +++ b/tools/deployment/package-helm/templates/database-statefulset.yaml @@ -33,7 +33,7 @@ spec: imagePullPolicy: "Always" env: - name: "MYSQL_DATABASE" - value: {{ .Values.clpConfig.database.name | quote }} + value: {{ .Values.clpConfig.database.names.clp | quote }} - name: "MYSQL_USER" valueFrom: secretKeyRef: diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index a65c38df6a..a974e5ee56 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -39,7 +39,8 @@ clpConfig: database: type: "mariadb" # "mariadb" or "mysql" port: 30306 - name: "clp-db" + names: + clp: "clp-db" compression_scheduler: jobs_poll_delay: 0.1 # seconds From 50906eadc4bf6c533fd3f5449cca744c88f3c6d8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:45:28 -0500 Subject: [PATCH 119/149] fix(helm): Update S3 profile condition to include storage type validation --- tools/deployment/package-helm/templates/configmap.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 55a8652f5b..9099c55d4e 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -244,7 +244,8 @@ data: "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, {{- end }} - {{- if eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "profile" }} + {{- if and (eq .Values.clpConfig.stream_output.storage.type "s3") + (eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "profile") }} "StreamFilesS3Profile": {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.profile | quote }}, {{- else }} "StreamFilesS3Profile": null, From 748c176532fc084ebc8a9f0b6f35e63953f0bc0b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:47:26 -0500 Subject: [PATCH 120/149] feat(helm): Add mcp_server configuration support in configmap template --- tools/deployment/package-helm/templates/configmap.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 9099c55d4e..b533112967 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -102,6 +102,14 @@ data: logging_level: {{ .Values.clpConfig.log_ingestor.logging_level | quote }} port: 3002 {{- end }} + {{- if .Values.clpConfig.mcp_server }} + mcp_server: + host: "localhost" + logging_level: {{ .Values.clpConfig.mcp_server.logging_level | quote }} + port: 8000 + {{- else }} + mcp_server: null + {{- end }} package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} From b807445f50ce24d9366eaa1054efa732437afde7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:55:20 -0500 Subject: [PATCH 121/149] chore(helm): Update api-server RUST_LOG environment variable to use "INFO" logging level --- .../package-helm/templates/api-server-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 2b46eb5e66..a1850c0dec 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -52,7 +52,7 @@ spec: - name: "CLP_LOGS_DIR" value: "/var/log/api_server" - name: "RUST_LOG" - value: "DEBUG" + value: "INFO" ports: - name: "api-server" containerPort: 3001 From 25c90036f1d9fea3781c739a75e63c6c0df471fb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:56:06 -0500 Subject: [PATCH 122/149] feat(helm): Add readiness and liveness probes to query-scheduler deployment --- .../templates/query-scheduler-deployment.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index ca7149dd75..b12b6fe5dc 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -85,6 +85,14 @@ spec: "-m", "job_orchestration.scheduler.query.query_scheduler", "--config", "/etc/clp-config.yaml" ] + readinessProbe: + tcpSocket: + port: 7000 + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + livenessProbe: + tcpSocket: + port: 7000 + {{- include "clp.livenessProbeTimings" . | nindent 12 }} volumes: - {{- include "clp.pvcVolume" (dict "root" . From e3a34e92bd02d2971cc048055b5ed779a657c731 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:57:58 -0500 Subject: [PATCH 123/149] fix(helm): Correct indentation for logsInputVolumeMount in webui-deployment template --- tools/deployment/package-helm/templates/webui-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index b418874783..eb2fa25c89 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -82,7 +82,7 @@ spec: mountPath: "/opt/clp/.aws" readOnly: true {{- end }} - {{- include "clp.logsInputVolumeMount" . | nindent 12 }} + - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" From dc878b9dc699f50733ca46d116cafeead53766d4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 19:59:02 -0500 Subject: [PATCH 124/149] feat(helm): Add S3 storage type support for query-worker staged streams configuration --- .../package-helm/templates/query-worker-deployment.yaml | 9 +++++++++ .../templates/query-worker-staged-streams-pv.yaml | 2 ++ .../templates/query-worker-staged-streams-pvc.yaml | 2 ++ 3 files changed, 13 insertions(+) diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 5c3491aac5..fcbb976519 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -68,6 +68,13 @@ spec: ) | quote }} mountPath: "/var/data/streams" {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + - name: {{ include "clp.volumeName" (dict + "component_category" "query-worker" + "name" "staged-streams" + ) | quote }} + mountPath: "/var/data/staged-streams" + {{- end }} {{- if .Values.clpConfig.aws_config_directory }} - name: "aws-config" mountPath: "/opt/clp/.aws" @@ -90,11 +97,13 @@ spec: "component_category" "query-worker" "name" "logs" ) | nindent 10 }} + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "query-worker" "name" "staged-streams" ) | nindent 10 }} + {{- end }} {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index 157c203f65..e443b1ace9 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "query-worker" @@ -7,3 +8,4 @@ "accessModes" (list "ReadWriteMany") "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml index cf9bb28eaa..c433a8b2ea 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} {{- include "clp.createPvc" (dict "root" . "component_category" "query-worker" @@ -5,3 +6,4 @@ "capacity" "20Gi" "accessModes" (list "ReadWriteMany") ) }} +{{- end }} From bdcfb5e89ad752a46cfcf5cdd94682d875006abe Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 20:03:05 -0500 Subject: [PATCH 125/149] feat(helm): Add Presto configuration support in values.yaml and configmap template --- tools/deployment/package-helm/templates/configmap.yaml | 5 +++++ tools/deployment/package-helm/values.yaml | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index b533112967..747f96df08 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -271,6 +271,11 @@ data: {{ .Values.clpConfig.archive_output.target_segment_size | int }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, + {{- if .Values.clpConfig.presto }} + "PrestoHost": {{ .Values.clpConfig.presto.host | quote }}, + "PrestoPort": {{ .Values.clpConfig.presto.port }} + {{- else }} "PrestoHost": null, "PrestoPort": null + {{- end }} } diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index a974e5ee56..603ea06b51 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -156,7 +156,8 @@ clpConfig: # Presto client config presto: null - # port: 30889 + # host: "localhost" + # port: 8080 # Location where other data (besides archives) are stored. It will be created if # it doesn't exist. From 6ad5757949f6bb93ba6f9c5d77b9ee7b9b6c37c7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 20:05:15 -0500 Subject: [PATCH 126/149] feat(helm): Add conditional checks for api-server and garbage-collector configurations across templates --- .../package-helm/templates/api-server-deployment.yaml | 2 ++ .../deployment/package-helm/templates/api-server-logs-pv.yaml | 2 ++ .../deployment/package-helm/templates/api-server-logs-pvc.yaml | 2 ++ .../deployment/package-helm/templates/api-server-service.yaml | 2 ++ .../package-helm/templates/api-server-staged-streams-pv.yaml | 3 ++- .../package-helm/templates/api-server-staged-streams-pvc.yaml | 3 ++- .../package-helm/templates/garbage-collector-deployment.yaml | 3 +++ .../package-helm/templates/garbage-collector-logs-pv.yaml | 3 +++ .../package-helm/templates/garbage-collector-logs-pvc.yaml | 3 +++ 9 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index a1850c0dec..40286036fc 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -1,3 +1,4 @@ +{{- if .Values.clpConfig.api_server }} apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -115,3 +116,4 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config +{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index 8d51f753fc..89fd30dda8 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1,3 +1,4 @@ +{{- if .Values.clpConfig.api_server }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "api-server" @@ -7,3 +8,4 @@ "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/api_server" .Values.clpConfig.logs_directory) ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml index d290ac58cf..d9429b6dad 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml @@ -1,3 +1,4 @@ +{{- if .Values.clpConfig.api_server }} {{- include "clp.createPvc" (dict "root" . "component_category" "api-server" @@ -5,3 +6,4 @@ "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-service.yaml b/tools/deployment/package-helm/templates/api-server-service.yaml index 500b5cd6b6..0aed0e7efa 100644 --- a/tools/deployment/package-helm/templates/api-server-service.yaml +++ b/tools/deployment/package-helm/templates/api-server-service.yaml @@ -1,3 +1,4 @@ +{{- if .Values.clpConfig.api_server }} apiVersion: "v1" kind: "Service" metadata: @@ -14,3 +15,4 @@ spec: - port: 3001 targetPort: "api-server" nodePort: {{ .Values.clpConfig.api_server.port }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml index 6187dd822a..0d191046d2 100644 --- a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml @@ -1,4 +1,5 @@ -{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} +{{- if and .Values.clpConfig.api_server + (eq .Values.clpConfig.stream_output.storage.type "s3") }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "api-server" diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml index 79ef5617c8..0979cd49aa 100644 --- a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml @@ -1,4 +1,5 @@ -{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} +{{- if and .Values.clpConfig.api_server + (eq .Values.clpConfig.stream_output.storage.type "s3") }} {{- include "clp.createPvc" (dict "root" . "component_category" "api-server" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index ab2a3e770e..155a99c9f5 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -1,3 +1,5 @@ +{{- if or .Values.clpConfig.archive_output.retention_period + .Values.clpConfig.results_cache.retention_period }} apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -120,3 +122,4 @@ spec: path: "{{ .Values.clpConfig.aws_config_directory }}" type: "Directory" {{- end }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index e03597028c..47c5417a64 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,3 +1,5 @@ +{{- if or .Values.clpConfig.archive_output.retention_period + .Values.clpConfig.results_cache.retention_period }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "garbage-collector" @@ -7,3 +9,4 @@ "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/garbage_collector" .Values.clpConfig.logs_directory) ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml index 45e39243b6..af6f5a0392 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml @@ -1,3 +1,5 @@ +{{- if or .Values.clpConfig.archive_output.retention_period + .Values.clpConfig.results_cache.retention_period }} {{- include "clp.createPvc" (dict "root" . "component_category" "garbage-collector" @@ -5,3 +7,4 @@ "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") ) }} +{{- end }} From 77743e722247d0abf03faede9088b3a01431549c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 20:20:40 -0500 Subject: [PATCH 127/149] fix(helm): Remove unused `port` field from query_scheduler configuration in values.yaml --- tools/deployment/package-helm/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 603ea06b51..7a674ea5d8 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -49,7 +49,6 @@ clpConfig: type: "celery" # "celery" or "spider" query_scheduler: - port: 7000 jobs_poll_delay: 0.1 # seconds num_archives_to_search_per_sub_job: 16 logging_level: "INFO" From 4e0cd58dd4e26e824b52b66af4493c3898fb2dca Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 20:22:28 -0500 Subject: [PATCH 128/149] feat(helm): Add support for container port 30302 in query-scheduler deployment --- tools/deployment/package-helm/test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index e9f43cda77..417f533f21 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -91,6 +91,9 @@ cat < Date: Thu, 18 Dec 2025 20:25:30 -0500 Subject: [PATCH 129/149] fix(helm): Add missing log_ingestor directory to test script --- tools/deployment/package-helm/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 417f533f21..d8daaf3bec 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -59,7 +59,7 @@ mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ "$CLP_HOME/var/data/"{archives,streams} \ "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ - "$CLP_HOME/var/log/"{garbage_collector,api_server,mcp_server} \ + "$CLP_HOME/var/log/"{garbage_collector,api_server,log_ingestor,mcp_server} \ "$CLP_HOME/var/tmp" \ "$CLP_HOME/samples" From ffcd41297a11aac821e4f383787a6d406e118cb6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 20:38:34 -0500 Subject: [PATCH 130/149] fix(helm): Simplify S3 credentials configuration in webui-deployment template --- .../package-helm/templates/webui-deployment.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index eb2fa25c89..ec6f7adfda 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -59,11 +59,13 @@ spec: value: "4000" - name: "RATE_LIMIT" value: {{ .Values.clpConfig.webui.rate_limit | quote }} - {{- if and (eq .Values.clpConfig.stream_output.storage.type "s3") (eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "credentials") }} + {{- with .Values.clpConfig.stream_output.storage }} + {{- if and (eq .type "s3") (eq .s3_config.aws_authentication.type "credentials") }} - name: "AWS_ACCESS_KEY_ID" - value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} + value: {{ .s3_config.aws_authentication.credentials.access_key_id | quote }} - name: "AWS_SECRET_ACCESS_KEY" - value: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + value: {{ .s3_config.aws_authentication.credentials.secret_access_key | quote }} + {{- end }} {{- end }} ports: - name: "webui" From aee79bcae2539d01dffdfd6524822725bb59c02d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:08:10 -0500 Subject: [PATCH 131/149] fix(helm): Correct placement of readiness and liveness probes in query-scheduler deployment template --- .../package-helm/templates/query-scheduler-deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index b12b6fe5dc..5ebc901a89 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -86,13 +86,13 @@ spec: "--config", "/etc/clp-config.yaml" ] readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} tcpSocket: port: 7000 - {{- include "clp.readinessProbeTimings" . | nindent 12 }} livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: port: 7000 - {{- include "clp.livenessProbeTimings" . | nindent 12 }} volumes: - {{- include "clp.pvcVolume" (dict "root" . From a9cdeddc89bd079cfde406223093f3f7a0aa1a05 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:18:09 -0500 Subject: [PATCH 132/149] fix(helm): Refactor configmap template to improve conditional handling and simplify values mapping --- .../package-helm/templates/configmap.yaml | 164 ++++++++++-------- 1 file changed, 89 insertions(+), 75 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 747f96df08..038f5c7b4f 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -6,51 +6,53 @@ metadata: {{- include "clp.labels" . | nindent 4 }} data: clp-config.yaml: | - {{- if eq .Values.clpConfig.package.storage_engine "clp-s" }} + {{- with .Values.clpConfig.api_server }} api_server: - default_max_num_query_results: {{ - .Values.clpConfig.api_server.default_max_num_query_results }} + default_max_num_query_results: {{ .default_max_num_query_results }} host: "localhost" port: 3001 query_job_polling: - initial_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.initial_backoff_ms }} - max_backoff_ms: {{ .Values.clpConfig.api_server.query_job_polling.max_backoff_ms }} + initial_backoff_ms: {{ .query_job_polling.initial_backoff_ms }} + max_backoff_ms: {{ .query_job_polling.max_backoff_ms }} {{- else }} api_server: null {{- end }} + {{- with .Values.clpConfig.archive_output }} archive_output: - compression_level: {{ .Values.clpConfig.archive_output.compression_level }} - retention_period: {{ .Values.clpConfig.archive_output.retention_period }} + compression_level: {{ .compression_level }} + retention_period: {{ .retention_period }} storage: - {{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} + {{- if eq .storage.type "fs" }} + type: "fs" + directory: "/var/data/archives" + {{- else }} type: "s3" staging_directory: "/var/data/staged-archives" + {{- with .storage.s3_config }} s3_config: - endpoint_url: {{ .Values.clpConfig.archive_output.storage.s3_config.endpoint_url | default "null" }} - region_code: {{ .Values.clpConfig.archive_output.storage.s3_config.region_code | quote }} - bucket: {{ .Values.clpConfig.archive_output.storage.s3_config.bucket | quote }} - key_prefix: {{ .Values.clpConfig.archive_output.storage.s3_config.key_prefix | quote }} + endpoint_url: {{ .endpoint_url | default "null" }} + region_code: {{ .region_code | quote }} + bucket: {{ .bucket | quote }} + key_prefix: {{ .key_prefix | quote }} + {{- with .aws_authentication }} aws_authentication: - type: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.type | quote }} - {{- if eq .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.type "credentials" }} + type: {{ .type | quote }} + {{- if eq .type "credentials" }} credentials: - access_key_id: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} - secret_access_key: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + access_key_id: {{ .credentials.access_key_id | quote }} + secret_access_key: {{ .credentials.secret_access_key | quote }} {{- end }} - {{- if eq .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.type "profile" }} - profile: {{ .Values.clpConfig.archive_output.storage.s3_config.aws_authentication.profile | quote }} + {{- if eq .type "profile" }} + profile: {{ .profile | quote }} {{- end }} - {{- else }} - type: "fs" - directory: "/var/data/archives" - {{- end }} - target_archive_size: {{ .Values.clpConfig.archive_output.target_archive_size | int }} - target_dictionaries_size: {{ .Values.clpConfig.archive_output.target_dictionaries_size - | int }} - target_encoded_file_size: {{ .Values.clpConfig.archive_output.target_encoded_file_size - | int }} - target_segment_size: {{ .Values.clpConfig.archive_output.target_segment_size - | int }} + {{- end }}{{/* with .aws_authentication */}} + {{- end }}{{/* with .storage.s3_config */}} + {{- end }}{{/* if eq .storage.type "fs" */}} + target_archive_size: {{ .target_archive_size | int }} + target_dictionaries_size: {{ .target_dictionaries_size | int }} + target_encoded_file_size: {{ .target_encoded_file_size | int }} + target_segment_size: {{ .target_segment_size | int }} + {{- end }}{{/* with .Values.clpConfig.archive_output */}} aws_config_directory: {{ .Values.clpConfig.aws_config_directory }} compression_scheduler: jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} @@ -76,36 +78,40 @@ data: archive: {{ .Values.clpConfig.garbage_collector.sweep_interval.archive }} search_result: {{ .Values.clpConfig.garbage_collector.sweep_interval.search_result }} logs_directory: "/var/log" + {{- with .Values.clpConfig.logs_input }} logs_input: - {{- if eq .Values.clpConfig.logs_input.type "s3" }} + {{- if eq .type "fs" }} + type: "fs" + directory: "/mnt/logs" + {{- else }} type: "s3" + {{- with .aws_authentication }} aws_authentication: - type: {{ .Values.clpConfig.logs_input.aws_authentication.type | quote }} - {{- if eq .Values.clpConfig.logs_input.aws_authentication.type "credentials" }} + type: {{ .type | quote }} + {{- if eq .type "credentials" }} credentials: - access_key_id: {{ .Values.clpConfig.logs_input.aws_authentication.credentials.access_key_id | quote }} - secret_access_key: {{ .Values.clpConfig.logs_input.aws_authentication.credentials.secret_access_key | quote }} + access_key_id: {{ .credentials.access_key_id | quote }} + secret_access_key: {{ .credentials.secret_access_key | quote }} {{- end }} - {{- if eq .Values.clpConfig.logs_input.aws_authentication.type "profile" }} - profile: {{ .Values.clpConfig.logs_input.aws_authentication.profile | quote }} + {{- if eq .type "profile" }} + profile: {{ .profile | quote }} {{- end }} - {{- else }} - type: "fs" - directory: "/mnt/logs" - {{- end }} - {{- if eq .Values.clpConfig.logs_input.type "s3" }} + {{- end }}{{/* with .aws_authentication */}} + {{- end }}{{/* if eq .type "fs" */}} + {{- end }}{{/* with .Values.clpConfig.logs_input */}} + {{- with .Values.clpConfig.log_ingestor }} log_ingestor: - buffer_flush_threshold: {{ .Values.clpConfig.log_ingestor.buffer_flush_threshold }} - buffer_flush_timeout: {{ .Values.clpConfig.log_ingestor.buffer_flush_timeout }} - channel_capacity: {{ .Values.clpConfig.log_ingestor.channel_capacity }} + buffer_flush_threshold: {{ .buffer_flush_threshold }} + buffer_flush_timeout: {{ .buffer_flush_timeout }} + channel_capacity: {{ .channel_capacity }} host: "localhost" - logging_level: {{ .Values.clpConfig.log_ingestor.logging_level | quote }} + logging_level: {{ .logging_level | quote }} port: 3002 {{- end }} - {{- if .Values.clpConfig.mcp_server }} + {{- with .Values.clpConfig.mcp_server }} mcp_server: host: "localhost" - logging_level: {{ .Values.clpConfig.mcp_server.logging_level | quote }} + logging_level: {{ .logging_level | quote }} port: 8000 {{- else }} mcp_server: null @@ -142,31 +148,37 @@ data: port: 27017 retention_period: {{ .Values.clpConfig.results_cache.retention_period }} stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} + {{- with .Values.clpConfig.stream_output }} stream_output: storage: - {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + {{- if eq .storage.type "fs" }} + type: "fs" + directory: "/var/data/streams" + {{- else }} type: "s3" staging_directory: "/var/data/staged-streams" + {{- with .storage.s3_config }} s3_config: - endpoint_url: {{ .Values.clpConfig.stream_output.storage.s3_config.endpoint_url | default "null" }} - region_code: {{ .Values.clpConfig.stream_output.storage.s3_config.region_code | quote }} - bucket: {{ .Values.clpConfig.stream_output.storage.s3_config.bucket | quote }} - key_prefix: {{ .Values.clpConfig.stream_output.storage.s3_config.key_prefix | quote }} + endpoint_url: {{ .endpoint_url | default "null" }} + region_code: {{ .region_code | quote }} + bucket: {{ .bucket | quote }} + key_prefix: {{ .key_prefix | quote }} + {{- with .aws_authentication }} aws_authentication: - type: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type | quote }} - {{- if eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "credentials" }} + type: {{ .type | quote }} + {{- if eq .type "credentials" }} credentials: - access_key_id: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.access_key_id | quote }} - secret_access_key: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.credentials.secret_access_key | quote }} + access_key_id: {{ .credentials.access_key_id | quote }} + secret_access_key: {{ .credentials.secret_access_key | quote }} {{- end }} - {{- if eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "profile" }} - profile: {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.profile | quote }} + {{- if eq .type "profile" }} + profile: {{ .profile | quote }} {{- end }} - {{- else }} - directory: "/var/data/streams" - type: "fs" - {{- end }} - target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }} + {{- end }}{{/* with .aws_authentication */}} + {{- end }}{{/* with .storage.s3_config */}} + {{- end }}{{/* if eq .storage.type "fs" */}} + target_uncompressed_size: {{ .target_uncompressed_size | int }} + {{- end }}{{/* with .Values.clpConfig.stream_output */}} tmp_directory: "/var/tmp" webui: host: "localhost" @@ -243,21 +255,23 @@ data: {{- else }} "LogsInputRootDir": null, {{- end }} - {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - "StreamFilesDir": null, - "StreamFilesS3Region": {{ .Values.clpConfig.stream_output.storage.s3_config.region_code | quote }}, - "StreamFilesS3PathPrefix": "{{ .Values.clpConfig.stream_output.storage.s3_config.bucket }}/{{ .Values.clpConfig.stream_output.storage.s3_config.key_prefix }}", - {{- else }} + {{- with .Values.clpConfig.stream_output.storage }} + {{- if eq .type "fs" }} "StreamFilesDir": "/var/data/streams", "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, - {{- end }} - {{- if and (eq .Values.clpConfig.stream_output.storage.type "s3") - (eq .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.type "profile") }} - "StreamFilesS3Profile": {{ .Values.clpConfig.stream_output.storage.s3_config.aws_authentication.profile | quote }}, + "StreamFilesS3Profile": null, + {{- else }} + "StreamFilesDir": null, + "StreamFilesS3Region": {{ .s3_config.region_code | quote }}, + "StreamFilesS3PathPrefix": "{{ .s3_config.bucket }}/{{ .s3_config.key_prefix }}", + {{- if eq .s3_config.aws_authentication.type "profile" }} + "StreamFilesS3Profile": {{ .s3_config.aws_authentication.profile | quote }}, {{- else }} "StreamFilesS3Profile": null, {{- end }} + {{- end }}{{/* if eq .type "fs" */}} + {{- end }}{{/* with .Values.clpConfig.stream_output.storage */}} "StreamTargetUncompressedSize": {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }}, "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level }}, @@ -271,9 +285,9 @@ data: {{ .Values.clpConfig.archive_output.target_segment_size | int }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, - {{- if .Values.clpConfig.presto }} - "PrestoHost": {{ .Values.clpConfig.presto.host | quote }}, - "PrestoPort": {{ .Values.clpConfig.presto.port }} + {{- with .Values.clpConfig.presto }} + "PrestoHost": {{ .host | quote }}, + "PrestoPort": {{ .port }} {{- else }} "PrestoHost": null, "PrestoPort": null From 3a5968a163e51cc0724a5217bfaa9dcf7db43fca Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:20:38 -0500 Subject: [PATCH 133/149] fix(helm): Use `with` for aws_config_directory in webui-deployment template to improve readability --- tools/deployment/package-helm/templates/webui-deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index ec6f7adfda..44f588eee3 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -118,9 +118,9 @@ spec: - name: "server-settings" configMap: name: {{ include "clp.fullname" . }}-config - {{- if .Values.clpConfig.aws_config_directory }} + {{- with .Values.clpConfig.aws_config_directory }} - name: "aws-config" hostPath: - path: "{{ .Values.clpConfig.aws_config_directory }}" + path: {{ . | quote }} type: "Directory" {{- end }} From 753d40288530b8d2bb97a8b24d64b2bfb3200fad Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:23:09 -0500 Subject: [PATCH 134/149] fix(helm): Replace `if` with `with` for aws_config_directory in deployment templates for consistency and cleaner syntax --- .../templates/compression-scheduler-deployment.yaml | 4 ++-- .../package-helm/templates/compression-worker-deployment.yaml | 4 ++-- .../package-helm/templates/garbage-collector-deployment.yaml | 4 ++-- .../package-helm/templates/query-worker-deployment.yaml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 97f9c484b8..86d897d0ac 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -108,10 +108,10 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - {{- if .Values.clpConfig.aws_config_directory }} + {{- with .Values.clpConfig.aws_config_directory }} - name: "aws-config" hostPath: - path: "{{ .Values.clpConfig.aws_config_directory }}" + path: {{ . | quote }} type: "Directory" {{- end }} {{- if eq .Values.clpConfig.logs_input.type "fs" }} diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index b48eebf08b..03ec9a290c 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -119,9 +119,9 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - {{- if .Values.clpConfig.aws_config_directory }} + {{- with .Values.clpConfig.aws_config_directory }} - name: "aws-config" hostPath: - path: "{{ .Values.clpConfig.aws_config_directory }}" + path: {{ . | quote }} type: "Directory" {{- end }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 155a99c9f5..29cde9ca1d 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -116,10 +116,10 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - {{- if .Values.clpConfig.aws_config_directory }} + {{- with .Values.clpConfig.aws_config_directory }} - name: "aws-config" hostPath: - path: "{{ .Values.clpConfig.aws_config_directory }}" + path: {{ . | quote }} type: "Directory" {{- end }} {{- end }} diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index fcbb976519..14d8fc301f 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -121,9 +121,9 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config - {{- if .Values.clpConfig.aws_config_directory }} + {{- with .Values.clpConfig.aws_config_directory }} - name: "aws-config" hostPath: - path: "{{ .Values.clpConfig.aws_config_directory }}" + path: {{ . | quote }} type: "Directory" {{- end }} From 896cb315421ea723390263c9eef4781007cbb6be Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:38:30 -0500 Subject: [PATCH 135/149] fix(helm): Update accessModes to ReadWriteOnce in PVC and PV templates for staged-streams and staged-archives --- .../package-helm/templates/api-server-staged-streams-pv.yaml | 2 +- .../package-helm/templates/api-server-staged-streams-pvc.yaml | 2 +- .../templates/compression-worker-staged-archives-pv.yaml | 2 +- .../templates/compression-worker-staged-archives-pvc.yaml | 2 +- .../package-helm/templates/query-worker-staged-streams-pv.yaml | 2 +- .../package-helm/templates/query-worker-staged-streams-pvc.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml index 0d191046d2..40c81da7a4 100644 --- a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml @@ -6,7 +6,7 @@ "name" "staged-streams" "nodeRole" "control-plane" "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") + "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) ) }} {{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml index 0979cd49aa..e695d9dcee 100644 --- a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml @@ -5,6 +5,6 @@ "component_category" "api-server" "name" "staged-streams" "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") + "accessModes" (list "ReadWriteOnce") ) }} {{- end }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index 924fe4639b..eb69332671 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -4,6 +4,6 @@ "name" "staged-archives" "nodeRole" "control-plane" "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") + "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/staged-archives" .Values.clpConfig.data_directory) ) }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml index d96f2a62e1..5f6f3a346a 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -3,5 +3,5 @@ "component_category" "compression-worker" "name" "staged-archives" "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") + "accessModes" (list "ReadWriteOnce") ) }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index e443b1ace9..c3f14f6b02 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -5,7 +5,7 @@ "name" "staged-streams" "nodeRole" "control-plane" "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") + "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) ) }} {{- end }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml index c433a8b2ea..e909a57b21 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -4,6 +4,6 @@ "component_category" "query-worker" "name" "staged-streams" "capacity" "20Gi" - "accessModes" (list "ReadWriteMany") + "accessModes" (list "ReadWriteOnce") ) }} {{- end }} From 5c81b0fc206b3f8fdaa87191613846ce8de7d84c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:49:31 -0500 Subject: [PATCH 136/149] fix(helm): Add missing quotes for host fields in configmap template to ensure proper YAML formatting --- .../deployment/package-helm/templates/configmap.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 038f5c7b4f..026726e4a6 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -66,7 +66,7 @@ data: database: auto_commit: false compress: true - host: {{ include "clp.fullname" . }}-database + host: "{{ include "clp.fullname" . }}-database" names: clp: {{ .Values.clpConfig.database.names.clp | quote }} port: 3306 @@ -121,7 +121,7 @@ data: storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} presto: {{ .Values.clpConfig.presto }} query_scheduler: - host: {{ include "clp.fullname" . }}-query-scheduler + host: "{{ include "clp.fullname" . }}-query-scheduler" jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} num_archives_to_search_per_sub_job: {{ @@ -130,21 +130,21 @@ data: query_worker: logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }} queue: - host: {{ include "clp.fullname" . }}-queue + host: "{{ include "clp.fullname" . }}-queue" port: 5672 redis: compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} - host: {{ include "clp.fullname" . }}-redis + host: "{{ include "clp.fullname" . }}-redis" port: 6379 query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} reducer: base_port: 14009 - host: {{ include "clp.fullname" . }}-reducer + host: "{{ include "clp.fullname" . }}-reducer" logging_level: {{ .Values.clpConfig.reducer.logging_level | quote }} upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval }} results_cache: db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} - host: {{ include "clp.fullname" . }}-results-cache + host: "{{ include "clp.fullname" . }}-results-cache" port: 27017 retention_period: {{ .Values.clpConfig.results_cache.retention_period }} stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} From f9a507de187fb007d18eceb21ab8912146cfeac4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:50:37 -0500 Subject: [PATCH 137/149] fix(helm): Remove S3-specific PVC/PV and volume mounts for staged-streams in API server deployment --- .../templates/api-server-deployment.yaml | 16 ++-------------- .../templates/api-server-staged-streams-pv.yaml | 12 ------------ .../templates/api-server-staged-streams-pvc.yaml | 10 ---------- 3 files changed, 2 insertions(+), 36 deletions(-) delete mode 100644 tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml delete mode 100644 tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 40286036fc..58add1c665 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -67,13 +67,7 @@ spec: "name" "logs" ) | quote }} mountPath: "/var/log/api_server" - {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - - name: {{ include "clp.volumeName" (dict - "component_category" "api-server" - "name" "staged-streams" - ) | quote }} - mountPath: "/var/data/staged-streams" - {{- else }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" "name" "streams" @@ -100,13 +94,7 @@ spec: "component_category" "api-server" "name" "logs" ) | nindent 10 }} - {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "api-server" - "name" "staged-streams" - ) | nindent 10 }} - {{- else }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml deleted file mode 100644 index 40c81da7a4..0000000000 --- a/tools/deployment/package-helm/templates/api-server-staged-streams-pv.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if and .Values.clpConfig.api_server - (eq .Values.clpConfig.stream_output.storage.type "s3") }} -{{- include "clp.createLocalPv" (dict - "root" . - "component_category" "api-server" - "name" "staged-streams" - "nodeRole" "control-plane" - "capacity" "20Gi" - "accessModes" (list "ReadWriteOnce") - "hostPath" (printf "%s/staged-streams" .Values.clpConfig.data_directory) -) }} -{{- end }} diff --git a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml deleted file mode 100644 index e695d9dcee..0000000000 --- a/tools/deployment/package-helm/templates/api-server-staged-streams-pvc.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{- if and .Values.clpConfig.api_server - (eq .Values.clpConfig.stream_output.storage.type "s3") }} -{{- include "clp.createPvc" (dict - "root" . - "component_category" "api-server" - "name" "staged-streams" - "capacity" "20Gi" - "accessModes" (list "ReadWriteOnce") -) }} -{{- end }} From ee248a1b508942f8d5b7cb63d8506b35fed8e4b4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 21:53:56 -0500 Subject: [PATCH 138/149] fix(helm): Add `| quote` and handle null case for aws_config_directory in configmap template --- tools/deployment/package-helm/templates/configmap.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 026726e4a6..831f6cd7f0 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -53,7 +53,11 @@ data: target_encoded_file_size: {{ .target_encoded_file_size | int }} target_segment_size: {{ .target_segment_size | int }} {{- end }}{{/* with .Values.clpConfig.archive_output */}} - aws_config_directory: {{ .Values.clpConfig.aws_config_directory }} + {{- with .Values.clpConfig.aws_config_directory }} + aws_config_directory: {{ . | quote }} + {{- else }} + aws_config_directory: null + {{- end }} compression_scheduler: jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} From be70bd34cb02499e26f7fab054911844a02dac2a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:01:23 -0500 Subject: [PATCH 139/149] fix(helm): Add conditional for logsInputVolumeMount in webui-deployment template --- tools/deployment/package-helm/templates/webui-deployment.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 44f588eee3..a10e393f6c 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -84,7 +84,9 @@ spec: mountPath: "/opt/clp/.aws" readOnly: true {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} + {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -104,7 +106,9 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: *webui-health-check volumes: + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . From c346734be1652e2efb1dfb4bc5d0a8fc59057608 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:01:27 -0500 Subject: [PATCH 140/149] fix(helm): Add conditional for logsInputVolume in compression-scheduler deployment template --- .../templates/compression-scheduler-deployment.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 86d897d0ac..f49fa6c3b9 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -94,7 +94,9 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: + {{- if eq .Values.clpConfig.logs_input.type "fs" }} - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "compression-scheduler" @@ -114,9 +116,3 @@ spec: path: {{ . | quote }} type: "Directory" {{- end }} - {{- if eq .Values.clpConfig.logs_input.type "fs" }} - - name: "logs-input" - hostPath: - path: "{{ .Values.clpConfig.logs_input.directory }}" - type: "Directory" - {{- end }} From 80251dcb451777ce3f68476dccacb9a03bd4645c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:03:58 -0500 Subject: [PATCH 141/149] fix(helm): Correct placement of logsInputVolume and pvcVolume conditionals in deployment templates --- .../compression-scheduler-deployment.yaml | 6 +++--- .../templates/webui-deployment.yaml | 20 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index f49fa6c3b9..5a93bb721c 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -94,9 +94,6 @@ spec: "--config", "/etc/clp-config.yaml" ] volumes: - {{- if eq .Values.clpConfig.logs_input.type "fs" }} - - {{- include "clp.logsInputVolume" . | nindent 10 }} - {{- end }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "compression-scheduler" @@ -116,3 +113,6 @@ spec: path: {{ . | quote }} type: "Directory" {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index a10e393f6c..80e5376a58 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -106,16 +106,6 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: *webui-health-check volumes: - {{- if eq .Values.clpConfig.logs_input.type "fs" }} - - {{- include "clp.logsInputVolume" . | nindent 10 }} - {{- end }} - {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "streams" - ) | nindent 10 }} - {{- end }} - name: "client-settings" configMap: name: {{ include "clp.fullname" . }}-config @@ -128,3 +118,13 @@ spec: path: {{ . | quote }} type: "Directory" {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "streams" + ) | nindent 10 }} + {{- end }} From 82dc4d7972b3b704e58a166fdfc09b5df2da7403 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:17:50 -0500 Subject: [PATCH 142/149] fix(helm): Add int casting for numerical values in configmap template to ensure proper YAML formatting --- .../package-helm/templates/configmap.yaml | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 831f6cd7f0..b4e104ae6f 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -8,19 +8,23 @@ data: clp-config.yaml: | {{- with .Values.clpConfig.api_server }} api_server: - default_max_num_query_results: {{ .default_max_num_query_results }} + default_max_num_query_results: {{ .default_max_num_query_results | int }} host: "localhost" port: 3001 query_job_polling: - initial_backoff_ms: {{ .query_job_polling.initial_backoff_ms }} - max_backoff_ms: {{ .query_job_polling.max_backoff_ms }} + initial_backoff_ms: {{ .query_job_polling.initial_backoff_ms | int }} + max_backoff_ms: {{ .query_job_polling.max_backoff_ms | int }} {{- else }} api_server: null {{- end }} {{- with .Values.clpConfig.archive_output }} archive_output: - compression_level: {{ .compression_level }} - retention_period: {{ .retention_period }} + compression_level: {{ .compression_level | int }} + {{ with .retention_period }} + retention_period: {{ . | int }} + {{ else }} + retention_period: null + {{ end }} storage: {{- if eq .storage.type "fs" }} type: "fs" @@ -62,7 +66,7 @@ data: jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} max_concurrent_tasks_per_job: {{ - .Values.clpConfig.compression_scheduler.max_concurrent_tasks_per_job }} + .Values.clpConfig.compression_scheduler.max_concurrent_tasks_per_job | int }} type: {{ .Values.clpConfig.compression_scheduler.type | quote }} compression_worker: logging_level: {{ .Values.clpConfig.compression_worker.logging_level | quote }} @@ -79,8 +83,8 @@ data: garbage_collector: logging_level: {{ .Values.clpConfig.garbage_collector.logging_level | quote }} sweep_interval: - archive: {{ .Values.clpConfig.garbage_collector.sweep_interval.archive }} - search_result: {{ .Values.clpConfig.garbage_collector.sweep_interval.search_result }} + archive: {{ .Values.clpConfig.garbage_collector.sweep_interval.archive | int }} + search_result: {{ .Values.clpConfig.garbage_collector.sweep_interval.search_result | int }} logs_directory: "/var/log" {{- with .Values.clpConfig.logs_input }} logs_input: @@ -105,9 +109,9 @@ data: {{- end }}{{/* with .Values.clpConfig.logs_input */}} {{- with .Values.clpConfig.log_ingestor }} log_ingestor: - buffer_flush_threshold: {{ .buffer_flush_threshold }} - buffer_flush_timeout: {{ .buffer_flush_timeout }} - channel_capacity: {{ .channel_capacity }} + buffer_flush_threshold: {{ .buffer_flush_threshold | int }} + buffer_flush_timeout: {{ .buffer_flush_timeout | int }} + channel_capacity: {{ .channel_capacity | int }} host: "localhost" logging_level: {{ .logging_level | quote }} port: 3002 @@ -129,7 +133,7 @@ data: jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }} num_archives_to_search_per_sub_job: {{ - .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job }} + .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job | int }} port: 7000 query_worker: logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }} @@ -137,20 +141,24 @@ data: host: "{{ include "clp.fullname" . }}-queue" port: 5672 redis: - compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database }} + compression_backend_database: {{ .Values.clpConfig.redis.compression_backend_database | int }} host: "{{ include "clp.fullname" . }}-redis" port: 6379 - query_backend_database: {{ .Values.clpConfig.redis.query_backend_database }} + query_backend_database: {{ .Values.clpConfig.redis.query_backend_database | int }} reducer: base_port: 14009 host: "{{ include "clp.fullname" . }}-reducer" logging_level: {{ .Values.clpConfig.reducer.logging_level | quote }} - upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval }} + upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval | int }} results_cache: db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} host: "{{ include "clp.fullname" . }}-results-cache" port: 27017 - retention_period: {{ .Values.clpConfig.results_cache.retention_period }} + {{ with .Values.clpConfig.results_cache.retention_period }} + retention_period: {{ . | int }} + {{ else }} + retention_period: null + {{ end }} stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} {{- with .Values.clpConfig.stream_output }} stream_output: @@ -187,7 +195,7 @@ data: webui: host: "localhost" port: 4000 - rate_limit: {{ .Values.clpConfig.webui.rate_limit }} + rate_limit: {{ .Values.clpConfig.webui.rate_limit | int }} results_metadata_collection_name: {{ .Values.clpConfig.webui.results_metadata_collection_name | quote }} @@ -278,7 +286,7 @@ data: {{- end }}{{/* with .Values.clpConfig.stream_output.storage */}} "StreamTargetUncompressedSize": {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }}, - "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level }}, + "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level | int }}, "ArchiveOutputTargetArchiveSize": {{ .Values.clpConfig.archive_output.target_archive_size | int }}, "ArchiveOutputTargetDictionariesSize": @@ -291,7 +299,7 @@ data: "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, {{- with .Values.clpConfig.presto }} "PrestoHost": {{ .host | quote }}, - "PrestoPort": {{ .port }} + "PrestoPort": {{ .port | int }} {{- else }} "PrestoHost": null, "PrestoPort": null From 870944375d13ded6e03d2aea9ae6ee7e3c371385 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:25:39 -0500 Subject: [PATCH 143/149] fix(helm): Adjust volume mounts and conditionals for staged archives and streams in worker deployments --- .../compression-worker-deployment.yaml | 52 +++++++++-------- ...compression-worker-staged-archives-pv.yaml | 2 + ...ompression-worker-staged-archives-pvc.yaml | 2 + .../templates/query-worker-deployment.yaml | 58 +++++++++---------- tools/deployment/package-helm/test.sh | 2 +- 5 files changed, 62 insertions(+), 54 deletions(-) diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 03ec9a290c..d8d3308353 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -45,30 +45,20 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "compression-worker" "name" "logs" ) | quote }} mountPath: "/var/log/compression_worker" - - name: {{ include "clp.volumeName" (dict - "component_category" "compression-worker" - "name" "staged-archives" - ) | quote }} - mountPath: "/var/data/staged-archives" - name: {{ include "clp.volumeName" (dict "component_category" "compression-worker" "name" "tmp" ) | quote }} mountPath: "/var/tmp" - {{- if .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - mountPath: "/opt/clp/.aws" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - {{- end }} {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -76,9 +66,21 @@ spec: ) | quote }} mountPath: "/var/data/archives" {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} {{- if eq .Values.clpConfig.logs_input.type "fs" }} - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} {{- end }} + {{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} + - name: {{ include "clp.volumeName" (dict + "component_category" "compression-worker" + "name" "staged-archives" + ) | quote }} + mountPath: "/var/data/staged-archives" + {{- end }} command: [ "python3", "-u", "/opt/clp/lib/python3/site-packages/bin/celery", @@ -91,24 +93,19 @@ spec: "-n", "compression-worker" ] volumes: - {{- if eq .Values.clpConfig.logs_input.type "fs" }} - - {{- include "clp.logsInputVolume" . | nindent 10 }} - {{- end }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "compression-worker" "name" "logs" ) | nindent 10 }} - - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "compression-worker" - "name" "staged-archives" - ) | nindent 10 }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "compression-worker" "name" "tmp" ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . @@ -116,12 +113,19 @@ spec: "name" "archives" ) | nindent 10 }} {{- end }} - - name: "config" - configMap: - name: {{ include "clp.fullname" . }}-config {{- with .Values.clpConfig.aws_config_directory }} - name: "aws-config" hostPath: path: {{ . | quote }} type: "Directory" {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-worker" + "name" "staged-archives" + ) | nindent 10 }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index eb69332671..c00226b797 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "compression-worker" @@ -7,3 +8,4 @@ "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/staged-archives" .Values.clpConfig.data_directory) ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml index 5f6f3a346a..b8a9367236 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} {{- include "clp.createPvc" (dict "root" . "component_category" "compression-worker" @@ -5,3 +6,4 @@ "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 14d8fc301f..cdac3c390e 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -45,15 +45,15 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "query-worker" "name" "logs" ) | quote }} mountPath: "/var/log/query_worker" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -61,12 +61,10 @@ spec: ) | quote }} mountPath: "/var/data/archives" {{- end }} - {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - - name: {{ include "clp.volumeName" (dict - "component_category" "shared-data" - "name" "streams" - ) | quote }} - mountPath: "/var/data/streams" + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - name: {{ include "clp.volumeName" (dict @@ -75,10 +73,12 @@ spec: ) | quote }} mountPath: "/var/data/staged-streams" {{- end }} - {{- if .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - mountPath: "/opt/clp/.aws" - readOnly: true + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} + - name: {{ include "clp.volumeName" (dict + "component_category" "shared-data" + "name" "streams" + ) | quote }} + mountPath: "/var/data/streams" {{- end }} command: [ "python3", "-u", @@ -97,18 +97,27 @@ spec: "component_category" "query-worker" "name" "logs" ) | nindent 10 }} - {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . - "component_category" "query-worker" - "name" "staged-streams" + "component_category" "shared-data" + "name" "archives" ) | nindent 10 }} {{- end }} - {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} + {{- with .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: {{ . | quote }} + type: "Directory" + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} - {{- include "clp.pvcVolume" (dict "root" . - "component_category" "shared-data" - "name" "archives" + "component_category" "query-worker" + "name" "staged-streams" ) | nindent 10 }} {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} @@ -118,12 +127,3 @@ spec: "name" "streams" ) | nindent 10 }} {{- end }} - - name: "config" - configMap: - name: {{ include "clp.fullname" . }}-config - {{- with .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - hostPath: - path: {{ . | quote }} - type: "Directory" - {{- end }} diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index d8daaf3bec..69b8139904 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -56,7 +56,7 @@ wait_for_pods() { kind delete cluster --name clp-test rm -rf "$CLP_HOME" mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ - "$CLP_HOME/var/data/"{archives,streams} \ + "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \ "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ "$CLP_HOME/var/log/"{garbage_collector,api_server,log_ingestor,mcp_server} \ From 7443e4e3cefb4315e05bca5518dd5432dd418515 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:28:46 -0500 Subject: [PATCH 144/149] fix(helm): Correct placement of config volume mounts across deployment templates for consistency --- .../templates/api-server-deployment.yaml | 14 ++++---- .../compression-scheduler-deployment.yaml | 8 ++--- .../garbage-collector-deployment.yaml | 34 +++++++++---------- .../templates/log-ingestor-deployment.yaml | 8 ++--- .../templates/mcp-server-deployment.yaml | 8 ++--- .../templates/query-scheduler-deployment.yaml | 8 ++--- .../templates/reducer-deployment.yaml | 8 ++--- 7 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 58add1c665..26da76486f 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -58,15 +58,15 @@ spec: - name: "api-server" containerPort: 3001 volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "api-server" "name" "logs" ) | quote }} mountPath: "/var/log/api_server" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -94,6 +94,9 @@ spec: "component_category" "api-server" "name" "logs" ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . @@ -101,7 +104,4 @@ spec: "name" "streams" ) | nindent 10 }} {{- end }} - - name: "config" - configMap: - name: {{ include "clp.fullname" . }}-config {{- end }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index 5a93bb721c..37a86ca019 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -66,10 +66,6 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "compression-scheduler" "name" "logs" @@ -80,6 +76,10 @@ spec: "name" "user-logs" ) | quote }} mountPath: "/var/log/user" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true {{- if .Values.clpConfig.aws_config_directory }} - name: "aws-config" mountPath: "/opt/clp/.aws" diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 29cde9ca1d..f50d0e4e57 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -60,20 +60,15 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "garbage-collector" "name" "logs" ) | quote }} mountPath: "/var/log/garbage_collector" - {{- if .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - mountPath: "/opt/clp/.aws" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" readOnly: true - {{- end }} {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -81,6 +76,11 @@ spec: ) | quote }} mountPath: "/var/data/archives" {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + mountPath: "/opt/clp/.aws" + readOnly: true + {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" @@ -99,6 +99,9 @@ spec: "component_category" "garbage-collector" "name" "logs" ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . @@ -106,6 +109,12 @@ spec: "name" "archives" ) | nindent 10 }} {{- end }} + {{- with .Values.clpConfig.aws_config_directory }} + - name: "aws-config" + hostPath: + path: {{ . | quote }} + type: "Directory" + {{- end }} {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . @@ -113,13 +122,4 @@ spec: "name" "streams" ) | nindent 10 }} {{- end }} - - name: "config" - configMap: - name: {{ include "clp.fullname" . }}-config - {{- with .Values.clpConfig.aws_config_directory }} - - name: "aws-config" - hostPath: - path: {{ . | quote }} - type: "Directory" - {{- end }} {{- end }} diff --git a/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml b/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml index 4c4946a892..bf753c7636 100644 --- a/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml +++ b/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml @@ -53,15 +53,15 @@ spec: - name: "log-ingestor" containerPort: 3002 volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "log-ingestor" "name" "logs" ) | quote }} mountPath: "/var/log/log_ingestor" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true command: [ "/opt/clp/bin/log-ingestor", "--config", "/etc/clp-config.yaml", diff --git a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml index 1238602ba3..09244b1b6b 100644 --- a/tools/deployment/package-helm/templates/mcp-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-deployment.yaml @@ -60,15 +60,15 @@ spec: - name: "mcp-server" containerPort: 8000 volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "mcp-server" "name" "logs" ) | quote }} mountPath: "/var/log/mcp_server" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true command: [ "python3", "-u", "-m", "clp_mcp_server.clp_mcp_server", diff --git a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml index 5ebc901a89..b42c6546da 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-deployment.yaml @@ -71,15 +71,15 @@ spec: - name: "query-scheduler" containerPort: 7000 volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "query-scheduler" "name" "logs" ) | quote }} mountPath: "/var/log/query_scheduler" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true command: [ "python3", "-u", "-m", "job_orchestration.scheduler.query.query_scheduler", diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index 09954063b5..51970785f7 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -50,15 +50,15 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - name: {{ include "clp.volumeName" (dict "component_category" "reducer" "name" "logs" ) | quote }} mountPath: "/var/log/reducer" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true command: [ "python3", "-u", "-m", "job_orchestration.reducer.reducer", From b32613b621e07a4f69e1e798687765ad38f03d6b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 18 Dec 2025 22:32:26 -0500 Subject: [PATCH 145/149] chore(helm): Bump chart version to 0.1.2-dev.8 --- tools/deployment/package-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 1da0d1598e..d26da895a4 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.1.2-dev.7" +version: "0.1.2-dev.8" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.7.1-dev" From 3b411a7a1eb2635c31dfc1f7ea2c16553a4526e9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Dec 2025 15:43:27 -0500 Subject: [PATCH 146/149] fix(helm): Handle null case for log_ingestor and update default clpConfig values. --- .../package-helm/templates/configmap.yaml | 2 ++ tools/deployment/package-helm/values.yaml | 23 ++++--------------- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index b4e104ae6f..881dfbb972 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -115,6 +115,8 @@ data: host: "localhost" logging_level: {{ .logging_level | quote }} port: 3002 + {{- else }} + log_ingestor: null {{- end }} {{- with .Values.clpConfig.mcp_server }} mcp_server: diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 7a674ea5d8..5fbee3177c 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -22,8 +22,8 @@ workerConcurrency: 8 clpConfig: package: - storage_engine: "clp-s" - query_engine: "clp-s" + storage_engine: "clp" + query_engine: "clp" # Location (e.g., directory) containing any logs you wish to compress. Must be reachable by all # workers. @@ -133,25 +133,10 @@ clpConfig: search_result: 30 # API server config - api_server: - port: 30301 - default_max_num_query_results: 1000 - query_job_polling: - initial_backoff_ms: 100 - max_backoff_ms: 5000 + api_server: null # log-ingestor config. Currently, the config is applicable only if `logs_input.type` is "s3". - log_ingestor: - port: 30302 - # The timeout (in seconds) after which the log buffer is flushed for compression if no new - # input arrives. - buffer_flush_timeout: 300 - # The log buffer size (in bytes) that triggers a flush for compression. - buffer_flush_threshold: 268435456 # 256 MiB - # The capacity of the internal channel used for communication between an ingestion job and the - # log buffer. - channel_capacity: 10 - logging_level: "INFO" + log_ingestor: null # Presto client config presto: null From f4272266756ceb4e232c1e88b1f628b09be24447 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 5 Jan 2026 18:34:29 -0500 Subject: [PATCH 147/149] WIP - feat(helm): Add Fluent Bit configuration for operational logging and display logs in the Web UI. --- components/api-server/src/bin/api_server.rs | 20 +- .../clp_mcp_server/clp_mcp_server.py | 10 +- .../clp_package_utils/controller.py | 25 ++ .../clp-py-utils/clp_py_utils/clp_config.py | 1 + .../executor/compress/compression_task.py | 46 +-- .../executor/query/extract_stream_task.py | 9 +- .../executor/query/fs_search_task.py | 9 +- .../job_orchestration/executor/query/utils.py | 26 +- .../archive_garbage_collector.py | 6 +- .../garbage_collector/garbage_collector.py | 9 +- .../search_result_garbage_collector.py | 6 +- .../garbage_collector/utils.py | 9 +- .../job_orchestration/reducer/reducer.py | 21 +- .../compress/compression_scheduler.py | 8 +- .../scheduler/query/query_scheduler.py | 8 +- .../log-ingestor/src/bin/log_ingestor.rs | 17 +- .../src/etc/fluent-bit/filters.conf | 8 + .../src/etc/fluent-bit/fluent-bit.conf | 20 ++ .../src/etc/fluent-bit/outputs.conf | 69 +++++ .../src/etc/fluent-bit/parsers.conf | 11 + .../components/Layout/MainLayout.module.css | 14 + .../src/components/Layout/MainLayout.tsx | 30 +- .../src/components/VirtualTable/index.tsx | 39 ++- .../SearchResultsTable/index.tsx | 4 +- .../SearchResultsTable/typings.ts | 2 +- components/webui/client/src/router.tsx | 2 + components/webui/server/settings.json | 4 +- docs/src/user-docs/guides-multi-host.md | 43 +++ .../package-helm/templates/configmap.yaml | 13 +- .../templates/fluent-bit-clusterrole.yaml | 20 ++ .../fluent-bit-clusterrolebinding.yaml | 19 ++ .../templates/fluent-bit-configmap.yaml | 287 ++++++++++++++++++ .../templates/fluent-bit-daemonset.yaml | 128 ++++++++ .../fluent-bit-operational-logs-pv.yaml | 13 + .../fluent-bit-operational-logs-pvc.yaml | 11 + .../templates/fluent-bit-service.yaml | 25 ++ .../templates/fluent-bit-serviceaccount.yaml | 11 + .../templates/webui-deployment.yaml | 19 ++ tools/deployment/package-helm/test.sh | 1 + tools/deployment/package-helm/values.yaml | 33 ++ .../package/docker-compose-all.yaml | 163 +++++++--- .../package/docker-compose-base.yaml | 5 + .../package/docker-compose-spider-base.yaml | 5 + .../package/docker-compose-spider.yaml | 5 + tools/deployment/package/docker-compose.yaml | 5 + 45 files changed, 1046 insertions(+), 193 deletions(-) create mode 100644 components/package-template/src/etc/fluent-bit/filters.conf create mode 100644 components/package-template/src/etc/fluent-bit/fluent-bit.conf create mode 100644 components/package-template/src/etc/fluent-bit/outputs.conf create mode 100644 components/package-template/src/etc/fluent-bit/parsers.conf create mode 100644 tools/deployment/package-helm/templates/fluent-bit-clusterrole.yaml create mode 100644 tools/deployment/package-helm/templates/fluent-bit-clusterrolebinding.yaml create mode 100644 tools/deployment/package-helm/templates/fluent-bit-configmap.yaml create mode 100644 tools/deployment/package-helm/templates/fluent-bit-daemonset.yaml create mode 100644 tools/deployment/package-helm/templates/fluent-bit-operational-logs-pv.yaml create mode 100644 tools/deployment/package-helm/templates/fluent-bit-operational-logs-pvc.yaml create mode 100644 tools/deployment/package-helm/templates/fluent-bit-service.yaml create mode 100644 tools/deployment/package-helm/templates/fluent-bit-serviceaccount.yaml diff --git a/components/api-server/src/bin/api_server.rs b/components/api-server/src/bin/api_server.rs index 739dc1e0c9..b365f6cd83 100644 --- a/components/api-server/src/bin/api_server.rs +++ b/components/api-server/src/bin/api_server.rs @@ -1,11 +1,7 @@ use anyhow::Context; use clap::Parser; use clp_rust_utils::{clp_config::package, serde::yaml}; -use tracing_appender::{ - non_blocking::WorkerGuard, - rolling::{RollingFileAppender, Rotation}, -}; -use tracing_subscriber::{self, fmt::writer::MakeWriterExt}; +use tracing_subscriber; #[derive(Parser)] #[command(version, about = "API Server for CLP.")] @@ -42,13 +38,8 @@ fn read_config_and_credentials( Ok((config, credentials)) } -fn set_up_logging() -> anyhow::Result { - let logs_directory = - std::env::var("CLP_LOGS_DIR").context("Expect `CLP_LOGS_DIR` environment variable.")?; - let logs_directory = std::path::Path::new(logs_directory.as_str()); - let file_appender = - RollingFileAppender::new(Rotation::HOURLY, logs_directory, "api_server.log"); - let (non_blocking_writer, guard) = tracing_appender::non_blocking(file_appender); +fn set_up_logging() { + // Logs to stdout only - Docker's fluentd driver captures and forwards to Fluent Bit tracing_subscriber::fmt() .event_format( tracing_subscriber::fmt::format() @@ -60,9 +51,8 @@ fn set_up_logging() -> anyhow::Result { ) .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .with_ansi(false) - .with_writer(std::io::stdout.and(non_blocking_writer)) + .with_writer(std::io::stdout) .init(); - Ok(guard) } async fn shutdown_signal() { @@ -81,7 +71,7 @@ async fn main() -> anyhow::Result<()> { let args = Args::parse(); let (config, credentials) = read_config_and_credentials(&args)?; - let _guard = set_up_logging()?; + set_up_logging(); let addr = format!( "{}:{}", diff --git a/components/clp-mcp-server/clp_mcp_server/clp_mcp_server.py b/components/clp-mcp-server/clp_mcp_server/clp_mcp_server.py index 413ff9f620..03148509fd 100644 --- a/components/clp-mcp-server/clp_mcp_server/clp_mcp_server.py +++ b/components/clp-mcp-server/clp_mcp_server/clp_mcp_server.py @@ -1,15 +1,13 @@ """CLP MCP Server entry point.""" import ipaddress -import logging import os import socket import sys -from pathlib import Path import click from clp_py_utils.clp_config import ClpConfig, MCP_SERVER_COMPONENT_NAME -from clp_py_utils.clp_logging import get_logger, get_logging_formatter, set_logging_level +from clp_py_utils.clp_logging import get_logger, set_logging_level from clp_py_utils.core import read_yaml_config_file from pydantic import ValidationError @@ -38,11 +36,7 @@ def main(host: str, port: int, config_path: Path) -> int: :param config_path: The path to server's configuration file. :return: Exit code (0 for success, non-zero for failure). """ - # Setup logging to file - log_file_path = Path(os.getenv("CLP_LOGS_DIR")) / "mcp_server.log" - logging_file_handler = logging.FileHandler(filename=log_file_path, encoding="utf-8") - logging_file_handler.setFormatter(get_logging_formatter()) - logger.addHandler(logging_file_handler) + # Set logging level from environment set_logging_level(logger, os.getenv("CLP_LOGGING_LEVEL")) exit_code = 0 diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 31e6b93d75..ecab3950a9 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -32,6 +32,7 @@ DatabaseEngine, DB_COMPONENT_NAME, DeploymentType, + FLUENT_BIT_COMPONENT_NAME, GARBAGE_COLLECTOR_COMPONENT_NAME, LOG_INGESTOR_COMPONENT_NAME, MCP_SERVER_COMPONENT_NAME, @@ -711,6 +712,7 @@ def _set_up_env_for_webui(self, container_clp_config: ClpConfig) -> EnvVarsDict: "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), "StreamTargetUncompressedSize": self._clp_config.stream_output.target_uncompressed_size, "ClpQueryEngine": self._clp_config.package.query_engine, + "OperationalLogsDir": "/var/log/clp", } stream_storage = self._clp_config.stream_output.storage @@ -852,6 +854,28 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: return env_vars + def _set_up_env_for_fluent_bit(self) -> EnvVarsDict: + """ + Sets up environment variables for the Fluent Bit component. + + Fluent Bit receives logs from Docker's fluentd logging driver and writes + them to the logs directory for hot tier access. + + :return: Dictionary of environment variables necessary to launch the component. + """ + component_name = FLUENT_BIT_COMPONENT_NAME + logger.info(f"Setting up environment for {component_name}...") + + env_vars = EnvVarsDict() + + # Paths + fluent_bit_conf_dir = self._conf_dir / "fluent-bit" + env_vars |= { + "CLP_FLUENT_BIT_CONF_DIR_HOST": str(fluent_bit_conf_dir), + } + + return env_vars + def _read_and_update_settings_json( self, settings_file_path: pathlib.Path, updates: dict[str, Any] ) -> dict[str, Any]: @@ -997,6 +1021,7 @@ def set_up_env(self) -> None: env_vars |= self._set_up_env_for_webui(container_clp_config) env_vars |= self._set_up_env_for_mcp_server() env_vars |= self._set_up_env_for_garbage_collector() + env_vars |= self._set_up_env_for_fluent_bit() # Write the environment variables to the `.env` file. with open(f"{self._clp_home}/.env", "w") as env_file: diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 4edb94b4f0..88658be9a6 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -45,6 +45,7 @@ WEBUI_COMPONENT_NAME = "webui" MCP_SERVER_COMPONENT_NAME = "mcp_server" GARBAGE_COLLECTOR_COMPONENT_NAME = "garbage_collector" +FLUENT_BIT_COMPONENT_NAME = "fluent_bit" # Action names ARCHIVE_MANAGER_ACTION_NAME = "archive_manager" diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index 89f9965301..fd849a285c 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -347,7 +347,6 @@ def run_clp( worker_config: WorkerConfig, clp_config: ClpIoConfig, clp_home: pathlib.Path, - logs_dir: pathlib.Path, job_id: int, task_id: int, tag_ids: list[int], @@ -362,7 +361,6 @@ def run_clp( :param worker_config: WorkerConfig :param clp_config: ClpIoConfig :param clp_home: - :param logs_dir: :param job_id: :param task_id: :param tag_ids: @@ -444,17 +442,22 @@ def cleanup_temporary_files(): if converted_inputs_dir is not None: shutil.rmtree(converted_inputs_dir) - # Open stderr log file - stderr_log_path = logs_dir / f"{instance_id_str}-stderr.log" - stderr_log_file = open(stderr_log_path, "w") + def log_stderr(stderr_data: bytes, process_name: str): + """Log stderr content through the logger (captured by Docker's fluentd driver).""" + stderr_text = stderr_data.decode("utf-8").strip() + if stderr_text: + for line in stderr_text.split("\n"): + logger.info(f"[{process_name} stderr] {line}") conversion_return_code = 0 if conversion_cmd is not None: logger.debug("Execute log-converter with command: %s", conversion_cmd) conversion_proc = subprocess.Popen( - conversion_cmd, stdout=subprocess.DEVNULL, stderr=stderr_log_file, env=conversion_env + conversion_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, env=conversion_env ) - conversion_return_code = conversion_proc.wait() + _, stderr_data = conversion_proc.communicate() + conversion_return_code = conversion_proc.returncode + log_stderr(stderr_data, "log-converter") if conversion_return_code != 0: cleanup_temporary_files() @@ -464,16 +467,15 @@ def cleanup_temporary_files(): worker_output = { "total_uncompressed_size": 0, "total_compressed_size": 0, - "error_message": f"Check logs in {stderr_log_path}", + "error_message": "See worker logs (captured by Fluent Bit)", } - stderr_log_file.close() return CompressionTaskStatus.FAILED, worker_output # Start compression logger.debug("Compressing...") compression_successful = False proc = subprocess.Popen( - compression_cmd, stdout=subprocess.PIPE, stderr=stderr_log_file, env=compression_env + compression_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=compression_env ) # Compute the total amount of data compressed @@ -550,14 +552,16 @@ def cleanup_temporary_files(): ) try: - subprocess.run( + result = subprocess.run( indexer_cmd, stdout=subprocess.DEVNULL, - stderr=stderr_log_file, + stderr=subprocess.PIPE, check=True, env=indexer_env, ) - except subprocess.CalledProcessError: + log_stderr(result.stderr, "indexer") + except subprocess.CalledProcessError as e: + log_stderr(e.stderr if e.stderr else b"", "indexer") logger.exception("Failed to index archive.") if enable_s3_write: @@ -565,8 +569,10 @@ def cleanup_temporary_files(): last_archive_stats = stats - # Wait for compression to finish + # Wait for compression to finish and capture stderr + compression_stderr = proc.stderr.read() return_code = proc.wait() + log_stderr(compression_stderr, "compression") if 0 != return_code: logger.error(f"Failed to compress, return_code={return_code!s}") @@ -576,9 +582,6 @@ def cleanup_temporary_files(): logger.debug("Compressed.") - # Close stderr log file - stderr_log_file.close() - worker_output = { "total_uncompressed_size": total_uncompressed_size, "total_compressed_size": total_compressed_size, @@ -588,7 +591,7 @@ def cleanup_temporary_files(): return CompressionTaskStatus.SUCCEEDED, worker_output error_msgs = [] if compression_successful is False: - error_msgs.append(f"See logs {stderr_log_path}") + error_msgs.append("See worker logs (captured by Fluent Bit)") if s3_error is not None: error_msgs.append(s3_error) worker_output["error_message"] = "\n".join(error_msgs) @@ -606,10 +609,8 @@ def compression_entry_point( ): clp_home = pathlib.Path(os.getenv("CLP_HOME")) - # Set logging level - logs_dir = pathlib.Path(os.getenv("CLP_LOGS_DIR")) - clp_logging_level = str(os.getenv("CLP_LOGGING_LEVEL")) - set_logging_level(logger, clp_logging_level) + # Set logging level from environment + set_logging_level(logger, os.getenv("CLP_LOGGING_LEVEL")) # Load configuration try: @@ -637,7 +638,6 @@ def compression_entry_point( worker_config, clp_io_config, clp_home, - logs_dir, job_id, task_id, tag_ids, diff --git a/components/job-orchestration/job_orchestration/executor/query/extract_stream_task.py b/components/job-orchestration/job_orchestration/executor/query/extract_stream_task.py index 2ac1214b7c..15507f221e 100644 --- a/components/job-orchestration/job_orchestration/executor/query/extract_stream_task.py +++ b/components/job-orchestration/job_orchestration/executor/query/extract_stream_task.py @@ -191,10 +191,8 @@ def extract_stream( ) -> dict[str, Any]: task_name = "Stream Extraction" - # Setup logging to file - clp_logs_dir = Path(os.getenv("CLP_LOGS_DIR")) - clp_logging_level = os.getenv("CLP_LOGGING_LEVEL") - set_logging_level(logger, clp_logging_level) + # Set logging level from environment + set_logging_level(logger, os.getenv("CLP_LOGGING_LEVEL")) logger.info(f"Started {task_name} task for job {job_id}") @@ -242,7 +240,6 @@ def extract_stream( task_results, task_stdout_str = run_query_task( sql_adapter=sql_adapter, logger=logger, - clp_logs_dir=clp_logs_dir, task_command=task_command, env_vars=core_clp_env_vars, task_name=task_name, @@ -289,7 +286,7 @@ def extract_stream( if upload_error: task_results.status = QueryTaskStatus.FAILED - task_results.error_log_path = str(os.getenv("CLP_WORKER_LOG_PATH")) + task_results.error_log_path = "See worker logs (captured by Fluent Bit)" else: logger.info("Finished uploading streams.") diff --git a/components/job-orchestration/job_orchestration/executor/query/fs_search_task.py b/components/job-orchestration/job_orchestration/executor/query/fs_search_task.py index 052f10cca9..7c4310d28f 100644 --- a/components/job-orchestration/job_orchestration/executor/query/fs_search_task.py +++ b/components/job-orchestration/job_orchestration/executor/query/fs_search_task.py @@ -199,10 +199,8 @@ def search( ) -> dict[str, Any]: task_name = "search" - # Setup logging to file - clp_logs_dir = Path(os.getenv("CLP_LOGS_DIR")) - clp_logging_level = os.getenv("CLP_LOGGING_LEVEL") - set_logging_level(logger, clp_logging_level) + # Set logging level from environment + set_logging_level(logger, os.getenv("CLP_LOGGING_LEVEL")) logger.info(f"Started {task_name} task for job {job_id}") @@ -242,7 +240,6 @@ def search( task_results, _ = run_query_task( sql_adapter=sql_adapter, logger=logger, - clp_logs_dir=clp_logs_dir, task_command=task_command, env_vars=core_clp_env_vars, task_name=task_name, @@ -268,7 +265,7 @@ def search( except Exception as err: logger.error(f"Failed to upload query results {dest_path}: {err}") task_results.status = QueryTaskStatus.FAILED - task_results.error_log_path = str(os.getenv("CLP_WORKER_LOG_PATH")) + task_results.error_log_path = "See worker logs (captured by Fluent Bit)" src_file.unlink() diff --git a/components/job-orchestration/job_orchestration/executor/query/utils.py b/components/job-orchestration/job_orchestration/executor/query/utils.py index 8505d7f594..5938de269b 100644 --- a/components/job-orchestration/job_orchestration/executor/query/utils.py +++ b/components/job-orchestration/job_orchestration/executor/query/utils.py @@ -3,9 +3,7 @@ import signal import subprocess import sys -from contextlib import closing from logging import Logger -from pathlib import Path from typing import Any from clp_py_utils.clp_config import QUERY_TASKS_TABLE_NAME @@ -14,12 +12,6 @@ from job_orchestration.scheduler.scheduler_data import QueryTaskResult, QueryTaskStatus -def get_task_log_file_path(clp_logs_dir: Path, job_id: str, task_id: int) -> Path: - worker_logs_dir = clp_logs_dir / job_id - worker_logs_dir.mkdir(exist_ok=True, parents=True) - return worker_logs_dir / f"{task_id}-clo.log" - - def report_task_failure( sql_adapter: SqlAdapter, task_id: int, @@ -42,7 +34,6 @@ def report_task_failure( def run_query_task( sql_adapter: SqlAdapter, logger: Logger, - clp_logs_dir: Path, task_command: list[str], env_vars: dict[str, str] | None, task_name: str, @@ -50,9 +41,6 @@ def run_query_task( task_id: int, start_time: datetime.datetime, ) -> tuple[QueryTaskResult, str]: - clo_log_path = get_task_log_file_path(clp_logs_dir, job_id, task_id) - clo_log_file = open(clo_log_path, "w") - task_status = QueryTaskStatus.RUNNING update_query_task_metadata( sql_adapter, task_id, dict(status=task_status, start_time=start_time) @@ -64,7 +52,7 @@ def run_query_task( preexec_fn=os.setpgrp, close_fds=True, stdout=subprocess.PIPE, - stderr=clo_log_file, + stderr=subprocess.PIPE, env=env_vars, ) @@ -86,8 +74,15 @@ def sigterm_handler(_signo, _stack_frame): logger.info(f"Waiting for {task_name} to finish") # `communicate` is equivalent to `wait` in this case, but avoids deadlocks when piping to # stdout/stderr. - stdout_data, _ = task_proc.communicate() + stdout_data, stderr_data = task_proc.communicate() return_code = task_proc.returncode + + # Log stderr content (captured by Docker's fluentd driver) + stderr_text = stderr_data.decode("utf-8").strip() + if stderr_text: + for line in stderr_text.split("\n"): + logger.info(f"[{task_name} stderr] {line}") + if 0 != return_code: task_status = QueryTaskStatus.FAILED logger.error( @@ -97,7 +92,6 @@ def sigterm_handler(_signo, _stack_frame): task_status = QueryTaskStatus.SUCCEEDED logger.info(f"{task_name} task {task_id} completed for job {job_id}") - clo_log_file.close() duration = (datetime.datetime.now() - start_time).total_seconds() update_query_task_metadata( @@ -111,7 +105,7 @@ def sigterm_handler(_signo, _stack_frame): ) if QueryTaskStatus.FAILED == task_status: - task_result.error_log_path = str(clo_log_path) + task_result.error_log_path = "See worker logs (captured by Fluent Bit)" return task_result, stdout_data.decode("utf-8") diff --git a/components/job-orchestration/job_orchestration/garbage_collector/archive_garbage_collector.py b/components/job-orchestration/job_orchestration/garbage_collector/archive_garbage_collector.py index c8dbcac0b4..dca4e7d627 100644 --- a/components/job-orchestration/job_orchestration/garbage_collector/archive_garbage_collector.py +++ b/components/job-orchestration/job_orchestration/garbage_collector/archive_garbage_collector.py @@ -192,10 +192,8 @@ def _collect_and_sweep_expired_archives( raise ValueError(f"Unsupported Storage engine: {storage_engine}.") -async def archive_garbage_collector( - clp_config: ClpConfig, log_directory: pathlib.Path, logging_level: str -) -> None: - configure_logger(logger, logging_level, log_directory, ARCHIVE_GARBAGE_COLLECTOR_NAME) +async def archive_garbage_collector(clp_config: ClpConfig, logging_level: str) -> None: + configure_logger(logger, logging_level) archive_output_config = clp_config.archive_output storage_engine = clp_config.package.storage_engine diff --git a/components/job-orchestration/job_orchestration/garbage_collector/garbage_collector.py b/components/job-orchestration/job_orchestration/garbage_collector/garbage_collector.py index 007459315b..4c0ecf1eee 100644 --- a/components/job-orchestration/job_orchestration/garbage_collector/garbage_collector.py +++ b/components/job-orchestration/job_orchestration/garbage_collector/garbage_collector.py @@ -35,10 +35,9 @@ async def main(argv: list[str]) -> int: args_parser.add_argument("--config", "-c", required=True, help="CLP configuration file.") parsed_args = args_parser.parse_args(argv[1:]) - # Setup logging to file - logs_directory = Path(os.getenv("CLP_LOGS_DIR")) + # Set logging level from environment logging_level = os.getenv("CLP_LOGGING_LEVEL") - configure_logger(logger, logging_level, logs_directory, GARBAGE_COLLECTOR_COMPONENT_NAME) + configure_logger(logger, logging_level) # Load configuration config_path = Path(parsed_args.config) @@ -71,9 +70,7 @@ async def main(argv: list[str]) -> int: continue logger.info(f"Creating {gc_name} with retention period = {retention_period} minutes") gc_tasks.append( - asyncio.create_task( - task_method(clp_config, logs_directory, logging_level), name=gc_name - ) + asyncio.create_task(task_method(clp_config, logging_level), name=gc_name) ) # Poll and report any task that finished unexpectedly diff --git a/components/job-orchestration/job_orchestration/garbage_collector/search_result_garbage_collector.py b/components/job-orchestration/job_orchestration/garbage_collector/search_result_garbage_collector.py index da38224b0e..2cc379d09f 100644 --- a/components/job-orchestration/job_orchestration/garbage_collector/search_result_garbage_collector.py +++ b/components/job-orchestration/job_orchestration/garbage_collector/search_result_garbage_collector.py @@ -70,10 +70,8 @@ def _collect_and_sweep_expired_search_results( logger.debug("No search results matched the expiry criteria.") -async def search_result_garbage_collector( - clp_config: ClpConfig, log_directory: pathlib.Path, logging_level: str -) -> None: - configure_logger(logger, logging_level, log_directory, SEARCH_RESULT_GARBAGE_COLLECTOR_NAME) +async def search_result_garbage_collector(clp_config: ClpConfig, logging_level: str) -> None: + configure_logger(logger, logging_level) sweep_interval_secs = clp_config.garbage_collector.sweep_interval.search_result * MIN_TO_SECONDS diff --git a/components/job-orchestration/job_orchestration/garbage_collector/utils.py b/components/job-orchestration/job_orchestration/garbage_collector/utils.py index 1305d546ef..4c76b395aa 100644 --- a/components/job-orchestration/job_orchestration/garbage_collector/utils.py +++ b/components/job-orchestration/job_orchestration/garbage_collector/utils.py @@ -18,13 +18,8 @@ from job_orchestration.garbage_collector.constants import MIN_TO_SECONDS -def configure_logger( - logger: logging.Logger, logging_level: str, log_directory: pathlib.Path, handler_name: str -): - log_file = log_directory / f"{handler_name}.log" - logging_file_handler = logging.FileHandler(filename=log_file, encoding="utf-8") - logging_file_handler.setFormatter(get_logging_formatter()) - logger.addHandler(logging_file_handler) +def configure_logger(logger: logging.Logger, logging_level: str): + """Configure the logging level for a logger (logs go to stdout, captured by Docker).""" set_logging_level(logger, logging_level) diff --git a/components/job-orchestration/job_orchestration/reducer/reducer.py b/components/job-orchestration/job_orchestration/reducer/reducer.py index 2bd6fe3f6a..9559f933a1 100644 --- a/components/job-orchestration/job_orchestration/reducer/reducer.py +++ b/components/job-orchestration/job_orchestration/reducer/reducer.py @@ -1,14 +1,13 @@ #!/usr/bin/env python3 import argparse -import logging import os import subprocess import sys from pathlib import Path from clp_py_utils.clp_config import ClpConfig -from clp_py_utils.clp_logging import get_logger, get_logging_formatter, set_logging_level +from clp_py_utils.clp_logging import get_logger, set_logging_level from clp_py_utils.core import read_yaml_config_file from pydantic import ValidationError @@ -29,14 +28,7 @@ def main(argv: list[str]) -> int: parsed_args = args_parser.parse_args(argv[1:]) - # Setup logging to file - logs_dir = Path(os.getenv("CLP_LOGS_DIR")) - log_file = Path(os.getenv("CLP_LOGS_DIR")) / "reducer.log" - logging_file_handler = logging.FileHandler(filename=log_file, encoding="utf-8") - logging_file_handler.setFormatter(get_logging_formatter()) - logger.addHandler(logging_file_handler) - - # Update logging level based on config + # Set logging level from environment set_logging_level(logger, os.getenv("CLP_LOGGING_LEVEL")) # Load configuration @@ -68,15 +60,11 @@ def main(argv: list[str]) -> int: for i in range(concurrency): reducer_instance_cmd = reducer_cmd + [str(clp_config.reducer.base_port + i)] - log_file_path = logs_dir / ("reducer-" + str(i) + ".log") - log_file = open(log_file_path, "a") - + # Let subprocess inherit stdout/stderr - Docker captures these via fluentd driver reducers.append( subprocess.Popen( reducer_instance_cmd, close_fds=True, - stdout=log_file, - stderr=log_file, ) ) @@ -93,9 +81,6 @@ def main(argv: list[str]) -> int: logger.error("All reducers terminated") - logger.removeHandler(logging_file_handler) - logging_file_handler.close() - return 0 diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py index ba5a077a19..8bb068d27b 100644 --- a/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py @@ -426,13 +426,7 @@ def main(argv) -> int | None: args_parser.add_argument("--config", "-c", required=True, help="CLP configuration file.") args = args_parser.parse_args(argv[1:]) - # Setup logging - log_file = Path(os.getenv("CLP_LOGS_DIR")) / "compression_scheduler.log" - logging_file_handler = logging.FileHandler(filename=log_file, encoding="utf-8") - logging_file_handler.setFormatter(get_logging_formatter()) - logger.addHandler(logging_file_handler) - - # Update logging level based on config + # Set logging level from environment set_logging_level(logger, os.getenv("CLP_LOGGING_LEVEL")) # Register the SIGTERM handler diff --git a/components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py b/components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py index a8be65dd62..2df2d970c1 100644 --- a/components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py +++ b/components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py @@ -1147,13 +1147,7 @@ async def main(argv: list[str]) -> int: parsed_args = args_parser.parse_args(argv[1:]) - # Setup logging to file - log_file = Path(os.getenv("CLP_LOGS_DIR")) / "query_scheduler.log" - logging_file_handler = logging.FileHandler(filename=log_file, encoding="utf-8") - logging_file_handler.setFormatter(get_logging_formatter()) - logger.addHandler(logging_file_handler) - - # Update logging level based on config + # Set logging level from environment set_logging_level(logger, os.getenv("CLP_LOGGING_LEVEL")) # Load configuration diff --git a/components/log-ingestor/src/bin/log_ingestor.rs b/components/log-ingestor/src/bin/log_ingestor.rs index 355ee80928..c71b588930 100644 --- a/components/log-ingestor/src/bin/log_ingestor.rs +++ b/components/log-ingestor/src/bin/log_ingestor.rs @@ -2,8 +2,7 @@ use anyhow::Context; use clap::Parser; use clp_rust_utils::{clp_config::package, serde::yaml}; use log_ingestor::{ingestion_job_manager::IngestionJobManagerState, routes::create_router}; -use tracing_appender::rolling::{RollingFileAppender, Rotation}; -use tracing_subscriber::{self, fmt::writer::MakeWriterExt}; +use tracing_subscriber; #[derive(Parser)] #[command(version, about = "log-ingestor for CLP.")] @@ -43,13 +42,8 @@ fn read_config_and_credentials( Ok((config, credentials)) } -fn set_up_logging() -> anyhow::Result { - let logs_directory = - std::env::var("CLP_LOGS_DIR").context("Expect `CLP_LOGS_DIR` environment variable.")?; - let logs_directory = std::path::Path::new(logs_directory.as_str()); - let file_appender = - RollingFileAppender::new(Rotation::HOURLY, logs_directory, "log_ingestor.log"); - let (non_blocking_writer, guard) = tracing_appender::non_blocking(file_appender); +fn set_up_logging() { + // Logs to stdout only - Docker's fluentd driver captures and forwards to Fluent Bit tracing_subscriber::fmt() .event_format( tracing_subscriber::fmt::format() @@ -61,9 +55,8 @@ fn set_up_logging() -> anyhow::Result anyhow::Result<()> { let args = Args::parse(); let (config, credentials) = read_config_and_credentials(&args)?; - let _guard = set_up_logging()?; + set_up_logging(); let addr = format!("{}:{}", args.host, args.port); let listener = tokio::net::TcpListener::bind(&addr) diff --git a/components/package-template/src/etc/fluent-bit/filters.conf b/components/package-template/src/etc/fluent-bit/filters.conf new file mode 100644 index 0000000000..0f76825759 --- /dev/null +++ b/components/package-template/src/etc/fluent-bit/filters.conf @@ -0,0 +1,8 @@ +# Filter configuration for CLP operational logging + +# Add common metadata to all logs +[FILTER] + Name modify + Match clp.* + Add clp_deployment clp-package + Add source fluentd-driver diff --git a/components/package-template/src/etc/fluent-bit/fluent-bit.conf b/components/package-template/src/etc/fluent-bit/fluent-bit.conf new file mode 100644 index 0000000000..7324c317b1 --- /dev/null +++ b/components/package-template/src/etc/fluent-bit/fluent-bit.conf @@ -0,0 +1,20 @@ +# Fluent Bit configuration for CLP operational logging +# This configuration collects logs from CLP components and stores them in organized file storage. + +[SERVICE] + Daemon Off + Flush 5 + Log_Level ${FLUENT_BIT_LOG_LEVEL} + Parsers_File /fluent-bit/etc/parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + Health_Check On + storage.path /var/log/clp-operational/buffer + storage.sync normal + storage.checksum off + storage.backlog.mem_limit 5M + +@INCLUDE inputs.conf +@INCLUDE filters.conf +@INCLUDE outputs.conf diff --git a/components/package-template/src/etc/fluent-bit/outputs.conf b/components/package-template/src/etc/fluent-bit/outputs.conf new file mode 100644 index 0000000000..f6f68b8d2f --- /dev/null +++ b/components/package-template/src/etc/fluent-bit/outputs.conf @@ -0,0 +1,69 @@ +# Output configuration for CLP operational logging +# Hot tier: Write to organized file storage for real-time access + +# First-party services +[OUTPUT] + Name file + Match clp.compression-scheduler + Path /var/log/clp/compression_scheduler + Mkdir On + +[OUTPUT] + Name file + Match clp.compression-worker + Path /var/log/clp/compression_worker + Mkdir On + +[OUTPUT] + Name file + Match clp.spider-compression-worker + Path /var/log/clp/spider_compression_worker + Mkdir On + +[OUTPUT] + Name file + Match clp.query-scheduler + Path /var/log/clp/query_scheduler + Mkdir On + +[OUTPUT] + Name file + Match clp.query-worker + Path /var/log/clp/query_worker + Mkdir On + +[OUTPUT] + Name file + Match clp.reducer + Path /var/log/clp/reducer + Mkdir On + +[OUTPUT] + Name file + Match clp.garbage-collector + Path /var/log/clp/garbage_collector + Mkdir On + +[OUTPUT] + Name file + Match clp.webui + Path /var/log/clp/webui + Mkdir On + +[OUTPUT] + Name file + Match clp.api-server + Path /var/log/clp/api_server + Mkdir On + +[OUTPUT] + Name file + Match clp.log-ingestor + Path /var/log/clp/log_ingestor + Mkdir On + +[OUTPUT] + Name file + Match clp.mcp-server + Path /var/log/clp/mcp_server + Mkdir On diff --git a/components/package-template/src/etc/fluent-bit/parsers.conf b/components/package-template/src/etc/fluent-bit/parsers.conf new file mode 100644 index 0000000000..9b38f9c1c6 --- /dev/null +++ b/components/package-template/src/etc/fluent-bit/parsers.conf @@ -0,0 +1,11 @@ +# Parser configuration for CLP operational logging +# Note: Most logs arrive as JSON from Docker's fluentd logging driver. +# These parsers are provided for optional post-processing. + +# Docker JSON log format (default from fluentd driver) +[PARSER] + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep On diff --git a/components/webui/client/src/components/Layout/MainLayout.module.css b/components/webui/client/src/components/Layout/MainLayout.module.css index 1cebf0aaad..61d63063e8 100644 --- a/components/webui/client/src/components/Layout/MainLayout.module.css +++ b/components/webui/client/src/components/Layout/MainLayout.module.css @@ -2,6 +2,20 @@ min-height: 100vh; } +.mainLayout :global(.ant-layout-sider) { + position: sticky; + top: 0; + height: 100vh; + overflow-y: auto; +} + +.siderContent { + display: flex; + flex-direction: column; + justify-content: space-between; + min-height: calc(100vh - 48px); /* Fill viewport height minus collapse trigger */ +} + .siderLogoContainer { display: flex; justify-content: center; diff --git a/components/webui/client/src/components/Layout/MainLayout.tsx b/components/webui/client/src/components/Layout/MainLayout.tsx index 39510c9a18..c53904b7cd 100644 --- a/components/webui/client/src/components/Layout/MainLayout.tsx +++ b/components/webui/client/src/components/Layout/MainLayout.tsx @@ -5,6 +5,7 @@ import { } from "react-router"; import { + FileTextOutlined, SearchOutlined, UploadOutlined, } from "@ant-design/icons"; @@ -21,11 +22,15 @@ const {Sider} = Layout; type MenuItem = Required["items"][number]; -const SIDEBAR_MENU_ITEMS: MenuItem[] = [ +const SIDEBAR_MAIN_MENU_ITEMS: MenuItem[] = [ {label: Ingest, key: "/ingest", icon: }, {label: Search, key: "/search", icon: }, ]; +const SIDEBAR_BOTTOM_MENU_ITEMS: MenuItem[] = [ + {label: Logs, key: "/logs", icon: }, +]; + /** * The main layout of web ui. * @@ -45,15 +50,22 @@ const MainLayout = () => { setCollapsed(value); }} > -
- {"CLP +
+
+
+ {"CLP +
+ +
+
- diff --git a/components/webui/client/src/components/VirtualTable/index.tsx b/components/webui/client/src/components/VirtualTable/index.tsx index 45276b6449..da5c7f3340 100644 --- a/components/webui/client/src/components/VirtualTable/index.tsx +++ b/components/webui/client/src/components/VirtualTable/index.tsx @@ -3,7 +3,7 @@ import React, { useRef, } from "react"; -import {Table} from "antd"; +import {Empty, Table} from "antd"; import { SCROLL_INCREMENT, @@ -11,6 +11,26 @@ import { type VirtualTableProps, } from "./typings"; +interface FullHeightEmptyProps { + height: number; +} + +/** + * Empty component that fills the specified height. + */ +const FullHeightEmpty = ({height}: FullHeightEmptyProps) => ( +
+ +
+); + /** * Virtual table that supports keyboard navigation. @@ -20,6 +40,8 @@ import { * @return */ const VirtualTable = >({ + scroll, + locale, ...tableProps }: VirtualTableProps) => { const containerRef = useRef(null); @@ -70,6 +92,17 @@ const VirtualTable = >({ e.preventDefault(); }, []); + // Use scroll.y height for the empty state if available + const emptyHeight = "number" === typeof scroll?.y ? scroll.y : undefined; + + // Build locale with full-height empty state + const fullHeightLocale = emptyHeight ? + { + ...locale, + emptyText: , + } : + locale; + return (
>({ > virtual={true} - {...tableProps}/> + {...tableProps} + {...(scroll && {scroll})} + {...(fullHeightLocale && {locale: fullHeightLocale})}/>
); }; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx index 71820701a9..9ccbe22811 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx @@ -8,6 +8,7 @@ import {CLP_QUERY_ENGINES} from "@webui/common/config"; import {SETTINGS_QUERY_ENGINE} from "../../../../config"; import usePrestoSearchState from "../../SearchState/Presto"; +import styles from "./index.module.css"; import SearchResultsVirtualTable from "./Native/SearchResultsVirtualTable"; import PrestoResultsVirtualTable from "./Presto/PrestoResultsVirtualTable"; import {TABLE_BOTTOM_PADDING} from "./typings"; @@ -45,7 +46,8 @@ const SearchResultsTable = () => { return (
0 ? tableHeight : undefined}} > {CLP_QUERY_ENGINES.PRESTO === SETTINGS_QUERY_ENGINE ? ( diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/typings.ts b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/typings.ts index 940970bd09..521dea25ad 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/typings.ts +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/typings.ts @@ -1,7 +1,7 @@ /** * Padding for the table to the bottom of the page. */ -const TABLE_BOTTOM_PADDING = 95; +const TABLE_BOTTOM_PADDING = 24; /** * The maximum number of results to retrieve for a search. diff --git a/components/webui/client/src/router.tsx b/components/webui/client/src/router.tsx index babeb1f411..03a3f894c2 100644 --- a/components/webui/client/src/router.tsx +++ b/components/webui/client/src/router.tsx @@ -6,6 +6,7 @@ import { import MainLayout from "./components/Layout/MainLayout"; import IngestPage from "./pages/IngestPage"; import QueryStatus from "./pages/LogViewerLoadingPage/QueryStatus"; +import OperationalLogsPage from "./pages/OperationalLogsPage"; import SearchPage from "./pages/SearchPage"; @@ -22,6 +23,7 @@ const router = createBrowserRouter([ }, {path: "ingest", Component: IngestPage}, {path: "search", Component: SearchPage}, + {path: "logs", Component: OperationalLogsPage}, ], }, { diff --git a/components/webui/server/settings.json b/components/webui/server/settings.json index da8b8094b5..839c87f4ac 100644 --- a/components/webui/server/settings.json +++ b/components/webui/server/settings.json @@ -29,5 +29,7 @@ "ClpQueryEngine": "clp-s", "ClpStorageEngine": "clp-s", "PrestoHost": "localhost", - "PrestoPort": 8889 + "PrestoPort": 8889, + + "OperationalLogsDir": "/var/log/clp" } diff --git a/docs/src/user-docs/guides-multi-host.md b/docs/src/user-docs/guides-multi-host.md index e9e5ecd436..88e7f7243b 100755 --- a/docs/src/user-docs/guides-multi-host.md +++ b/docs/src/user-docs/guides-multi-host.md @@ -319,6 +319,49 @@ To view logs for a specific service: docker compose --project-name clp-package- logs -f ``` +## Operational logging + +CLP uses [Fluent Bit][fluent-bit] to collect and aggregate operational logs from all services. Logs +are written to `var/log//` on each host. + +### Current limitations in multi-host deployments + +:::{warning} +In multi-host deployments, operational logs are currently stored locally on each host. This means: + +* Each host runs its own Fluent Bit instance that collects logs from containers on that host. +* Logs are written to the local filesystem (`var/log//`) and are not automatically + aggregated across hosts. +* To view logs from a specific host, you must access that host directly. +::: + +### Workarounds + +Until centralized log aggregation is implemented, you can: + +1. **Use a shared filesystem**: Mount `var/log/` to a shared filesystem (e.g., NFS, SeaweedFS) so + all hosts write logs to the same location. Note that you may need to include the hostname in the + log path to avoid conflicts. + +2. **Use `docker compose logs`**: View real-time logs for a service using: + + ```bash + docker compose --project-name clp-package- logs -f + ``` + +3. **Manual log collection**: Periodically copy logs from each host to a central location for + analysis. + +### Future improvements + +A future release will add support for shipping logs to S3 or other centralized storage, enabling: + +* Centralized log aggregation across all hosts +* Log viewing through the webui regardless of which host generated the logs +* Long-term log retention with tiered storage (hot/warm/cold) + +[fluent-bit]: https://fluentbit.io/ + ## Setting up SeaweedFS The instructions below are for running a simple SeaweedFS cluster on a set of hosts. For other use diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 881dfbb972..09562afcbf 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -301,9 +301,18 @@ data: "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, {{- with .Values.clpConfig.presto }} "PrestoHost": {{ .host | quote }}, - "PrestoPort": {{ .port | int }} + "PrestoPort": {{ .port | int }}, {{- else }} "PrestoHost": null, - "PrestoPort": null + "PrestoPort": null, + {{- end }} + {{- if .Values.clpConfig.fluent_bit }} + {{- if .Values.clpConfig.fluent_bit.enabled }} + "OperationalLogsDir": "/var/log/clp" + {{- else }} + "OperationalLogsDir": null + {{- end }} + {{- else }} + "OperationalLogsDir": null {{- end }} } diff --git a/tools/deployment/package-helm/templates/fluent-bit-clusterrole.yaml b/tools/deployment/package-helm/templates/fluent-bit-clusterrole.yaml new file mode 100644 index 0000000000..3d0ac38864 --- /dev/null +++ b/tools/deployment/package-helm/templates/fluent-bit-clusterrole.yaml @@ -0,0 +1,20 @@ +{{- if .Values.clpConfig.fluent_bit }} +{{- if .Values.clpConfig.fluent_bit.enabled }} +apiVersion: "rbac.authorization.k8s.io/v1" +kind: "ClusterRole" +metadata: + name: {{ include "clp.fullname" . }}-fluent-bit + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "fluent-bit" +rules: + - apiGroups: [""] + resources: + - "namespaces" + - "pods" + verbs: + - "get" + - "list" + - "watch" +{{- end }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/fluent-bit-clusterrolebinding.yaml b/tools/deployment/package-helm/templates/fluent-bit-clusterrolebinding.yaml new file mode 100644 index 0000000000..341761e16c --- /dev/null +++ b/tools/deployment/package-helm/templates/fluent-bit-clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.clpConfig.fluent_bit }} +{{- if .Values.clpConfig.fluent_bit.enabled }} +apiVersion: "rbac.authorization.k8s.io/v1" +kind: "ClusterRoleBinding" +metadata: + name: {{ include "clp.fullname" . }}-fluent-bit + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "fluent-bit" +roleRef: + apiGroup: "rbac.authorization.k8s.io" + kind: "ClusterRole" + name: {{ include "clp.fullname" . }}-fluent-bit +subjects: + - kind: "ServiceAccount" + name: {{ include "clp.fullname" . }}-fluent-bit + namespace: {{ .Release.Namespace }} +{{- end }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/fluent-bit-configmap.yaml b/tools/deployment/package-helm/templates/fluent-bit-configmap.yaml new file mode 100644 index 0000000000..2519ba6217 --- /dev/null +++ b/tools/deployment/package-helm/templates/fluent-bit-configmap.yaml @@ -0,0 +1,287 @@ +{{- if .Values.clpConfig.fluent_bit }} +{{- if .Values.clpConfig.fluent_bit.enabled }} +apiVersion: "v1" +kind: "ConfigMap" +metadata: + name: {{ include "clp.fullname" . }}-fluent-bit-config + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "fluent-bit" +data: + fluent-bit.conf: | + [SERVICE] + Daemon Off + Flush {{ .Values.clpConfig.fluent_bit.flush_interval | default 5 }} + Log_Level ${FLUENT_BIT_LOG_LEVEL} + Parsers_File /fluent-bit/etc/parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + Health_Check On + storage.path /var/log/clp-operational/buffer + storage.sync normal + storage.checksum off + storage.backlog.mem_limit 5M + + @INCLUDE inputs.conf + @INCLUDE filters.conf + @INCLUDE outputs.conf + + inputs.conf: | + # Kubernetes container logs (stdout/stderr captured by container runtime) + [INPUT] + Name tail + Tag kube.* + Path /var/log/containers/*{{ include "clp.fullname" . }}*.log + Parser cri + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval {{ .Values.clpConfig.fluent_bit.refresh_interval | default 10 }} + DB /var/log/clp-operational/buffer/kube-tail.db + + # Forward input for receiving logs from other Fluent Bit instances + [INPUT] + Name forward + Listen 0.0.0.0 + Port 24224 + + filters.conf: | + # Kubernetes metadata enrichment + [FILTER] + Name kubernetes + Match kube.* + Kube_URL https://kubernetes.default.svc:443 + Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token + Merge_Log On + Keep_Log Off + K8S-Logging.Parser On + K8S-Logging.Exclude On + Labels Off + Annotations Off + + # Extract component name from Kubernetes labels + [FILTER] + Name rewrite_tag + Match kube.* + Rule $kubernetes['labels']['app.kubernetes.io/component'] ^(.+)$ component.$1 false + + # Add common metadata + [FILTER] + Name modify + Match * + Add clp_deployment {{ include "clp.fullname" . }} + Add node ${NODE_NAME} + + # Nest record under 'log' key for consistent structure + [FILTER] + Name nest + Match component.* + Operation nest + Wildcard * + Nest_under body + Remove_prefix body_ + + outputs.conf: | + # Hot tier: Write to organized file storage for real-time access + [OUTPUT] + Name file + Match component.* + Path /var/log/clp-operational/hot + Mkdir On + Format out_file + Template {time} {component} {log} + + # Write all logs to daily rotating files organized by component + [OUTPUT] + Name file + Match component.compression-scheduler + Path /var/log/clp-operational/hot/compression_scheduler + Mkdir On + + [OUTPUT] + Name file + Match component.compression-worker + Path /var/log/clp-operational/hot/compression_worker + Mkdir On + + [OUTPUT] + Name file + Match component.query-scheduler + Path /var/log/clp-operational/hot/query_scheduler + Mkdir On + + [OUTPUT] + Name file + Match component.query-worker + Path /var/log/clp-operational/hot/query_worker + Mkdir On + + [OUTPUT] + Name file + Match component.reducer + Path /var/log/clp-operational/hot/reducer + Mkdir On + + [OUTPUT] + Name file + Match component.webui + Path /var/log/clp-operational/hot/webui + Mkdir On + + [OUTPUT] + Name file + Match component.api-server + Path /var/log/clp-operational/hot/api_server + Mkdir On + + [OUTPUT] + Name file + Match component.log-ingestor + Path /var/log/clp-operational/hot/log_ingestor + Mkdir On + + [OUTPUT] + Name file + Match component.mcp-server + Path /var/log/clp-operational/hot/mcp_server + Mkdir On + + [OUTPUT] + Name file + Match component.garbage-collector + Path /var/log/clp-operational/hot/garbage_collector + Mkdir On + + [OUTPUT] + Name file + Match component.database + Path /var/log/clp-operational/hot/database + Mkdir On + + [OUTPUT] + Name file + Match component.queue + Path /var/log/clp-operational/hot/queue + Mkdir On + + [OUTPUT] + Name file + Match component.redis + Path /var/log/clp-operational/hot/redis + Mkdir On + + [OUTPUT] + Name file + Match component.results-cache + Path /var/log/clp-operational/hot/results_cache + Mkdir On + + {{- if .Values.clpConfig.fluent_bit.s3_output }} + {{- if .Values.clpConfig.fluent_bit.s3_output.enabled }} + # Cold tier: Upload to S3 for archival with CLP compression + [OUTPUT] + Name s3 + Match component.* + bucket {{ .Values.clpConfig.fluent_bit.s3_output.bucket }} + region {{ .Values.clpConfig.fluent_bit.s3_output.region }} + {{- if .Values.clpConfig.fluent_bit.s3_output.endpoint }} + endpoint {{ .Values.clpConfig.fluent_bit.s3_output.endpoint }} + {{- end }} + s3_key_format /{{ .Values.clpConfig.fluent_bit.s3_output.key_prefix | default + "operational-logs" }}/$TAG[1]/%Y/%m/%d/%H/$UUID.log + total_file_size {{ .Values.clpConfig.fluent_bit.s3_output.total_file_size | default "10M" }} + upload_timeout {{ .Values.clpConfig.fluent_bit.s3_output.upload_timeout | default "5m" }} + use_put_object On + compression gzip + store_dir /var/log/clp-operational/s3-buffer + static_file_path Off + {{- if eq .Values.clpConfig.fluent_bit.s3_output.auth_type "profile" }} + {{- if .Values.clpConfig.fluent_bit.s3_output.profile }} + profile {{ .Values.clpConfig.fluent_bit.s3_output.profile }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + # Debug output (can be enabled for troubleshooting) + {{- if .Values.clpConfig.fluent_bit.debug_output }} + [OUTPUT] + Name stdout + Match * + Format json_lines + {{- end }} + + parsers.conf: | + # CRI log format (containerd/CRI-O) + [PARSER] + Name cri + Format regex + Regex ^(?