diff --git a/tools/deployment/package-helm/.set-up-common.sh b/tools/deployment/package-helm/.set-up-common.sh new file mode 100755 index 0000000000..2a0204d0d2 --- /dev/null +++ b/tools/deployment/package-helm/.set-up-common.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +# Common utilities for Helm chart set-up scripts +# Source this file from set-up-*.sh scripts + +set -o errexit +set -o nounset +set -o pipefail + +# Creates required directories for CLP data and logs +create_clp_directories() { + echo "Creating CLP directories at ${CLP_HOME}..." + mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ + "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \ + "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ + "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ + "$CLP_HOME/var/log/"{api_server,garbage_collector,log_ingestor,mcp_server} \ + "$CLP_HOME/var/tmp" \ + "$CLP_HOME/samples" +} + +# Downloads sample datasets in the background +# Sets SAMPLE_DOWNLOAD_PID global variable +download_samples() { + echo "Downloading sample datasets..." + wget -O - https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1 \ + | tar xz -C "$CLP_HOME/samples" & + SAMPLE_DOWNLOAD_PID=$! + + # Generate sample log file for garbage collector testing + cat < "$CLP_HOME/samples/test-gc.jsonl" +{"timestamp": $(date +%s%3N), "level": "INFO", "message": "User login successful"} +{"timestamp": $(date +%s%3N), "level": "ERROR", "message": "Database connection failed"} +EOF +} + +# Cleans up existing cluster and prepares environment +# @param {string} cluster_name Name of the kind cluster +prepare_environment() { + local cluster_name=$1 + + echo "Deleting existing cluster if present..." + kind delete cluster --name "${cluster_name}" 2>/dev/null || true + + rm -rf "$CLP_HOME" + create_clp_directories + download_samples +} + +# Generates kind cluster configuration YAML +# +# @param {int} num_workers Number of worker nodes (0 for single-node cluster) +generate_kind_config() { + local num_workers=${1:-0} + + cat </dev/null \ + && kubectl wait pods \ + --all \ + --selector='!job-name' \ + --for=condition=Ready \ + --timeout="${wait_timeout_seconds}s" 2>/dev/null + then + echo "All jobs completed and services are ready." + return 0 + fi + + if [[ ${SECONDS} -ge ${timeout_seconds} ]]; then + echo "ERROR: Timed out waiting for pods to be ready" + return 1 + fi + + echo "---" + done +} + +# Waits for sample download to complete and all pods to be ready +wait_for_cluster_ready() { + if wait "$SAMPLE_DOWNLOAD_PID"; then + echo "Sample download and extraction complete" + else + echo "ERROR: Sample download failed" + return 1 + fi + + wait_for_pods 300 5 5 +} diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 8385e19fae..1805a262a3 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.1.2-dev.17" +version: "0.1.2-dev.18" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.7.1-dev" diff --git a/tools/deployment/package-helm/set-up-multi-dedicated-test.sh b/tools/deployment/package-helm/set-up-multi-dedicated-test.sh new file mode 100755 index 0000000000..2e8ab3c23d --- /dev/null +++ b/tools/deployment/package-helm/set-up-multi-dedicated-test.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# Multi-node cluster setup with dedicated worker nodes for each worker type +# Demonstrates nodeSelector scheduling with separate node pools +# TODO: Migrate into integration test + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +CLP_HOME="${CLP_HOME:-/tmp/clp}" +CLUSTER_NAME="${CLUSTER_NAME:-clp-test}" +NUM_COMPRESSION_NODES="${NUM_COMPRESSION_NODES:-2}" +NUM_QUERY_NODES="${NUM_QUERY_NODES:-2}" +COMPRESSION_WORKER_REPLICAS="${COMPRESSION_WORKER_REPLICAS:-2}" +QUERY_WORKER_REPLICAS="${QUERY_WORKER_REPLICAS:-2}" +REDUCER_REPLICAS="${REDUCER_REPLICAS:-2}" + +# shellcheck source=.set-up-common.sh +source "${script_dir}/.set-up-common.sh" + +echo "=== Multi-node setup with dedicated worker nodes ===" +echo "Cluster: ${CLUSTER_NAME}" +echo "Compression nodes: ${NUM_COMPRESSION_NODES}" +echo "Query nodes: ${NUM_QUERY_NODES}" +echo "Compression workers: ${COMPRESSION_WORKER_REPLICAS}" +echo "Query workers: ${QUERY_WORKER_REPLICAS}" +echo "Reducers: ${REDUCER_REPLICAS}" +echo "" + +prepare_environment "${CLUSTER_NAME}" + +total_workers=$((NUM_COMPRESSION_NODES + NUM_QUERY_NODES)) + +echo "Creating kind cluster..." +generate_kind_config "${total_workers}" | kind create cluster --name "${CLUSTER_NAME}" --config=- + +echo "Labeling worker nodes..." +mapfile -t worker_nodes < <(kubectl get nodes --selector='!node-role.kubernetes.io/control-plane' -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n') + +# Label compression nodes +for ((i = 0; i < NUM_COMPRESSION_NODES; i++)); do + echo "Labeling ${worker_nodes[$i]} as compression node" + kubectl label node "${worker_nodes[$i]}" yscope.io/nodeType=compression --overwrite +done + +# Label query nodes +for ((i = NUM_COMPRESSION_NODES; i < total_workers; i++)); do + echo "Labeling ${worker_nodes[$i]} as query node" + kubectl label node "${worker_nodes[$i]}" yscope.io/nodeType=query --overwrite +done + +echo "Installing Helm chart..." +helm uninstall test --ignore-not-found +sleep 2 +helm install test "${script_dir}" \ + --set "distributedDeployment=true" \ + --set "compressionWorker.replicas=${COMPRESSION_WORKER_REPLICAS}" \ + --set "compressionWorker.scheduling.nodeSelector.yscope\.io/nodeType=compression" \ + --set "queryWorker.replicas=${QUERY_WORKER_REPLICAS}" \ + --set "queryWorker.scheduling.nodeSelector.yscope\.io/nodeType=query" \ + --set "reducer.replicas=${REDUCER_REPLICAS}" + +wait_for_cluster_ready diff --git a/tools/deployment/package-helm/set-up-multi-shared-test.sh b/tools/deployment/package-helm/set-up-multi-shared-test.sh new file mode 100755 index 0000000000..766e836ac3 --- /dev/null +++ b/tools/deployment/package-helm/set-up-multi-shared-test.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Multi-node cluster setup with shared worker nodes +# Both compression and query workers share the same node pool +# TODO: Migrate into integration test + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +CLP_HOME="${CLP_HOME:-/tmp/clp}" +CLUSTER_NAME="${CLUSTER_NAME:-clp-test}" +NUM_WORKER_NODES="${NUM_WORKER_NODES:-2}" +COMPRESSION_WORKER_REPLICAS="${COMPRESSION_WORKER_REPLICAS:-2}" +QUERY_WORKER_REPLICAS="${QUERY_WORKER_REPLICAS:-2}" +REDUCER_REPLICAS="${REDUCER_REPLICAS:-2}" + +# shellcheck source=.set-up-common.sh +source "${script_dir}/.set-up-common.sh" + +echo "=== Multi-node setup with shared worker nodes ===" +echo "Cluster: ${CLUSTER_NAME}" +echo "Worker nodes: ${NUM_WORKER_NODES}" +echo "Compression workers: ${COMPRESSION_WORKER_REPLICAS}" +echo "Query workers: ${QUERY_WORKER_REPLICAS}" +echo "Reducers: ${REDUCER_REPLICAS}" +echo "" + +prepare_environment "${CLUSTER_NAME}" + +echo "Creating kind cluster..." +generate_kind_config "${NUM_WORKER_NODES}" | kind create cluster --name "${CLUSTER_NAME}" --config=- + +echo "Installing Helm chart..." +helm uninstall test --ignore-not-found +sleep 2 +helm install test "${script_dir}" \ + --set "distributedDeployment=true" \ + --set "compressionWorker.replicas=${COMPRESSION_WORKER_REPLICAS}" \ + --set "queryWorker.replicas=${QUERY_WORKER_REPLICAS}" \ + --set "reducer.replicas=${REDUCER_REPLICAS}" + +wait_for_cluster_ready diff --git a/tools/deployment/package-helm/set-up-test.sh b/tools/deployment/package-helm/set-up-test.sh new file mode 100755 index 0000000000..f8290c12a2 --- /dev/null +++ b/tools/deployment/package-helm/set-up-test.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Single-node cluster setup for testing +# TODO: Migrate into integration test + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +CLP_HOME="${CLP_HOME:-/tmp/clp}" +CLUSTER_NAME="${CLUSTER_NAME:-clp-test}" + +# shellcheck source=.set-up-common.sh +source "${script_dir}/.set-up-common.sh" + +echo "=== Single-node setup ===" +echo "Cluster: ${CLUSTER_NAME}" +echo "" + +prepare_environment "${CLUSTER_NAME}" + +echo "Creating kind cluster..." +generate_kind_config 0 | kind create cluster --name "${CLUSTER_NAME}" --config=- + +echo "Installing Helm chart..." +helm uninstall test --ignore-not-found +sleep 2 +helm install test "${script_dir}" + +wait_for_cluster_ready diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 96097b11e0..c148c04349 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -111,20 +111,22 @@ Used for: {{- end }} {{/* -Creates a local PersistentVolume. +Creates a PersistentVolume that does not use dynamic provisioning. + +Behavior depends on the `distributedDeployment` value: +- distributedDeployment=false: Uses local volume type with node affinity targeting control-plane + nodes +- distributedDeployment=true: Uses hostPath without node affinity (assumes shared storage like NFS) @param {object} root Root template context @param {string} component_category (e.g., "database", "shared-data") @param {string} name (e.g., "archives", "data", "logs") -@param {string} nodeRole Node role for affinity. Targets nodes with label - "node-role.kubernetes.io/". Always falls back to - "node-role.kubernetes.io/control-plane" @param {string} capacity Storage capacity @param {string[]} accessModes Access modes @param {string} hostPath Absolute path on host @return {string} YAML-formatted PersistentVolume resource */}} -{{- define "clp.createLocalPv" -}} +{{- define "clp.createStaticPv" -}} apiVersion: "v1" kind: "PersistentVolume" metadata: @@ -137,19 +139,22 @@ spec: storage: {{ .capacity }} accessModes: {{ .accessModes }} persistentVolumeReclaimPolicy: "Retain" - storageClassName: "local-storage" + storageClassName: {{ .root.Values.storage.storageClassName | quote }} + {{- if .root.Values.distributedDeployment }} + hostPath: + path: {{ .hostPath | quote }} + type: "DirectoryOrCreate" + {{- else }} local: path: {{ .hostPath | quote }} nodeAffinity: required: nodeSelectorTerms: - - matchExpressions: - - key: {{ printf "node-role.kubernetes.io/%s" .nodeRole | quote }} - operator: "Exists" - matchExpressions: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" -{{- end }} + {{- end }}{{/* if .root.Values.distributedDeployment */}} +{{- end }}{{/* define "clp.createStaticPv" */}} {{/* Creates a PersistentVolumeClaim for the given component. @@ -171,7 +176,7 @@ metadata: app.kubernetes.io/component: {{ .component_category | quote }} spec: accessModes: {{ .accessModes }} - storageClassName: "local-storage" + storageClassName: {{ .root.Values.storage.storageClassName | quote }} selector: matchLabels: {{- include "clp.selectorLabels" .root | nindent 6 }} @@ -297,3 +302,44 @@ command: [ "--timeout=300s" ] {{- end }} + +{{/* +Creates scheduling configuration (nodeSelector, affinity, tolerations, topologySpreadConstraints) +for a component. + +When distributedDeployment is false (single-node mode), a control-plane toleration is automatically +added so pods can be scheduled on tainted control-plane nodes without manual untainting. + +@param {object} root Root template context +@param {string} component Key name in top-level Values (e.g., "compressionWorker", "queryWorker") +@return {string} YAML-formatted scheduling fields (nodeSelector, affinity, tolerations, + topologySpreadConstraints) +*/}} +{{- define "clp.createSchedulingConfigs" -}} +{{- $componentConfig := index .root.Values .component | default dict -}} +{{- $scheduling := $componentConfig.scheduling | default dict -}} +{{- $tolerations := $scheduling.tolerations | default list -}} +{{- if not .root.Values.distributedDeployment -}} +{{- $tolerations = append $tolerations (dict + "key" "node-role.kubernetes.io/control-plane" + "operator" "Exists" + "effect" "NoSchedule" +) -}} +{{- end -}} +{{- with $scheduling.nodeSelector }} +nodeSelector: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with $scheduling.affinity }} +affinity: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with $tolerations }} +tolerations: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with $scheduling.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }}{{/* define "clp.createSchedulingConfigs" */}} diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml index 89fd30dda8..e117f83bd1 100644 --- a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml @@ -1,9 +1,8 @@ {{- if .Values.clpConfig.api_server }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "api-server" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/api_server" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml index 565f5dc40b..0a886c7c53 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "compression-scheduler" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression_scheduler" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml index 9b667a58bf..2e30551fbd 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-user-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "compression-scheduler" "name" "user-logs" - "nodeRole" "control-plane" "capacity" "10Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/user" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index efd2b3e0d5..c6c24f355c 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "compression-worker" spec: - replicas: 1 + replicas: {{ .Values.compressionWorker.replicas }} selector: matchLabels: {{- include "clp.selectorLabels" . | nindent 6 }} @@ -17,6 +17,10 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "compression-worker" spec: + {{- include "clp.createSchedulingConfigs" (dict + "root" . + "component" "compressionWorker" + ) | nindent 6 }} terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} diff --git a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml index 4b6d55466b..55f243cf8c 100644 --- a/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "compression-worker" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/compression_worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml index bc98706949..1934dc7bc5 100644 --- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -1,9 +1,8 @@ {{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "compression-worker" "name" "staged-archives" - "nodeRole" "worker" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" .Values.clpConfig.archive_output.storage.staging_directory diff --git a/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml index 43e46c2503..d7107e14ae 100644 --- a/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-tmp-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "compression-worker" "name" "tmp" - "nodeRole" "control-plane" "capacity" "10Gi" "accessModes" (list "ReadWriteOnce") "hostPath" .Values.clpConfig.tmp_directory diff --git a/tools/deployment/package-helm/templates/database-data-pv.yaml b/tools/deployment/package-helm/templates/database-data-pv.yaml index 3bd6ea2b9a..1456cb9f9f 100644 --- a/tools/deployment/package-helm/templates/database-data-pv.yaml +++ b/tools/deployment/package-helm/templates/database-data-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "database" "name" "data" - "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/database" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/database-logs-pv.yaml b/tools/deployment/package-helm/templates/database-logs-pv.yaml index 794215cf3a..c9f2e63793 100644 --- a/tools/deployment/package-helm/templates/database-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/database-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "database" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/database" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index 17b1a2e1ba..313940bd6b 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,11 +1,10 @@ {{- if or .Values.clpConfig.archive_output.retention_period .Values.clpConfig.results_cache.retention_period }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "garbage-collector" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/garbage_collector" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml b/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml index 6ed940bc0c..227ac47988 100644 --- a/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml @@ -1,9 +1,8 @@ {{- if .Values.clpConfig.log_ingestor }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "log-ingestor" "name" "logs" - "nodeRole" "control-plane" "capacity" "10Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/log_ingestor" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml index 9c53c9c7f1..fc297f5020 100644 --- a/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/mcp-server-logs-pv.yaml @@ -1,9 +1,8 @@ {{- if .Values.clpConfig.mcp_server }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "mcp-server" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/mcp_server" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml index de7633da55..bc66ac1ae3 100644 --- a/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-scheduler-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "query-scheduler" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query_scheduler" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 763710c4eb..33e4e20b86 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "query-worker" spec: - replicas: 1 + replicas: {{ .Values.queryWorker.replicas }} selector: matchLabels: {{- include "clp.selectorLabels" . | nindent 6 }} @@ -17,6 +17,10 @@ spec: {{- include "clp.labels" . | nindent 8 }} app.kubernetes.io/component: "query-worker" spec: + {{- include "clp.createSchedulingConfigs" (dict + "root" . + "component" "queryWorker" + ) | nindent 6 }} terminationGracePeriodSeconds: 60 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} diff --git a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml index 4f602a8c74..1fd12c2e84 100644 --- a/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "query-worker" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/query_worker" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml index 92ba1c858f..1223310166 100644 --- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -1,9 +1,8 @@ {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "query-worker" "name" "staged-streams" - "nodeRole" "worker" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" .Values.clpConfig.stream_output.storage.staging_directory diff --git a/tools/deployment/package-helm/templates/queue-logs-pv.yaml b/tools/deployment/package-helm/templates/queue-logs-pv.yaml index c9315630f5..37f45a5d4e 100644 --- a/tools/deployment/package-helm/templates/queue-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/queue-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "queue" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/queue" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/redis-data-pv.yaml b/tools/deployment/package-helm/templates/redis-data-pv.yaml index 56efc9d19c..5e30c1c1c4 100644 --- a/tools/deployment/package-helm/templates/redis-data-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-data-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "redis" "name" "data" - "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/redis" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/redis-logs-pv.yaml b/tools/deployment/package-helm/templates/redis-logs-pv.yaml index 7f03c8cdad..811b48a4d8 100644 --- a/tools/deployment/package-helm/templates/redis-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/redis-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "redis" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/redis" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/reducer-deployment.yaml b/tools/deployment/package-helm/templates/reducer-deployment.yaml index 51970785f7..0c8714f59e 100644 --- a/tools/deployment/package-helm/templates/reducer-deployment.yaml +++ b/tools/deployment/package-helm/templates/reducer-deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "clp.labels" . | nindent 4 }} app.kubernetes.io/component: "reducer" spec: - replicas: 1 + replicas: {{ .Values.reducer.replicas }} selector: matchLabels: {{- include "clp.selectorLabels" . | nindent 6 }} @@ -20,6 +20,10 @@ spec: serviceAccountName: {{ include "clp.fullname" . }}-job-watcher hostname: {{ include "clp.fullname" . }}-reducer subdomain: {{ include "clp.fullname" . }}-reducer + {{- include "clp.createSchedulingConfigs" (dict + "root" . + "component" "reducer" + ) | nindent 6 }} terminationGracePeriodSeconds: 10 securityContext: runAsUser: {{ .Values.securityContext.firstParty.uid }} diff --git a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml index 0aab54233a..4c467877ae 100644 --- a/tools/deployment/package-helm/templates/reducer-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/reducer-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "reducer" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/reducer" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml index 8410734b2e..9e8ccea38e 100644 --- a/tools/deployment/package-helm/templates/results-cache-data-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-data-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "results-cache" "name" "data" - "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/results_cache" .Values.clpConfig.data_directory) diff --git a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml index e76aaf90af..2064685ee0 100644 --- a/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/results-cache-logs-pv.yaml @@ -1,8 +1,7 @@ -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "results-cache" "name" "logs" - "nodeRole" "control-plane" "capacity" "5Gi" "accessModes" (list "ReadWriteOnce") "hostPath" (printf "%s/results_cache" .Values.clpConfig.logs_directory) diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml index 9c8ce5a7f3..455f522351 100644 --- a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml @@ -1,9 +1,8 @@ {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "shared-data" "name" "archives" - "nodeRole" "control-plane" "capacity" "50Gi" "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.archive_output.storage.directory diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml index 1a4b0a88e6..f13f8e09d0 100644 --- a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml @@ -1,9 +1,8 @@ {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} -{{- include "clp.createLocalPv" (dict +{{- include "clp.createStaticPv" (dict "root" . "component_category" "shared-data" "name" "streams" - "nodeRole" "control-plane" "capacity" "20Gi" "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.stream_output.storage.directory diff --git a/tools/deployment/package-helm/templates/storage-class.yaml b/tools/deployment/package-helm/templates/storage-class.yaml new file mode 100644 index 0000000000..acbb3c4334 --- /dev/null +++ b/tools/deployment/package-helm/templates/storage-class.yaml @@ -0,0 +1,10 @@ +{{- if eq .Values.storage.storageClassName "local-storage" }} +apiVersion: "storage.k8s.io/v1" +kind: "StorageClass" +metadata: + name: "local-storage" + labels: + {{- include "clp.labels" . | nindent 4 }} +provisioner: "kubernetes.io/no-provisioner" +volumeBindingMode: "WaitForFirstConsumer" +{{- end }} diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh deleted file mode 100755 index f88d5e04b4..0000000000 --- a/tools/deployment/package-helm/test.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash - -# TODO: Migrate into integration test - -set -o errexit -set -o nounset -set -o pipefail - -CLP_HOME="/tmp/clp" - -# Waits for all jobs to complete and all non-job pods to be ready. -# -# @param {int} timeout_seconds Overall timeout in seconds -# @param {int} poll_interval_seconds Interval between status checks -# @param {int} wait_timeout_seconds Timeout for each kubectl wait call -# @return {int} 0 on success, 1 on timeout -wait_for_pods() { - local timeout_seconds=$1 - local poll_interval_seconds=$2 - local wait_timeout_seconds=$3 - - echo "Waiting for all pods to be ready" \ - "(timeout=${timeout_seconds}s, poll=${poll_interval_seconds}s," \ - "wait=${wait_timeout_seconds}s)..." - - # Reset bash built-in SECONDS counter - SECONDS=0 - - while true; do - sleep "${poll_interval_seconds}" - kubectl get pods - - if kubectl wait job \ - --all \ - --for=condition=Complete \ - --timeout="${wait_timeout_seconds}s" 2>/dev/null \ - && kubectl wait pods \ - --all \ - --selector='!job-name' \ - --for=condition=Ready \ - --timeout="${wait_timeout_seconds}s" 2>/dev/null - then - echo "All jobs completed and services are ready." - return 0 - fi - - if [[ ${SECONDS} -ge ${timeout_seconds} ]]; then - echo "ERROR: Timed out waiting for pods to be ready" - return 1 - fi - - echo "---" - done -} - -kind delete cluster --name clp-test -rm -rf "$CLP_HOME" -mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ - "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \ - "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ - "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ - "$CLP_HOME/var/log/"{api_server,garbage_collector,log_ingestor,mcp_server} \ - "$CLP_HOME/var/tmp" \ - "$CLP_HOME/samples" - -# Download sample datasets in the background -wget -O - https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1 \ - | tar xz -C "$CLP_HOME/samples" & -SAMPLE_DOWNLOAD_PID=$! - -# Generate sample log file for garbage collector testing. -cat < /tmp/clp/samples/test-gc.jsonl -{"timestamp": $(date +%s%3N), "level": "INFO", "message": "User login successful"} -{"timestamp": $(date +%s%3N), "level": "ERROR", "message": "Database connection failed"} -EOF - -cat <