From 9a88d1e717f1f5392605f80583086a94797b2858 Mon Sep 17 00:00:00 2001
From: Junhao Liao <junhao@junhao.ca>
Date: Fri, 19 Dec 2025 05:50:15 -0500
Subject: [PATCH 1/9] feat(helm): Add S3 storage support for archives, streams,
 and logs input.

---
 tools/deployment/package-helm/Chart.yaml      |   2 +-
 .../package-helm/templates/_helpers.tpl       |  25 ++++
 .../templates/api-server-deployment.yaml      | 107 ++++++++++++++
 .../templates/api-server-logs-pv.yaml         |  11 ++
 .../templates/api-server-logs-pvc.yaml        |   9 ++
 .../templates/api-server-service.yaml         |  18 +++
 .../compression-scheduler-deployment.yaml     |  22 ++-
 .../compression-worker-deployment.yaml        |  36 ++++-
 ...compression-worker-staged-archives-pv.yaml |  11 ++
 ...ompression-worker-staged-archives-pvc.yaml |   9 ++
 .../package-helm/templates/configmap.yaml     | 136 +++++++++++++++++-
 .../garbage-collector-deployment.yaml         | 125 ++++++++++++++++
 .../templates/garbage-collector-logs-pv.yaml  |  13 ++
 .../templates/garbage-collector-logs-pvc.yaml |  11 ++
 .../templates/query-worker-deployment.yaml    |  40 +++++-
 .../query-worker-staged-streams-pv.yaml       |  11 ++
 .../query-worker-staged-streams-pvc.yaml      |   9 ++
 .../templates/webui-deployment.yaml           |  37 ++++-
 tools/deployment/package-helm/test.sh         |  13 +-
 tools/deployment/package-helm/values.yaml     |  22 +++
 20 files changed, 639 insertions(+), 28 deletions(-)
 create mode 100644 tools/deployment/package-helm/templates/api-server-deployment.yaml
 create mode 100644 tools/deployment/package-helm/templates/api-server-logs-pv.yaml
 create mode 100644 tools/deployment/package-helm/templates/api-server-logs-pvc.yaml
 create mode 100644 tools/deployment/package-helm/templates/api-server-service.yaml
 create mode 100644 tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml
 create mode 100644 tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml
 create mode 100644 tools/deployment/package-helm/templates/garbage-collector-deployment.yaml
 create mode 100644 tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml
 create mode 100644 tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml
 create mode 100644 tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml
 create mode 100644 tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml

diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml
index feeffca787..c2d6977c4b 100644
--- a/tools/deployment/package-helm/Chart.yaml
+++ b/tools/deployment/package-helm/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: "v2"
 name: "clp"
-version: "0.1.2-dev.10"
+version: "0.1.2-dev.12"
 description: "A Helm chart for CLP's (Compressed Log Processor) package deployment"
 type: "application"
 appVersion: "0.7.1-dev"
diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl
index 882c8a3b17..96097b11e0 100644
--- a/tools/deployment/package-helm/templates/_helpers.tpl
+++ b/tools/deployment/package-helm/templates/_helpers.tpl
@@ -247,6 +247,31 @@ hostPath:
   type: "Directory"
 {{- end }}
 
+{{/*
+Creates a volumeMount for the AWS config directory.
+
+@param {object} . Root template context
+@return {string} YAML-formatted volumeMount definition
+*/}}
+{{- define "clp.awsConfigVolumeMount" -}}
+name: "aws-config"
+mountPath: {{ .Values.clpConfig.aws_config_directory | quote }}
+readOnly: true
+{{- end }}
+
+{{/*
+Creates a volume for the AWS config directory.
+
+@param {object} . Root template context
+@return {string} YAML-formatted volume definition
+*/}}
+{{- define "clp.awsConfigVolume" -}}
+name: "aws-config"
+hostPath:
+  path: {{ .Values.clpConfig.aws_config_directory | quote }}
+  type: "Directory"
+{{- end }}
+
 {{/*
 Creates an initContainer that waits for a Kubernetes resource to be ready.
 
diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml
new file mode 100644
index 0000000000..26da76486f
--- /dev/null
+++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml
@@ -0,0 +1,107 @@
+{{- if .Values.clpConfig.api_server }}
+apiVersion: "apps/v1"
+kind: "Deployment"
+metadata:
+  name: {{ include "clp.fullname" . }}-api-server
+  labels:
+    {{- include "clp.labels" . | nindent 4 }}
+    app.kubernetes.io/component: "api-server"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "clp.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: "api-server"
+  template:
+    metadata:
+      labels:
+        {{- include "clp.labels" . | nindent 8 }}
+        app.kubernetes.io/component: "api-server"
+    spec:
+      serviceAccountName: {{ include "clp.fullname" . }}-job-watcher
+      terminationGracePeriodSeconds: 10
+      securityContext:
+        runAsUser: {{ .Values.securityContext.firstParty.uid }}
+        runAsGroup: {{ .Values.securityContext.firstParty.gid }}
+        fsGroup: {{ .Values.securityContext.firstParty.gid }}
+      initContainers:
+        - {{- include "clp.waitFor" (dict
+            "root" .
+            "type" "job"
+            "name" "db-table-creator"
+          ) | nindent 10 }}
+        - {{- include "clp.waitFor" (dict
+            "root" .
+            "type" "job"
+            "name" "results-cache-indices-creator"
+          ) | nindent 10 }}
+      containers:
+        - name: "api-server"
+          image: "{{ include "clp.image.ref" . }}"
+          imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
+          env:
+            - name: "CLP_DB_PASS"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "clp.fullname" . }}-database
+                  key: "password"
+            - name: "CLP_DB_USER"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "clp.fullname" . }}-database
+                  key: "username"
+            - name: "CLP_LOGS_DIR"
+              value: "/var/log/api_server"
+            - name: "RUST_LOG"
+              value: "INFO"
+          ports:
+            - name: "api-server"
+              containerPort: 3001
+          volumeMounts:
+            - name: {{ include "clp.volumeName" (dict
+                "component_category" "api-server"
+                "name" "logs"
+              ) | quote }}
+              mountPath: "/var/log/api_server"
+            - name: "config"
+              mountPath: "/etc/clp-config.yaml"
+              subPath: "clp-config.yaml"
+              readOnly: true
+            {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
+            - name: {{ include "clp.volumeName" (dict
+                "component_category" "shared-data"
+                "name" "streams"
+              ) | quote }}
+              mountPath: "/var/data/streams"
+            {{- end }}
+          command: [
+            "/opt/clp/bin/api_server",
+            "--host", "0.0.0.0",
+            "--port", "3001",
+            "--config", "/etc/clp-config.yaml"
+          ]
+          readinessProbe:
+            {{- include "clp.readinessProbeTimings" . | nindent 12 }}
+            httpGet: &api-server-health-check
+              path: "/health"
+              port: 3001
+          livenessProbe:
+            {{- include "clp.livenessProbeTimings" . | nindent 12 }}
+            httpGet: *api-server-health-check
+      volumes:
+        - {{- include "clp.pvcVolume" (dict
+            "root" .
+            "component_category" "api-server"
+            "name" "logs"
+          ) | nindent 10 }}
+        - name: "config"
+          configMap:
+            name: {{ include "clp.fullname" . }}-config
+        {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
+        - {{- include "clp.pvcVolume" (dict
+            "root" .
+            "component_category" "shared-data"
+            "name" "streams"
+          ) | nindent 10 }}
+        {{- end }}
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/api-server-logs-pv.yaml b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml
new file mode 100644
index 0000000000..89fd30dda8
--- /dev/null
+++ b/tools/deployment/package-helm/templates/api-server-logs-pv.yaml
@@ -0,0 +1,11 @@
+{{- if .Values.clpConfig.api_server }}
+{{- include "clp.createLocalPv" (dict
+  "root" .
+  "component_category" "api-server"
+  "name" "logs"
+  "nodeRole" "control-plane"
+  "capacity" "5Gi"
+  "accessModes" (list "ReadWriteOnce")
+  "hostPath" (printf "%s/api_server" .Values.clpConfig.logs_directory)
+) }}
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml
new file mode 100644
index 0000000000..d9429b6dad
--- /dev/null
+++ b/tools/deployment/package-helm/templates/api-server-logs-pvc.yaml
@@ -0,0 +1,9 @@
+{{- if .Values.clpConfig.api_server }}
+{{- include "clp.createPvc" (dict
+  "root" .
+  "component_category" "api-server"
+  "name" "logs"
+  "capacity" "5Gi"
+  "accessModes" (list "ReadWriteOnce")
+) }}
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/api-server-service.yaml b/tools/deployment/package-helm/templates/api-server-service.yaml
new file mode 100644
index 0000000000..0aed0e7efa
--- /dev/null
+++ b/tools/deployment/package-helm/templates/api-server-service.yaml
@@ -0,0 +1,18 @@
+{{- if .Values.clpConfig.api_server }}
+apiVersion: "v1"
+kind: "Service"
+metadata:
+  name: {{ include "clp.fullname" . }}-api-server
+  labels:
+    {{- include "clp.labels" . | nindent 4 }}
+    app.kubernetes.io/component: "api-server"
+spec:
+  type: "NodePort"
+  selector:
+    {{- include "clp.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: "api-server"
+  ports:
+    - port: 3001
+      targetPort: "api-server"
+      nodePort: {{ .Values.clpConfig.api_server.port }}
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml
index f58e9594b8..1f9bd0835c 100644
--- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml
+++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml
@@ -66,11 +66,6 @@ spec:
             - name: "PYTHONPATH"
               value: "/opt/clp/lib/python3/site-packages"
           volumeMounts:
-            - {{- include "clp.logsInputVolumeMount" . | nindent 14 }}
-            - name: "config"
-              mountPath: "/etc/clp-config.yaml"
-              subPath: "clp-config.yaml"
-              readOnly: true
             - name: {{ include "clp.volumeName" (dict
                 "component_category" "compression-scheduler"
                 "name" "logs"
@@ -81,13 +76,22 @@ spec:
                 "name" "user-logs"
               ) | quote }}
               mountPath: "/var/log/user"
+            - name: "config"
+              mountPath: "/etc/clp-config.yaml"
+              subPath: "clp-config.yaml"
+              readOnly: true
+            {{- if .Values.clpConfig.aws_config_directory }}
+            - {{- include "clp.awsConfigVolumeMount" . | nindent 14 }}
+            {{- end }}
+            {{- if eq .Values.clpConfig.logs_input.type "fs" }}
+            - {{- include "clp.logsInputVolumeMount" . | nindent 14 }}
+            {{- end }}
           command: [
             "python3", "-u",
             "-m", "job_orchestration.scheduler.compress.compression_scheduler",
             "--config", "/etc/clp-config.yaml"
           ]
       volumes:
-        - {{- include "clp.logsInputVolume" . | nindent 10 }}
         - {{- include "clp.pvcVolume" (dict
             "root" .
             "component_category" "compression-scheduler"
@@ -101,3 +105,9 @@ spec:
         - name: "config"
           configMap:
             name: {{ include "clp.fullname" . }}-config
+        {{- if .Values.clpConfig.aws_config_directory }}
+        - {{- include "clp.awsConfigVolume" . | nindent 10 }}
+        {{- end }}
+        {{- if eq .Values.clpConfig.logs_input.type "fs" }}
+        - {{- include "clp.logsInputVolume" . | nindent 10 }}
+        {{- end }}
diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml
index 70a6165154..efd2b3e0d5 100644
--- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml
+++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml
@@ -45,7 +45,6 @@ spec:
             - name: "PYTHONPATH"
               value: "/opt/clp/lib/python3/site-packages"
           volumeMounts:
-            - {{- include "clp.logsInputVolumeMount" . | nindent 14 }}
             - name: {{ include "clp.volumeName" (dict
                 "component_category" "compression-worker"
                 "name" "logs"
@@ -60,11 +59,25 @@ spec:
               mountPath: "/etc/clp-config.yaml"
               subPath: "clp-config.yaml"
               readOnly: true
+            {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }}
             - name: {{ include "clp.volumeName" (dict
                 "component_category" "shared-data"
                 "name" "archives"
               ) | quote }}
               mountPath: "/var/data/archives"
+            {{- else if eq .Values.clpConfig.archive_output.storage.type "s3" }}
+            - name: {{ include "clp.volumeName" (dict
+                "component_category" "compression-worker"
+                "name" "staged-archives"
+              ) | quote }}
+              mountPath: "/var/data/staged-archives"
+            {{- end }}
+            {{- if .Values.clpConfig.aws_config_directory }}
+            - {{- include "clp.awsConfigVolumeMount" . | nindent 14 }}
+            {{- end }}
+            {{- if eq .Values.clpConfig.logs_input.type "fs" }}
+            - {{- include "clp.logsInputVolumeMount" . | nindent 14 }}
+            {{- end }}
           command: [
             "python3", "-u",
             "/opt/clp/lib/python3/site-packages/bin/celery",
@@ -77,7 +90,6 @@ spec:
             "-n", "compression-worker"
           ]
       volumes:
-        - {{- include "clp.logsInputVolume" . | nindent 10 }}
         - {{- include "clp.pvcVolume" (dict
             "root" .
             "component_category" "compression-worker"
@@ -88,11 +100,25 @@ spec:
             "component_category" "compression-worker"
             "name" "tmp"
           ) | nindent 10 }}
+        - name: "config"
+          configMap:
+            name: {{ include "clp.fullname" . }}-config
+        {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }}
         - {{- include "clp.pvcVolume" (dict
             "root" .
             "component_category" "shared-data"
             "name" "archives"
           ) | nindent 10 }}
-        - name: "config"
-          configMap:
-            name: {{ include "clp.fullname" . }}-config
+        {{- else if eq .Values.clpConfig.archive_output.storage.type "s3" }}
+        - {{- include "clp.pvcVolume" (dict
+            "root" .
+            "component_category" "compression-worker"
+            "name" "staged-archives"
+          ) | nindent 10 }}
+        {{- end }}
+        {{- if .Values.clpConfig.aws_config_directory }}
+        - {{- include "clp.awsConfigVolume" . | nindent 10 }}
+        {{- end }}
+        {{- if eq .Values.clpConfig.logs_input.type "fs" }}
+        - {{- include "clp.logsInputVolume" . | nindent 10 }}
+        {{- end }}
diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml
new file mode 100644
index 0000000000..f017568b46
--- /dev/null
+++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml
@@ -0,0 +1,11 @@
+{{- if eq .Values.clpConfig.archive_output.storage.type "s3" }}
+{{- include "clp.createLocalPv" (dict
+  "root" .
+  "component_category" "compression-worker"
+  "name" "staged-archives"
+  "nodeRole" "worker"
+  "capacity" "50Gi"
+  "accessModes" (list "ReadWriteOnce")
+  "hostPath" (printf "%s/var/data/staged-archives" .Values.clpConfig.data_directory)
+) }}
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml
new file mode 100644
index 0000000000..b40e8720f9
--- /dev/null
+++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml
@@ -0,0 +1,9 @@
+{{- if eq .Values.clpConfig.archive_output.storage.type "s3" }}
+{{- include "clp.createPvc" (dict
+  "root" .
+  "component_category" "compression-worker"
+  "name" "staged-archives"
+  "capacity" "50Gi"
+  "accessModes" (list "ReadWriteOnce")
+) }}
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml
index dd9299b734..dcc3956a97 100644
--- a/tools/deployment/package-helm/templates/configmap.yaml
+++ b/tools/deployment/package-helm/templates/configmap.yaml
@@ -6,11 +6,56 @@ metadata:
     {{- include "clp.labels" . | nindent 4 }}
 data:
   clp-config.yaml: |
+    {{- with .Values.clpConfig.api_server }}
+    api_server:
+      default_max_num_query_results: {{ .default_max_num_query_results | int }}
+      host: "localhost"
+      port: 3001
+      query_job_polling:
+        initial_backoff_ms: {{ .query_job_polling.initial_backoff_ms | int }}
+        max_backoff_ms: {{ .query_job_polling.max_backoff_ms | int }}
+    {{- else }}
+    api_server: null
+    {{- end }}
     archive_output:
       compression_level: {{ .Values.clpConfig.archive_output.compression_level }}
+      {{- with .Values.clpConfig.archive_output.storage }}
       storage:
+        {{- if eq .type "fs" }}
         directory: "/var/data/archives"
         type: "fs"
+        {{- else if eq .type "s3" }}
+        type: "s3"
+        staging_directory: "/var/data/staged-archives"
+        {{- with .s3_config }}
+        s3_config:
+          {{- if .endpoint_url }}
+          endpoint_url: {{ .endpoint_url | quote }}
+          {{- end }}
+          {{- if .region_code }}
+          region_code: {{ .region_code | quote }}
+          {{- end }}
+          bucket: {{ .bucket | quote }}
+          key_prefix: {{ .key_prefix | quote }}
+          {{- with .aws_authentication }}
+          aws_authentication:
+            type: {{ .type | quote }}
+            {{- if .profile }}
+            profile: {{ .profile | quote }}
+            {{- end }}
+            {{- if .credentials }}
+            credentials:
+              access_key_id: {{ .credentials.access_key_id | quote }}
+              secret_access_key: {{ .credentials.secret_access_key | quote }}
+              {{- if .credentials.session_token }}
+              session_token: {{ .credentials.session_token | quote }}
+              {{- end }}
+            {{- end }}
+          {{- end }}{{/* with .aws_authentication */}}
+        {{- end }}{{/* with .s3_config */}}
+        {{- end }}{{/* if eq .type "fs" */}}
+      {{- end }}{{/* with .Values.clpConfig.archive_output.storage */}}
+      retention_period: {{ .Values.clpConfig.archive_output.retention_period | int }}
       target_archive_size: {{ .Values.clpConfig.archive_output.target_archive_size | int }}
       target_dictionaries_size: {{ .Values.clpConfig.archive_output.target_dictionaries_size
         | int }}
@@ -26,6 +71,11 @@ data:
     compression_worker:
       logging_level: {{ .Values.clpConfig.compression_worker.logging_level | quote }}
     data_directory: "/var/data"
+    garbage_collector:
+      logging_level: {{ .Values.clpConfig.garbage_collector.logging_level | quote }}
+      sweep_interval:
+        archive: {{ .Values.clpConfig.garbage_collector.sweep_interval.archive | int }}
+        search_result: {{ .Values.clpConfig.garbage_collector.sweep_interval.search_result | int }}
     database:
       auto_commit: false
       compress: true
@@ -36,9 +86,30 @@ data:
       ssl_cert: null
       type: {{ .Values.clpConfig.database.type | quote }}
     logs_directory: "/var/log"
+    {{- with .Values.clpConfig.logs_input }}
     logs_input:
+      {{- if eq .type "fs" }}
       directory: "/mnt/logs"
       type: "fs"
+      {{- else if eq .type "s3" }}
+      type: "s3"
+      {{- with .aws_authentication }}
+      aws_authentication:
+        type: {{ .type | quote }}
+        {{- if .profile }}
+        profile: {{ .profile | quote }}
+        {{- end }}
+        {{- if .credentials }}
+        credentials:
+          access_key_id: {{ .credentials.access_key_id | quote }}
+          secret_access_key: {{ .credentials.secret_access_key | quote }}
+          {{- if .credentials.session_token }}
+          session_token: {{ .credentials.session_token | quote }}
+          {{- end }}
+        {{- end }}
+      {{- end }}{{/* with .aws_authentication */}}
+      {{- end }}{{/* if eq .type "fs" */}}
+    {{- end }}{{/* with .Values.clpConfig.logs_input */}}
     package:
       query_engine: {{ .Values.clpConfig.package.query_engine | quote }}
       storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }}
@@ -68,13 +139,52 @@ data:
       db_name: {{ .Values.clpConfig.results_cache.db_name | quote }}
       host: {{ include "clp.fullname" . }}-results-cache
       port: 27017
+      retention_period: {{ .Values.clpConfig.results_cache.retention_period | int }}
       stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}
+    {{- with .Values.clpConfig.stream_output }}
     stream_output:
+      {{- with .storage }}
       storage:
+        {{- if eq .type "fs" }}
         directory: "/var/data/streams"
         type: "fs"
-      target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }}
+        {{- else if eq .type "s3" }}
+        type: "s3"
+        staging_directory: "/var/data/staged-streams"
+        {{- with .s3_config }}
+        s3_config:
+          {{- if .endpoint_url }}
+          endpoint_url: {{ .endpoint_url | quote }}
+          {{- end }}
+          {{- if .region_code }}
+          region_code: {{ .region_code | quote }}
+          {{- end }}
+          bucket: {{ .bucket | quote }}
+          key_prefix: {{ .key_prefix | quote }}
+          {{- with .aws_authentication }}
+          aws_authentication:
+            type: {{ .type | quote }}
+            {{- if .profile }}
+            profile: {{ .profile | quote }}
+            {{- end }}
+            {{- if .credentials }}
+            credentials:
+              access_key_id: {{ .credentials.access_key_id | quote }}
+              secret_access_key: {{ .credentials.secret_access_key | quote }}
+              {{- if .credentials.session_token }}
+              session_token: {{ .credentials.session_token | quote }}
+              {{- end }}
+            {{- end }}
+          {{- end }}{{/* with .aws_authentication */}}
+        {{- end }}{{/* with .s3_config */}}
+        {{- end }}{{/* if eq .type "fs" */}}
+      {{- end }}{{/* with .storage */}}
+      target_uncompressed_size: {{ .target_uncompressed_size | int }}
+    {{- end }}{{/* with .Values.clpConfig.stream_output */}}
     tmp_directory: "/var/tmp"
+    {{- if .Values.clpConfig.aws_config_directory }}
+    aws_config_directory: {{ .Values.clpConfig.aws_config_directory | quote }}
+    {{- end }}
     webui:
       host: "localhost"
       port: 4000
@@ -115,7 +225,11 @@ data:
       "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }},
       "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }},
       "LogsInputType": {{ .Values.clpConfig.logs_input.type | quote }},
+      {{- if eq .Values.clpConfig.logs_input.type "fs" }}
       "LogsInputRootDir": "/mnt/logs",
+      {{- else }}
+      "LogsInputRootDir": null,
+      {{- end }}
       "MongoDbSearchResultsMetadataCollectionName":
         {{ .Values.clpConfig.webui.results_metadata_collection_name | quote }},
       "SqlDbClpArchivesTableName": "clp_archives",
@@ -141,13 +255,33 @@ data:
         {{ .Values.clpConfig.results_cache.stream_collection_name | quote }},
       "ClientDir": "/opt/clp/var/www/webui/client",
       "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer",
+      {{- if eq .Values.clpConfig.logs_input.type "fs" }}
       "LogsInputRootDir": "/mnt/logs",
+      {{- else }}
+      "LogsInputRootDir": null,
+      {{- end }}
+      {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
       "StreamFilesDir": "/var/data/streams",
+      {{- else }}
+      "StreamFilesDir": null,
+      {{- end }}
       "StreamTargetUncompressedSize":
         {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }},
+      {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }}
+      {{- with .Values.clpConfig.stream_output.storage.s3_config }}
+      "StreamFilesS3Region": {{ .region_code | default "null" | quote }},
+      "StreamFilesS3PathPrefix": {{ printf "s3://%s/%s" .bucket .key_prefix | quote }},
+      {{- if eq .aws_authentication.type "profile" }}
+      "StreamFilesS3Profile": {{ .aws_authentication.profile | quote }},
+      {{- else }}
+      "StreamFilesS3Profile": null,
+      {{- end }}
+      {{- end }}{{/* with .Values.clpConfig.stream_output.storage.s3_config */}}
+      {{- else }}
       "StreamFilesS3Region": null,
       "StreamFilesS3PathPrefix": null,
       "StreamFilesS3Profile": null,
+      {{- end }}
       "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level }},
       "ArchiveOutputTargetArchiveSize":
         {{ .Values.clpConfig.archive_output.target_archive_size | int }},
diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml
new file mode 100644
index 0000000000..cccacd96bb
--- /dev/null
+++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml
@@ -0,0 +1,125 @@
+{{- $archiveRetention := .Values.clpConfig.archive_output.retention_period }}
+{{- $cacheRetention := .Values.clpConfig.results_cache.retention_period }}
+{{- if or $archiveRetention $cacheRetention }}
+apiVersion: "apps/v1"
+kind: "Deployment"
+metadata:
+  name: {{ include "clp.fullname" . }}-garbage-collector
+  labels:
+    {{- include "clp.labels" . | nindent 4 }}
+    app.kubernetes.io/component: "garbage-collector"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "clp.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: "garbage-collector"
+  template:
+    metadata:
+      labels:
+        {{- include "clp.labels" . | nindent 8 }}
+        app.kubernetes.io/component: "garbage-collector"
+    spec:
+      serviceAccountName: {{ include "clp.fullname" . }}-job-watcher
+      terminationGracePeriodSeconds: 10
+      securityContext:
+        runAsUser: {{ .Values.securityContext.firstParty.uid }}
+        runAsGroup: {{ .Values.securityContext.firstParty.gid }}
+        fsGroup: {{ .Values.securityContext.firstParty.gid }}
+      initContainers:
+        - {{- include "clp.waitFor" (dict
+            "root" .
+            "type" "job"
+            "name" "db-table-creator"
+          ) | nindent 10 }}
+        - {{- include "clp.waitFor" (dict
+            "root" .
+            "type" "job"
+            "name" "results-cache-indices-creator"
+          ) | nindent 10 }}
+      containers:
+        - name: "garbage-collector"
+          image: "{{ include "clp.image.ref" . }}"
+          imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
+          env:
+            - name: "CLP_DB_PASS"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "clp.fullname" . }}-database
+                  key: "password"
+            - name: "CLP_DB_USER"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "clp.fullname" . }}-database
+                  key: "username"
+            - name: "CLP_HOME"
+              value: "/opt/clp"
+            - name: "CLP_LOGGING_LEVEL"
+              value: {{ .Values.clpConfig.garbage_collector.logging_level | quote }}
+            - name: "CLP_LOGS_DIR"
+              value: "/var/log/garbage_collector"
+            - name: "PYTHONPATH"
+              value: "/opt/clp/lib/python3/site-packages"
+          volumeMounts:
+            - name: {{ include "clp.volumeName" (dict
+                "component_category" "garbage-collector"
+                "name" "logs"
+              ) | quote }}
+              mountPath: "/var/log/garbage_collector"
+            - name: "config"
+              mountPath: "/etc/clp-config.yaml"
+              subPath: "clp-config.yaml"
+              readOnly: true
+            - name: "tmp"
+              mountPath: "/var/log"
+            {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }}
+            - name: {{ include "clp.volumeName" (dict
+                "component_category" "shared-data"
+                "name" "archives"
+              ) | quote }}
+              mountPath: "/var/data/archives"
+            {{- end }}
+            {{- if .Values.clpConfig.aws_config_directory }}
+            - {{- include "clp.awsConfigVolumeMount" . | nindent 14 }}
+            {{- end }}
+            {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
+            - name: {{ include "clp.volumeName" (dict
+                "component_category" "shared-data"
+                "name" "streams"
+              ) | quote }}
+              mountPath: "/var/data/streams"
+            {{- end }}
+          command: [
+            "python3", "-u",
+            "-m", "job_orchestration.garbage_collector.garbage_collector",
+            "--config", "/etc/clp-config.yaml"
+          ]
+      volumes:
+        - {{- include "clp.pvcVolume" (dict
+            "root" .
+            "component_category" "garbage-collector"
+            "name" "logs"
+          ) | nindent 10 }}
+        - name: "config"
+          configMap:
+            name: {{ include "clp.fullname" . }}-config
+        - name: "tmp"
+          emptyDir: {}
+        {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }}
+        - {{- include "clp.pvcVolume" (dict
+            "root" .
+            "component_category" "shared-data"
+            "name" "archives"
+          ) | nindent 10 }}
+        {{- end }}
+        {{- if .Values.clpConfig.aws_config_directory }}
+        - {{- include "clp.awsConfigVolume" . | nindent 10 }}
+        {{- end }}
+        {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
+        - {{- include "clp.pvcVolume" (dict
+            "root" .
+            "component_category" "shared-data"
+            "name" "streams"
+          ) | nindent 10 }}
+        {{- end }}
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml
new file mode 100644
index 0000000000..ccabc3abfa
--- /dev/null
+++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml
@@ -0,0 +1,13 @@
+{{- $archiveRetention := .Values.clpConfig.archive_output.retention_period }}
+{{- $cacheRetention := .Values.clpConfig.results_cache.retention_period }}
+{{- if or $archiveRetention $cacheRetention }}
+{{- include "clp.createLocalPv" (dict
+  "root" .
+  "component_category" "garbage-collector"
+  "name" "logs"
+  "nodeRole" "control-plane"
+  "capacity" "5Gi"
+  "accessModes" (list "ReadWriteOnce")
+  "hostPath" (printf "%s/garbage_collector" .Values.clpConfig.logs_directory)
+) }}
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml
new file mode 100644
index 0000000000..bf31d6807a
--- /dev/null
+++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml
@@ -0,0 +1,11 @@
+{{- $archiveRetention := .Values.clpConfig.archive_output.retention_period }}
+{{- $cacheRetention := .Values.clpConfig.results_cache.retention_period }}
+{{- if or $archiveRetention $cacheRetention }}
+{{- include "clp.createPvc" (dict
+  "root" .
+  "component_category" "garbage-collector"
+  "name" "logs"
+  "capacity" "5Gi"
+  "accessModes" (list "ReadWriteOnce")
+) }}
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml
index f95e4ddaf7..763710c4eb 100644
--- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml
+++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml
@@ -45,25 +45,38 @@ spec:
             - name: "PYTHONPATH"
               value: "/opt/clp/lib/python3/site-packages"
           volumeMounts:
-            - name: "config"
-              mountPath: "/etc/clp-config.yaml"
-              subPath: "clp-config.yaml"
-              readOnly: true
             - name: {{ include "clp.volumeName" (dict
                 "component_category" "query-worker"
                 "name" "logs"
               ) | quote }}
               mountPath: "/var/log/query_worker"
+            - name: "config"
+              mountPath: "/etc/clp-config.yaml"
+              subPath: "clp-config.yaml"
+              readOnly: true
+            {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }}
             - name: {{ include "clp.volumeName" (dict
                 "component_category" "shared-data"
                 "name" "archives"
               ) | quote }}
               mountPath: "/var/data/archives"
+            {{- end }}
+            {{- if .Values.clpConfig.aws_config_directory }}
+            - {{- include "clp.awsConfigVolumeMount" . | nindent 14 }}
+            {{- end }}
+            {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
             - name: {{ include "clp.volumeName" (dict
                 "component_category" "shared-data"
                 "name" "streams"
               ) | quote }}
               mountPath: "/var/data/streams"
+            {{- else if eq .Values.clpConfig.stream_output.storage.type "s3" }}
+            - name: {{ include "clp.volumeName" (dict
+                "component_category" "query-worker"
+                "name" "staged-streams"
+              ) | quote }}
+              mountPath: "/var/data/staged-streams"
+            {{- end }}
           command: [
             "python3", "-u",
             "/opt/clp/lib/python3/site-packages/bin/celery",
@@ -81,16 +94,29 @@ spec:
             "component_category" "query-worker"
             "name" "logs"
           ) | nindent 10 }}
+        - name: "config"
+          configMap:
+            name: {{ include "clp.fullname" . }}-config
+        {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }}
         - {{- include "clp.pvcVolume" (dict
             "root" .
             "component_category" "shared-data"
             "name" "archives"
           ) | nindent 10 }}
+        {{- end }}
+        {{- if .Values.clpConfig.aws_config_directory }}
+        - {{- include "clp.awsConfigVolume" . | nindent 10 }}
+        {{- end }}
+        {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
         - {{- include "clp.pvcVolume" (dict
             "root" .
             "component_category" "shared-data"
             "name" "streams"
           ) | nindent 10 }}
-        - name: "config"
-          configMap:
-            name: {{ include "clp.fullname" . }}-config
+        {{- else if eq .Values.clpConfig.stream_output.storage.type "s3" }}
+        - {{- include "clp.pvcVolume" (dict
+            "root" .
+            "component_category" "query-worker"
+            "name" "staged-streams"
+          ) | nindent 10 }}
+        {{- end }}
diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml
new file mode 100644
index 0000000000..ee6cb197eb
--- /dev/null
+++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml
@@ -0,0 +1,11 @@
+{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }}
+{{- include "clp.createLocalPv" (dict
+  "root" .
+  "component_category" "query-worker"
+  "name" "staged-streams"
+  "nodeRole" "worker"
+  "capacity" "50Gi"
+  "accessModes" (list "ReadWriteOnce")
+  "hostPath" (printf "%s/var/data/staged-streams" .Values.clpConfig.data_directory)
+) }}
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml
new file mode 100644
index 0000000000..0fac69eeb5
--- /dev/null
+++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml
@@ -0,0 +1,9 @@
+{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }}
+{{- include "clp.createPvc" (dict
+  "root" .
+  "component_category" "query-worker"
+  "name" "staged-streams"
+  "capacity" "50Gi"
+  "accessModes" (list "ReadWriteOnce")
+) }}
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml
index ec4ae5c13a..78b9d6b0f8 100644
--- a/tools/deployment/package-helm/templates/webui-deployment.yaml
+++ b/tools/deployment/package-helm/templates/webui-deployment.yaml
@@ -59,11 +59,19 @@ spec:
               value: "4000"
             - name: "RATE_LIMIT"
               value: {{ .Values.clpConfig.webui.rate_limit | quote }}
+            {{- with .Values.clpConfig.stream_output.storage }}
+            {{- if and (eq .type "s3") (eq .s3_config.aws_authentication.type "credentials") }}
+            - name: "AWS_ACCESS_KEY_ID"
+              value: {{ .s3_config.aws_authentication.credentials.access_key_id | quote }}
+            - name: "AWS_SECRET_ACCESS_KEY"
+              value: {{ .s3_config.aws_authentication.credentials.secret_access_key | quote }}
+            {{- end }}{{/* if and (eq .type "s3")
+                                  (eq .s3_config.aws_authentication.type "credentials") */}}
+            {{- end }}{{/* with .Values.clpConfig.stream_output.storage */}}
           ports:
             - name: "webui"
               containerPort: 4000
           volumeMounts:
-            - {{- include "clp.logsInputVolumeMount" . | nindent 14 }}
             - name: "client-settings"
               mountPath: "/opt/clp/var/www/webui/client/settings.json"
               subPath: "webui-client-settings.json"
@@ -72,11 +80,19 @@ spec:
               mountPath: "/opt/clp/var/www/webui/server/dist/settings.json"
               subPath: "webui-server-settings.json"
               readOnly: true
+            {{- if .Values.clpConfig.aws_config_directory }}
+            - {{- include "clp.awsConfigVolumeMount" . | nindent 14 }}
+            {{- end }}
+            {{- if eq .Values.clpConfig.logs_input.type "fs" }}
+            - {{- include "clp.logsInputVolumeMount" . | nindent 14 }}
+            {{- end }}
+            {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
             - name: {{ include "clp.volumeName" (dict
                 "component_category" "shared-data"
                 "name" "streams"
               ) | quote }}
               mountPath: "/var/data/streams"
+            {{- end }}
           command: [
             "/opt/clp/bin/node-22",
             "/opt/clp/var/www/webui/server/dist/src/main.js"
@@ -89,15 +105,22 @@ spec:
             {{- include "clp.livenessProbeTimings" . | nindent 12 }}
             tcpSocket: *webui-health-check
       volumes:
-        - {{- include "clp.logsInputVolume" . | nindent 10 }}
-        - {{- include "clp.pvcVolume" (dict
-            "root" .
-            "component_category" "shared-data"
-            "name" "streams"
-          ) | nindent 10 }}
         - name: "client-settings"
           configMap:
             name: {{ include "clp.fullname" . }}-config
         - name: "server-settings"
           configMap:
             name: {{ include "clp.fullname" . }}-config
+        {{- if .Values.clpConfig.aws_config_directory }}
+        - {{- include "clp.awsConfigVolume" . | nindent 10 }}
+        {{- end }}
+        {{- if eq .Values.clpConfig.logs_input.type "fs" }}
+        - {{- include "clp.logsInputVolume" . | nindent 10 }}
+        {{- end }}
+        {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
+        - {{- include "clp.pvcVolume" (dict
+            "root" .
+            "component_category" "shared-data"
+            "name" "streams"
+          ) | nindent 10 }}
+        {{- end }}
diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh
index abdd857512..14f7ee5497 100755
--- a/tools/deployment/package-helm/test.sh
+++ b/tools/deployment/package-helm/test.sh
@@ -56,9 +56,11 @@ wait_for_pods() {
 kind delete cluster --name clp-test
 rm -rf "$CLP_HOME"
 mkdir -p  "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \
-          "$CLP_HOME/var/data/"{archives,streams} \
+          "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \
           "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \
           "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \
+          "$CLP_HOME/var/log/garbage_collector" \
+          "$CLP_HOME/var/log/api_server" \
           "$CLP_HOME/var/tmp" \
           "$CLP_HOME/samples"
 
@@ -67,6 +69,12 @@ wget -O - https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1
   | tar xz -C "$CLP_HOME/samples" &
 SAMPLE_DOWNLOAD_PID=$!
 
+# Generate sample log file for garbage collector testing.
+cat <<EOF > /tmp/clp/samples/test-gc.jsonl
+{"timestamp": $(date +%s%3N), "level": "INFO", "message": "User login successful"}
+{"timestamp": $(date +%s%3N), "level": "ERROR", "message": "Database connection failed"}
+EOF
+
 cat <<EOF | kind create cluster --name clp-test --config=-
   kind: Cluster
   apiVersion: kind.x-k8s.io/v1alpha4
@@ -87,6 +95,9 @@ cat <<EOF | kind create cluster --name clp-test --config=-
     - containerPort: 30000
       hostPort: 30000
       protocol: TCP
+    - containerPort: 30301
+      hostPort: 30301
+      protocol: TCP
 EOF
 
 helm uninstall test --ignore-not-found
diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml
index 005211e562..1b20dfd5c6 100644
--- a/tools/deployment/package-helm/values.yaml
+++ b/tools/deployment/package-helm/values.yaml
@@ -63,6 +63,9 @@ clpConfig:
     db_name: "clp-query-results"
     stream_collection_name: "stream-files"
 
+    # Retention period for search results, in minutes. Set to null to disable automatic deletion.
+    retention_period: 60
+
   compression_worker:
     logging_level: "INFO"
 
@@ -74,6 +77,14 @@ clpConfig:
     results_metadata_collection_name: "results-metadata"
     rate_limit: 1000
 
+  # API server config
+  api_server:
+    port: 30301
+    default_max_num_query_results: 1000
+    query_job_polling:
+      initial_backoff_ms: 100
+      max_backoff_ms: 5000
+
   # Where archives should be output to
   archive_output:
     storage:
@@ -113,6 +124,14 @@ clpConfig:
     # How large each stream file should be before being split into a new stream file
     target_uncompressed_size: 134217728  # 128 MB
 
+  garbage_collector:
+    logging_level: "INFO"
+
+    # Interval (in minutes) at which garbage collector jobs run
+    sweep_interval:
+      archive: 60
+      search_result: 30
+
   # Location where other data (besides archives) are stored. It will be created if
   # it doesn't exist.
   # NOTE: This directory must not overlap with any path used in CLP's execution container. An error
@@ -130,6 +149,9 @@ clpConfig:
   # will be raised if so.
   tmp_directory: "/tmp/clp/var/tmp"
 
+  # Location of the AWS tools' config files (e.g., `~/.aws`). Set to null to disable.
+  aws_config_directory: null
+
 credentials:
   database:
     username: "clp-user"

From d37168c63a0d32542cd56c6be2f257be4aff6646 Mon Sep 17 00:00:00 2001
From: Junhao Liao <junhao@junhao.ca>
Date: Fri, 19 Dec 2025 05:56:03 -0500
Subject: [PATCH 2/9] chore(helm): Bump chart version to 0.1.2-dev.15

---
 tools/deployment/package-helm/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml
index c2d6977c4b..e3fa6a2035 100644
--- a/tools/deployment/package-helm/Chart.yaml
+++ b/tools/deployment/package-helm/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: "v2"
 name: "clp"
-version: "0.1.2-dev.12"
+version: "0.1.2-dev.15"
 description: "A Helm chart for CLP's (Compressed Log Processor) package deployment"
 type: "application"
 appVersion: "0.7.1-dev"

From ffb7199b8ec976b2cf308ef531933101865d7552 Mon Sep 17 00:00:00 2001
From: Junhao Liao <junhao@junhao.ca>
Date: Mon, 5 Jan 2026 14:37:29 -0500
Subject: [PATCH 3/9] style(helm): Format conditional statements for better
 readability in garbage collector YAML files

---
 .../package-helm/templates/garbage-collector-deployment.yaml  | 4 +++-
 .../package-helm/templates/garbage-collector-logs-pv.yaml     | 4 +++-
 .../package-helm/templates/garbage-collector-logs-pvc.yaml    | 4 +++-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml
index fb0aebfb36..b2926231f0 100644
--- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml
+++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml
@@ -1,4 +1,6 @@
-{{- if or .Values.clpConfig.archive_output.retention_period .Values.clpConfig.results_cache.retention_period }}
+{{- if or
+  .Values.clpConfig.archive_output.retention_period
+  .Values.clpConfig.results_cache.retention_period }}
 apiVersion: "apps/v1"
 kind: "Deployment"
 metadata:
diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml
index 608e03739b..17b1a2e1ba 100644
--- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml
+++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml
@@ -1,4 +1,6 @@
-{{- if or .Values.clpConfig.archive_output.retention_period .Values.clpConfig.results_cache.retention_period }}
+{{- if or
+  .Values.clpConfig.archive_output.retention_period
+  .Values.clpConfig.results_cache.retention_period }}
 {{- include "clp.createLocalPv" (dict
   "root" .
   "component_category" "garbage-collector"
diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml
index 9f40e096e8..00f570d57a 100644
--- a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml
+++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml
@@ -1,4 +1,6 @@
-{{- if or .Values.clpConfig.archive_output.retention_period .Values.clpConfig.results_cache.retention_period }}
+{{- if or
+  .Values.clpConfig.archive_output.retention_period
+  .Values.clpConfig.results_cache.retention_period }}
 {{- include "clp.createPvc" (dict
   "root" .
   "component_category" "garbage-collector"

From 0f8c6b4a064492291e1daa2dfc96f660ed1fec04 Mon Sep 17 00:00:00 2001
From: Junhao Liao <junhao@junhao.ca>
Date: Mon, 5 Jan 2026 15:37:42 -0500
Subject: [PATCH 4/9] Conditionally enable shared data archive and streams only
 when the type is fs

---
 .../package-helm/templates/shared-data-archives-pv.yaml         | 2 ++
 .../package-helm/templates/shared-data-archives-pvc.yaml        | 2 ++
 .../package-helm/templates/shared-data-streams-pv.yaml          | 2 ++
 .../package-helm/templates/shared-data-streams-pvc.yaml         | 2 ++
 4 files changed, 8 insertions(+)

diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml
index 3e17545925..423e9f12eb 100644
--- a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml
+++ b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml
@@ -1,3 +1,4 @@
+{{- if eq .Values.clpConfig.archive_output.storage.type "fs" }}
 {{- include "clp.createLocalPv" (dict
   "root" .
   "component_category" "shared-data"
@@ -7,3 +8,4 @@
   "accessModes" (list "ReadWriteMany")
   "hostPath" .Values.clpConfig.archive_output.storage.directory
 ) }}
+{{- end }}
\ No newline at end of file
diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml
index 1cbc9c3f4d..279b4aa5f6 100644
--- a/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml
+++ b/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml
@@ -1,3 +1,4 @@
+{{- if eq .Values.clpConfig.archive_output.storage.type "fs" }}
 {{- include "clp.createPvc" (dict
   "root" .
   "component_category" "shared-data"
@@ -5,3 +6,4 @@
   "capacity" "50Gi"
   "accessModes" (list "ReadWriteMany")
 ) }}
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml
index 8cc2cd8019..c958eef2a8 100644
--- a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml
+++ b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml
@@ -1,3 +1,4 @@
+{{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
 {{- include "clp.createLocalPv" (dict
   "root" .
   "component_category" "shared-data"
@@ -7,3 +8,4 @@
   "accessModes" (list "ReadWriteMany")
   "hostPath" .Values.clpConfig.stream_output.storage.directory
 ) }}
+{{- end }}
\ No newline at end of file
diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml
index 3477876d73..f35f3a8ef6 100644
--- a/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml
+++ b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml
@@ -1,3 +1,4 @@
+{{- if eq .Values.clpConfig.stream_output.storage.type "fs" }}
 {{- include "clp.createPvc" (dict
   "root" .
   "component_category" "shared-data"
@@ -5,3 +6,4 @@
   "capacity" "20Gi"
   "accessModes" (list "ReadWriteMany")
 ) }}
+{{- end }}
\ No newline at end of file

From 2ce299959e2cef851c986a0578b793d359dc95f9 Mon Sep 17 00:00:00 2001
From: Junhao Liao <junhao@junhao.ca>
Date: Mon, 5 Jan 2026 15:38:01 -0500
Subject: [PATCH 5/9] fix(storage): Update hostPath for staged archives and
 streams

---
 .../templates/compression-worker-staged-archives-pv.yaml        | 2 +-
 .../package-helm/templates/query-worker-staged-streams-pv.yaml  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml
index f017568b46..280ad461be 100644
--- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml
+++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml
@@ -6,6 +6,6 @@
   "nodeRole" "worker"
   "capacity" "50Gi"
   "accessModes" (list "ReadWriteOnce")
-  "hostPath" (printf "%s/var/data/staged-archives" .Values.clpConfig.data_directory)
+  "hostPath" .Values.clpConfig.archive_output.storage.staging_directory
 ) }}
 {{- end }}
diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml
index ee6cb197eb..b62dd44c68 100644
--- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml
+++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml
@@ -6,6 +6,6 @@
   "nodeRole" "worker"
   "capacity" "50Gi"
   "accessModes" (list "ReadWriteOnce")
-  "hostPath" (printf "%s/var/data/staged-streams" .Values.clpConfig.data_directory)
+  "hostPath" .Values.clpConfig.stream_output.storage.staging_directory
 ) }}
 {{- end }}

From ea9decb2a39feca49d76e46ed157433e8f6e6eb6 Mon Sep 17 00:00:00 2001
From: Junhao Liao <junhao@junhao.ca>
Date: Mon, 5 Jan 2026 15:58:55 -0500
Subject: [PATCH 6/9] fix(helm): Correct S3 path prefix formatting for webui
 config in configmap.yaml

---
 tools/deployment/package-helm/templates/configmap.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml
index 42caabd26a..3339ac909f 100644
--- a/tools/deployment/package-helm/templates/configmap.yaml
+++ b/tools/deployment/package-helm/templates/configmap.yaml
@@ -278,7 +278,7 @@ data:
       {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }}
       {{- with .Values.clpConfig.stream_output.storage.s3_config }}
       "StreamFilesS3Region": {{ .region_code | default "null" | quote }},
-      "StreamFilesS3PathPrefix": {{ printf "s3://%s/%s" .bucket .key_prefix | quote }},
+      "StreamFilesS3PathPrefix": {{ printf "%s/%s" .bucket .key_prefix | quote }},
       {{- if eq .aws_authentication.type "profile" }}
       "StreamFilesS3Profile": {{ .aws_authentication.profile | quote }},
       {{- else }}

From 715163ad9a9fa852bf23d565e199111b657770b6 Mon Sep 17 00:00:00 2001
From: Junhao Liao <junhao@junhao.ca>
Date: Mon, 5 Jan 2026 16:11:12 -0500
Subject: [PATCH 7/9] fix(helm): Add missing newlines at the end of shared data
 YAML files

---
 .../package-helm/templates/shared-data-archives-pv.yaml         | 2 +-
 .../package-helm/templates/shared-data-streams-pv.yaml          | 2 +-
 .../package-helm/templates/shared-data-streams-pvc.yaml         | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml
index 423e9f12eb..9c8ce5a7f3 100644
--- a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml
+++ b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml
@@ -8,4 +8,4 @@
   "accessModes" (list "ReadWriteMany")
   "hostPath" .Values.clpConfig.archive_output.storage.directory
 ) }}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml
index c958eef2a8..1a4b0a88e6 100644
--- a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml
+++ b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml
@@ -8,4 +8,4 @@
   "accessModes" (list "ReadWriteMany")
   "hostPath" .Values.clpConfig.stream_output.storage.directory
 ) }}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml
index f35f3a8ef6..535be374d2 100644
--- a/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml
+++ b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml
@@ -6,4 +6,4 @@
   "capacity" "20Gi"
   "accessModes" (list "ReadWriteMany")
 ) }}
-{{- end }}
\ No newline at end of file
+{{- end }}

From 2e35de04d67a7d0b0f4f6c996152c64e185d7149 Mon Sep 17 00:00:00 2001
From: Junhao Liao <junhao@junhao.ca>
Date: Tue, 6 Jan 2026 16:09:04 -0500
Subject: [PATCH 8/9] fix(helm): Reduce storage capacity for staged archives
 and streams from 50Gi to 20Gi

---
 .../templates/compression-worker-staged-archives-pv.yaml        | 2 +-
 .../templates/compression-worker-staged-archives-pvc.yaml       | 2 +-
 .../package-helm/templates/query-worker-staged-streams-pv.yaml  | 2 +-
 .../package-helm/templates/query-worker-staged-streams-pvc.yaml | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml
index 280ad461be..bc98706949 100644
--- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml
+++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml
@@ -4,7 +4,7 @@
   "component_category" "compression-worker"
   "name" "staged-archives"
   "nodeRole" "worker"
-  "capacity" "50Gi"
+  "capacity" "20Gi"
   "accessModes" (list "ReadWriteOnce")
   "hostPath" .Values.clpConfig.archive_output.storage.staging_directory
 ) }}
diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml
index b40e8720f9..b8a9367236 100644
--- a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml
+++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml
@@ -3,7 +3,7 @@
   "root" .
   "component_category" "compression-worker"
   "name" "staged-archives"
-  "capacity" "50Gi"
+  "capacity" "20Gi"
   "accessModes" (list "ReadWriteOnce")
 ) }}
 {{- end }}
diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml
index b62dd44c68..92ba1c858f 100644
--- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml
+++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml
@@ -4,7 +4,7 @@
   "component_category" "query-worker"
   "name" "staged-streams"
   "nodeRole" "worker"
-  "capacity" "50Gi"
+  "capacity" "20Gi"
   "accessModes" (list "ReadWriteOnce")
   "hostPath" .Values.clpConfig.stream_output.storage.staging_directory
 ) }}
diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml
index 0fac69eeb5..e909a57b21 100644
--- a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml
+++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml
@@ -3,7 +3,7 @@
   "root" .
   "component_category" "query-worker"
   "name" "staged-streams"
-  "capacity" "50Gi"
+  "capacity" "20Gi"
   "accessModes" (list "ReadWriteOnce")
 ) }}
 {{- end }}

From c9dcb526ea39bbe478fc0502a2d16c9f6ccbb493 Mon Sep 17 00:00:00 2001
From: Junhao Liao <junhao.liao@yscope.com>
Date: Wed, 7 Jan 2026 15:21:24 -0500
Subject: [PATCH 9/9] bump chart version

---
 tools/deployment/package-helm/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml
index e3fa6a2035..192d61a98f 100644
--- a/tools/deployment/package-helm/Chart.yaml
+++ b/tools/deployment/package-helm/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: "v2"
 name: "clp"
-version: "0.1.2-dev.15"
+version: "0.1.2-dev.16"
 description: "A Helm chart for CLP's (Compressed Log Processor) package deployment"
 type: "application"
 appVersion: "0.7.1-dev"