diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index e3fa6a2035..192d61a98f 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.1.2-dev.15" +version: "0.1.2-dev.16" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.7.1-dev" diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 882c8a3b17..96097b11e0 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -247,6 +247,31 @@ hostPath: type: "Directory" {{- end }} +{{/* +Creates a volumeMount for the AWS config directory. + +@param {object} . Root template context +@return {string} YAML-formatted volumeMount definition +*/}} +{{- define "clp.awsConfigVolumeMount" -}} +name: "aws-config" +mountPath: {{ .Values.clpConfig.aws_config_directory | quote }} +readOnly: true +{{- end }} + +{{/* +Creates a volume for the AWS config directory. + +@param {object} . Root template context +@return {string} YAML-formatted volume definition +*/}} +{{- define "clp.awsConfigVolume" -}} +name: "aws-config" +hostPath: + path: {{ .Values.clpConfig.aws_config_directory | quote }} + type: "Directory" +{{- end }} + {{/* Creates an initContainer that waits for a Kubernetes resource to be ready. diff --git a/tools/deployment/package-helm/templates/api-server-deployment.yaml b/tools/deployment/package-helm/templates/api-server-deployment.yaml index 0231757f62..4ecaea6e4e 100644 --- a/tools/deployment/package-helm/templates/api-server-deployment.yaml +++ b/tools/deployment/package-helm/templates/api-server-deployment.yaml @@ -67,11 +67,13 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" "name" "streams" ) | quote }} mountPath: "/var/data/streams" + {{- end }} command: [ "/opt/clp/bin/api_server", "--host", "0.0.0.0", @@ -92,12 +94,14 @@ spec: "component_category" "api-server" "name" "logs" ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "streams" ) | nindent 10 }} - - name: "config" - configMap: - name: {{ include "clp.fullname" . }}-config + {{- end }} {{- end }} diff --git a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml index af8b9e225a..1f9bd0835c 100644 --- a/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml @@ -66,7 +66,6 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} - name: {{ include "clp.volumeName" (dict "component_category" "compression-scheduler" "name" "logs" @@ -81,13 +80,18 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true + {{- if .Values.clpConfig.aws_config_directory }} + - {{- include "clp.awsConfigVolumeMount" . | nindent 14 }} + {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} + {{- end }} command: [ "python3", "-u", "-m", "job_orchestration.scheduler.compress.compression_scheduler", "--config", "/etc/clp-config.yaml" ] volumes: - - {{- include "clp.logsInputVolume" . | nindent 10 }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "compression-scheduler" @@ -101,3 +105,9 @@ spec: - name: "config" configMap: name: {{ include "clp.fullname" . }}-config + {{- if .Values.clpConfig.aws_config_directory }} + - {{- include "clp.awsConfigVolume" . | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml index 70a6165154..efd2b3e0d5 100644 --- a/tools/deployment/package-helm/templates/compression-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/compression-worker-deployment.yaml @@ -45,7 +45,6 @@ spec: - name: "PYTHONPATH" value: "/opt/clp/lib/python3/site-packages" volumeMounts: - - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} - name: {{ include "clp.volumeName" (dict "component_category" "compression-worker" "name" "logs" @@ -60,11 +59,25 @@ spec: mountPath: "/etc/clp-config.yaml" subPath: "clp-config.yaml" readOnly: true + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" "name" "archives" ) | quote }} mountPath: "/var/data/archives" + {{- else if eq .Values.clpConfig.archive_output.storage.type "s3" }} + - name: {{ include "clp.volumeName" (dict + "component_category" "compression-worker" + "name" "staged-archives" + ) | quote }} + mountPath: "/var/data/staged-archives" + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - {{- include "clp.awsConfigVolumeMount" . | nindent 14 }} + {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} + {{- end }} command: [ "python3", "-u", "/opt/clp/lib/python3/site-packages/bin/celery", @@ -77,7 +90,6 @@ spec: "-n", "compression-worker" ] volumes: - - {{- include "clp.logsInputVolume" . | nindent 10 }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "compression-worker" @@ -88,11 +100,25 @@ spec: "component_category" "compression-worker" "name" "tmp" ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "archives" ) | nindent 10 }} - - name: "config" - configMap: - name: {{ include "clp.fullname" . }}-config + {{- else if eq .Values.clpConfig.archive_output.storage.type "s3" }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "compression-worker" + "name" "staged-archives" + ) | nindent 10 }} + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - {{- include "clp.awsConfigVolume" . | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml new file mode 100644 index 0000000000..bc98706949 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pv.yaml @@ -0,0 +1,11 @@ +{{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} +{{- include "clp.createLocalPv" (dict + "root" . + "component_category" "compression-worker" + "name" "staged-archives" + "nodeRole" "worker" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" .Values.clpConfig.archive_output.storage.staging_directory +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml new file mode 100644 index 0000000000..b8a9367236 --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-worker-staged-archives-pvc.yaml @@ -0,0 +1,9 @@ +{{- if eq .Values.clpConfig.archive_output.storage.type "s3" }} +{{- include "clp.createPvc" (dict + "root" . + "component_category" "compression-worker" + "name" "staged-archives" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 067c349d36..5cb9aa1dc1 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -19,9 +19,42 @@ data: {{- end }} archive_output: compression_level: {{ .Values.clpConfig.archive_output.compression_level | int }} + {{- with .Values.clpConfig.archive_output.storage }} storage: + {{- if eq .type "fs" }} directory: "/var/data/archives" type: "fs" + {{- else if eq .type "s3" }} + type: "s3" + staging_directory: "/var/data/staged-archives" + {{- with .s3_config }} + s3_config: + {{- if .endpoint_url }} + endpoint_url: {{ .endpoint_url | quote }} + {{- end }} + {{- if .region_code }} + region_code: {{ .region_code | quote }} + {{- end }} + bucket: {{ .bucket | quote }} + key_prefix: {{ .key_prefix | quote }} + {{- with .aws_authentication }} + aws_authentication: + type: {{ .type | quote }} + {{- if .profile }} + profile: {{ .profile | quote }} + {{- end }} + {{- if .credentials }} + credentials: + access_key_id: {{ .credentials.access_key_id | quote }} + secret_access_key: {{ .credentials.secret_access_key | quote }} + {{- if .credentials.session_token }} + session_token: {{ .credentials.session_token | quote }} + {{- end }} + {{- end }} + {{- end }}{{/* with .aws_authentication */}} + {{- end }}{{/* with .s3_config */}} + {{- end }}{{/* if eq .type "fs" */}} + {{- end }}{{/* with .Values.clpConfig.archive_output.storage */}} {{- if .Values.clpConfig.archive_output.retention_period }} retention_period: {{ .Values.clpConfig.archive_output.retention_period | int }} {{- else }} @@ -58,9 +91,30 @@ data: ssl_cert: null type: {{ .Values.clpConfig.database.type | quote }} logs_directory: "/var/log" + {{- with .Values.clpConfig.logs_input }} logs_input: + {{- if eq .type "fs" }} directory: "/mnt/logs" type: "fs" + {{- else if eq .type "s3" }} + type: "s3" + {{- with .aws_authentication }} + aws_authentication: + type: {{ .type | quote }} + {{- if .profile }} + profile: {{ .profile | quote }} + {{- end }} + {{- if .credentials }} + credentials: + access_key_id: {{ .credentials.access_key_id | quote }} + secret_access_key: {{ .credentials.secret_access_key | quote }} + {{- if .credentials.session_token }} + session_token: {{ .credentials.session_token | quote }} + {{- end }} + {{- end }} + {{- end }}{{/* with .aws_authentication */}} + {{- end }}{{/* if eq .type "fs" */}} + {{- end }}{{/* with .Values.clpConfig.logs_input */}} package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} @@ -96,12 +150,50 @@ data: retention_period: null {{- end }} stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} + {{- with .Values.clpConfig.stream_output }} stream_output: + {{- with .storage }} storage: + {{- if eq .type "fs" }} directory: "/var/data/streams" type: "fs" - target_uncompressed_size: {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }} + {{- else if eq .type "s3" }} + type: "s3" + staging_directory: "/var/data/staged-streams" + {{- with .s3_config }} + s3_config: + {{- if .endpoint_url }} + endpoint_url: {{ .endpoint_url | quote }} + {{- end }} + {{- if .region_code }} + region_code: {{ .region_code | quote }} + {{- end }} + bucket: {{ .bucket | quote }} + key_prefix: {{ .key_prefix | quote }} + {{- with .aws_authentication }} + aws_authentication: + type: {{ .type | quote }} + {{- if .profile }} + profile: {{ .profile | quote }} + {{- end }} + {{- if .credentials }} + credentials: + access_key_id: {{ .credentials.access_key_id | quote }} + secret_access_key: {{ .credentials.secret_access_key | quote }} + {{- if .credentials.session_token }} + session_token: {{ .credentials.session_token | quote }} + {{- end }} + {{- end }} + {{- end }}{{/* with .aws_authentication */}} + {{- end }}{{/* with .s3_config */}} + {{- end }}{{/* if eq .type "fs" */}} + {{- end }}{{/* with .storage */}} + target_uncompressed_size: {{ .target_uncompressed_size | int }} + {{- end }}{{/* with .Values.clpConfig.stream_output */}} tmp_directory: "/var/tmp" + {{- if .Values.clpConfig.aws_config_directory }} + aws_config_directory: {{ .Values.clpConfig.aws_config_directory | quote }} + {{- end }} webui: host: "localhost" port: 4000 @@ -150,7 +242,11 @@ data: "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }}, "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }}, "LogsInputType": {{ .Values.clpConfig.logs_input.type | quote }}, + {{- if eq .Values.clpConfig.logs_input.type "fs" }} "LogsInputRootDir": "/mnt/logs", + {{- else }} + "LogsInputRootDir": null, + {{- end }} "MongoDbSearchResultsMetadataCollectionName": {{ .Values.clpConfig.webui.results_metadata_collection_name | quote }}, "SqlDbClpArchivesTableName": "clp_archives", @@ -176,13 +272,33 @@ data: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }}, "ClientDir": "/opt/clp/var/www/webui/client", "LogViewerDir": "/opt/clp/var/www/webui/yscope-log-viewer", + {{- if eq .Values.clpConfig.logs_input.type "fs" }} "LogsInputRootDir": "/mnt/logs", + {{- else }} + "LogsInputRootDir": null, + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} "StreamFilesDir": "/var/data/streams", + {{- else }} + "StreamFilesDir": null, + {{- end }} "StreamTargetUncompressedSize": {{ .Values.clpConfig.stream_output.target_uncompressed_size | int }}, + {{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} + {{- with .Values.clpConfig.stream_output.storage.s3_config }} + "StreamFilesS3Region": {{ .region_code | default "null" | quote }}, + "StreamFilesS3PathPrefix": {{ printf "%s/%s" .bucket .key_prefix | quote }}, + {{- if eq .aws_authentication.type "profile" }} + "StreamFilesS3Profile": {{ .aws_authentication.profile | quote }}, + {{- else }} + "StreamFilesS3Profile": null, + {{- end }} + {{- end }}{{/* with .Values.clpConfig.stream_output.storage.s3_config */}} + {{- else }} "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, + {{- end }} "ArchiveOutputCompressionLevel": {{ .Values.clpConfig.archive_output.compression_level | int }}, "ArchiveOutputTargetArchiveSize": {{ .Values.clpConfig.archive_output.target_archive_size | int }}, diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml index 2e1a8a9811..b2926231f0 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -1,4 +1,6 @@ -{{- if or .Values.clpConfig.archive_output.retention_period .Values.clpConfig.results_cache.retention_period }} +{{- if or + .Values.clpConfig.archive_output.retention_period + .Values.clpConfig.results_cache.retention_period }} apiVersion: "apps/v1" kind: "Deployment" metadata: @@ -64,22 +66,29 @@ spec: "name" "logs" ) | quote }} mountPath: "/var/log/garbage_collector" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true + - name: "tmp" + mountPath: "/var/log" + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" "name" "archives" ) | quote }} mountPath: "/var/data/archives" + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - {{- include "clp.awsConfigVolumeMount" . | nindent 14 }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" "name" "streams" ) | quote }} mountPath: "/var/data/streams" - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true - - name: "tmp" - mountPath: "/var/log" + {{- end }} command: [ "python3", "-u", "-m", "job_orchestration.garbage_collector.garbage_collector", @@ -91,19 +100,26 @@ spec: "component_category" "garbage-collector" "name" "logs" ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + - name: "tmp" + emptyDir: {} + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "archives" ) | nindent 10 }} + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - {{- include "clp.awsConfigVolume" . | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "streams" ) | nindent 10 }} - - name: "config" - configMap: - name: {{ include "clp.fullname" . }}-config - - name: "tmp" - emptyDir: {} + {{- end }} {{- end }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml index 608e03739b..17b1a2e1ba 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -1,4 +1,6 @@ -{{- if or .Values.clpConfig.archive_output.retention_period .Values.clpConfig.results_cache.retention_period }} +{{- if or + .Values.clpConfig.archive_output.retention_period + .Values.clpConfig.results_cache.retention_period }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "garbage-collector" diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml index 9f40e096e8..00f570d57a 100644 --- a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml @@ -1,4 +1,6 @@ -{{- if or .Values.clpConfig.archive_output.retention_period .Values.clpConfig.results_cache.retention_period }} +{{- if or + .Values.clpConfig.archive_output.retention_period + .Values.clpConfig.results_cache.retention_period }} {{- include "clp.createPvc" (dict "root" . "component_category" "garbage-collector" diff --git a/tools/deployment/package-helm/templates/query-worker-deployment.yaml b/tools/deployment/package-helm/templates/query-worker-deployment.yaml index 8acf0a0494..763710c4eb 100644 --- a/tools/deployment/package-helm/templates/query-worker-deployment.yaml +++ b/tools/deployment/package-helm/templates/query-worker-deployment.yaml @@ -50,20 +50,33 @@ spec: "name" "logs" ) | quote }} mountPath: "/var/log/query_worker" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" "name" "archives" ) | quote }} mountPath: "/var/data/archives" + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - {{- include "clp.awsConfigVolumeMount" . | nindent 14 }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - name: {{ include "clp.volumeName" (dict "component_category" "shared-data" "name" "streams" ) | quote }} mountPath: "/var/data/streams" - - name: "config" - mountPath: "/etc/clp-config.yaml" - subPath: "clp-config.yaml" - readOnly: true + {{- else if eq .Values.clpConfig.stream_output.storage.type "s3" }} + - name: {{ include "clp.volumeName" (dict + "component_category" "query-worker" + "name" "staged-streams" + ) | quote }} + mountPath: "/var/data/staged-streams" + {{- end }} command: [ "python3", "-u", "/opt/clp/lib/python3/site-packages/bin/celery", @@ -81,16 +94,29 @@ spec: "component_category" "query-worker" "name" "logs" ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + {{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "archives" ) | nindent 10 }} + {{- end }} + {{- if .Values.clpConfig.aws_config_directory }} + - {{- include "clp.awsConfigVolume" . | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} - {{- include "clp.pvcVolume" (dict "root" . "component_category" "shared-data" "name" "streams" ) | nindent 10 }} - - name: "config" - configMap: - name: {{ include "clp.fullname" . }}-config + {{- else if eq .Values.clpConfig.stream_output.storage.type "s3" }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "query-worker" + "name" "staged-streams" + ) | nindent 10 }} + {{- end }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml new file mode 100644 index 0000000000..92ba1c858f --- /dev/null +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pv.yaml @@ -0,0 +1,11 @@ +{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} +{{- include "clp.createLocalPv" (dict + "root" . + "component_category" "query-worker" + "name" "staged-streams" + "nodeRole" "worker" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" .Values.clpConfig.stream_output.storage.staging_directory +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml new file mode 100644 index 0000000000..e909a57b21 --- /dev/null +++ b/tools/deployment/package-helm/templates/query-worker-staged-streams-pvc.yaml @@ -0,0 +1,9 @@ +{{- if eq .Values.clpConfig.stream_output.storage.type "s3" }} +{{- include "clp.createPvc" (dict + "root" . + "component_category" "query-worker" + "name" "staged-streams" + "capacity" "20Gi" + "accessModes" (list "ReadWriteOnce") +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml index 3e17545925..9c8ce5a7f3 100644 --- a/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml +++ b/tools/deployment/package-helm/templates/shared-data-archives-pv.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "shared-data" @@ -7,3 +8,4 @@ "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.archive_output.storage.directory ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml index 1cbc9c3f4d..279b4aa5f6 100644 --- a/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml +++ b/tools/deployment/package-helm/templates/shared-data-archives-pvc.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.clpConfig.archive_output.storage.type "fs" }} {{- include "clp.createPvc" (dict "root" . "component_category" "shared-data" @@ -5,3 +6,4 @@ "capacity" "50Gi" "accessModes" (list "ReadWriteMany") ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml index 8cc2cd8019..1a4b0a88e6 100644 --- a/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml +++ b/tools/deployment/package-helm/templates/shared-data-streams-pv.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} {{- include "clp.createLocalPv" (dict "root" . "component_category" "shared-data" @@ -7,3 +8,4 @@ "accessModes" (list "ReadWriteMany") "hostPath" .Values.clpConfig.stream_output.storage.directory ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml index 3477876d73..535be374d2 100644 --- a/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml +++ b/tools/deployment/package-helm/templates/shared-data-streams-pvc.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} {{- include "clp.createPvc" (dict "root" . "component_category" "shared-data" @@ -5,3 +6,4 @@ "capacity" "20Gi" "accessModes" (list "ReadWriteMany") ) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/webui-deployment.yaml b/tools/deployment/package-helm/templates/webui-deployment.yaml index 3567a83c33..9e7f68f60b 100644 --- a/tools/deployment/package-helm/templates/webui-deployment.yaml +++ b/tools/deployment/package-helm/templates/webui-deployment.yaml @@ -59,16 +59,19 @@ spec: value: "4000" - name: "RATE_LIMIT" value: {{ .Values.clpConfig.webui.rate_limit | quote }} + {{- with .Values.clpConfig.stream_output.storage }} + {{- if and (eq .type "s3") (eq .s3_config.aws_authentication.type "credentials") }} + - name: "AWS_ACCESS_KEY_ID" + value: {{ .s3_config.aws_authentication.credentials.access_key_id | quote }} + - name: "AWS_SECRET_ACCESS_KEY" + value: {{ .s3_config.aws_authentication.credentials.secret_access_key | quote }} + {{- end }}{{/* if and (eq .type "s3") + (eq .s3_config.aws_authentication.type "credentials") */}} + {{- end }}{{/* with .Values.clpConfig.stream_output.storage */}} ports: - name: "webui" containerPort: 4000 volumeMounts: - - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} - - name: {{ include "clp.volumeName" (dict - "component_category" "shared-data" - "name" "streams" - ) | quote }} - mountPath: "/var/data/streams" - name: "client-settings" mountPath: "/opt/clp/var/www/webui/client/settings.json" subPath: "webui-client-settings.json" @@ -77,6 +80,19 @@ spec: mountPath: "/opt/clp/var/www/webui/server/dist/settings.json" subPath: "webui-server-settings.json" readOnly: true + {{- if .Values.clpConfig.aws_config_directory }} + - {{- include "clp.awsConfigVolumeMount" . | nindent 14 }} + {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolumeMount" . | nindent 14 }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} + - name: {{ include "clp.volumeName" (dict + "component_category" "shared-data" + "name" "streams" + ) | quote }} + mountPath: "/var/data/streams" + {{- end }} command: [ "/opt/clp/bin/node-22", "/opt/clp/var/www/webui/server/dist/src/main.js" @@ -89,15 +105,22 @@ spec: {{- include "clp.livenessProbeTimings" . | nindent 12 }} tcpSocket: *webui-health-check volumes: - - {{- include "clp.logsInputVolume" . | nindent 10 }} - - {{- include "clp.pvcVolume" (dict - "root" . - "component_category" "shared-data" - "name" "streams" - ) | nindent 10 }} - name: "client-settings" configMap: name: {{ include "clp.fullname" . }}-config - name: "server-settings" configMap: name: {{ include "clp.fullname" . }}-config + {{- if .Values.clpConfig.aws_config_directory }} + - {{- include "clp.awsConfigVolume" . | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.logs_input.type "fs" }} + - {{- include "clp.logsInputVolume" . | nindent 10 }} + {{- end }} + {{- if eq .Values.clpConfig.stream_output.storage.type "fs" }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "streams" + ) | nindent 10 }} + {{- end }} diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 31c1e2ddde..474e1f1ab3 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -56,7 +56,7 @@ wait_for_pods() { kind delete cluster --name clp-test rm -rf "$CLP_HOME" mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ - "$CLP_HOME/var/data/"{archives,streams} \ + "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \ "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ "$CLP_HOME/var/log/"{api_server,garbage_collector,mcp_server} \ diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 53963bf497..382f113239 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -154,6 +154,9 @@ clpConfig: # will be raised if so. tmp_directory: "/tmp/clp/var/tmp" + # Location of the AWS tools' config files (e.g., `~/.aws`). Set to null to disable. + aws_config_directory: null + credentials: database: username: "clp-user"