From b099ec7542a2fc1ff294678c27e02d7ec53af3a9 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 7 Oct 2025 18:26:26 -0700 Subject: [PATCH 1/4] feat: Add Backup Helm Chart to Gen3-Helm --- helm/backups/.helmignore | 23 ++++++++++ helm/backups/Chart.yaml | 25 ++++++++++ helm/backups/README.md | 25 ++++++++++ helm/backups/templates/cronjob.yaml | 71 +++++++++++++++++++++++++++++ helm/backups/values.yaml | 23 ++++++++++ 5 files changed, 167 insertions(+) create mode 100644 helm/backups/.helmignore create mode 100644 helm/backups/Chart.yaml create mode 100644 helm/backups/README.md create mode 100644 helm/backups/templates/cronjob.yaml create mode 100644 helm/backups/values.yaml diff --git a/helm/backups/.helmignore b/helm/backups/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/helm/backups/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/backups/Chart.yaml b/helm/backups/Chart.yaml new file mode 100644 index 000000000..3d0ee4d02 --- /dev/null +++ b/helm/backups/Chart.yaml @@ -0,0 +1,25 @@ +apiVersion: v2 +name: backups +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.4.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +# Ref: https://github.com/calypr/backup-service/releases +appVersion: "1.13.0" diff --git a/helm/backups/README.md b/helm/backups/README.md new file mode 100644 index 000000000..5a4aff027 --- /dev/null +++ b/helm/backups/README.md @@ -0,0 +1,25 @@ +# backups + +![Version: 0.4.0](https://img.shields.io/badge/Version-0.4.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.13.0](https://img.shields.io/badge/AppVersion-1.13.0-informational?style=flat-square) + +A Helm chart for Kubernetes + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| backups.grip.graph | string | `"CALYPR"` | | +| backups.grip.host | string | `"grip-service.default.svc.cluster.local"` | | +| backups.image | string | `"quay.io/ohsu-comp-bio/backup-service:latest"` | | +| backups.postgres.host | string | `"local-postgresql.default.svc.cluster.local"` | | +| backups.postgres.secret | string | `"local-postgresql"` | | +| backups.postgres.user | string | `"postgres"` | | +| backups.s3.bucket | string | `"calypr-backups"` | | +| backups.s3.dir | string | `"calypr-dev"` | | +| backups.s3.endpoint | string | `"https://aced-storage.ohsu.edu"` | | +| backups.s3.secret.key | string | `"s3-credentials-key"` | | +| backups.s3.secret.secret | string | `"s3-credentials-secret"` | | +| backups.schedule | string | `"0 2 * * *"` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/helm/backups/templates/cronjob.yaml b/helm/backups/templates/cronjob.yaml new file mode 100644 index 000000000..36c67f888 --- /dev/null +++ b/helm/backups/templates/cronjob.yaml @@ -0,0 +1,71 @@ +# templates/cronjob.yaml (or the new CronJob.yaml location in a Helm chart) +apiVersion: batch/v1 +kind: CronJob +metadata: + name: backup-service-cronjob +spec: + # Use the schedule from values.yaml + schedule: "{{ .Values.backups.schedule }}" + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: backup-service + image: "{{ .Values.backups.image }}" + imagePullPolicy: Always + + env: + # GRIP Host + - name: GRIP_HOST + value: "{{ .Values.backups.grip.host }}" + + # GRIP Graph + - name: GRIP_GRAPH + value: "{{ .Values.backups.grip.graph }}" + + # Postgres Host + - name: PGHOST + value: "{{ .Values.backups.postgres.host }}" + + # Postgres User + - name: PGUSER + value: "{{ .Values.backups.postgres.user }}" + + # Postgres Password + # Note: this secret relies on the existing Gen3 Postgres secret + # and is not expected to be created separately. + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: "{{ .Values.backups.postgres.secret }}" + key: postgres-password + + # S3 Endpoint + - name: ENDPOINT + value: "{{ .Values.backups.s3.endpoint }}" + + # S3 Bucket + - name: BUCKET + value: "{{ .Values.backups.s3.bucket }}" + + # S3 Directory + - name: DIR + value: "{{ .Values.backups.s3.dir }}" + + # S3 Access Key + # TODO: This secret requires creation; it is not part of Gen3 by default. + - name: ACCESS_KEY + valueFrom: + secretKeyRef: + name: "{{ .Values.backups.s3.secret }}" + key: ACCESS_KEY + + # S3 Secret Key + # TODO: This secret requires creation; it is not part of Gen3 by default. + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: "{{ .Values.backups.s3.secret }}" + key: SECRET_KEY diff --git a/helm/backups/values.yaml b/helm/backups/values.yaml new file mode 100644 index 000000000..1408949a0 --- /dev/null +++ b/helm/backups/values.yaml @@ -0,0 +1,23 @@ +# values.yaml +backups: + # CronJob configuration (Daily at 2am) + schedule: "0 2 * * *" + image: "quay.io/ohsu-comp-bio/backup-service:docs-contributing.md" + + # GRIP + grip: + host: grip-service.default.svc.cluster.local + graph: CALYPR + + # Postgres + postgres: + host: local-postgresql.default.svc.cluster.local + user: postgres + secret: local-postgresql + + # S3 + s3: + endpoint: https://aced-storage.ohsu.edu + bucket: calypr-backups + dir: "calypr-dev" + secret: s3-credentials From c5bea8119f669f271ecdaffebd01e243f045b081 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 6 Jan 2026 12:52:37 -0800 Subject: [PATCH 2/4] feat: Add Funnel Chart --- helm/backups/Chart.yaml | 4 +- helm/backups/README.md | 7 +- helm/backups/templates/cronjob.yaml | 10 +- helm/backups/values.yaml | 2 +- helm/funnel/.helmignore | 23 + helm/funnel/Chart.yaml | 37 ++ helm/funnel/README.md | 182 ++++++ helm/funnel/dev.sh | 13 + helm/funnel/example_dev_values.yaml | 37 ++ helm/funnel/files/executor-job.yaml | 85 +++ helm/funnel/files/role.yaml | 39 ++ helm/funnel/files/rolebinding.yaml | 16 + helm/funnel/files/server-config.yaml | 193 +++++++ helm/funnel/files/serviceaccount.yaml | 18 + helm/funnel/files/worker-job.yaml | 104 ++++ helm/funnel/files/worker-pv.yaml | 31 ++ helm/funnel/files/worker-pvc.yaml | 17 + helm/funnel/templates/NOTES.txt | 3 + helm/funnel/templates/_helpers.tpl | 67 +++ helm/funnel/templates/clusterrole.yaml | 65 +++ helm/funnel/templates/clusterrolebinding.yaml | 19 + helm/funnel/templates/job-resources.yaml | 0 helm/funnel/templates/plugin-server.yaml | 66 +++ helm/funnel/templates/server-configmap.yaml | 15 + helm/funnel/templates/server-deployment.yaml | 96 ++++ helm/funnel/templates/service.yaml | 18 + helm/funnel/templates/serviceaccount.yaml | 22 + .../templates/tests/test-connection.yaml | 15 + helm/funnel/templates/worker-configmap.yaml | 22 + helm/funnel/values.yaml | 526 ++++++++++++++++++ 30 files changed, 1743 insertions(+), 9 deletions(-) create mode 100644 helm/funnel/.helmignore create mode 100644 helm/funnel/Chart.yaml create mode 100644 helm/funnel/README.md create mode 100644 helm/funnel/dev.sh create mode 100644 helm/funnel/example_dev_values.yaml create mode 100644 helm/funnel/files/executor-job.yaml create mode 100644 helm/funnel/files/role.yaml create mode 100644 helm/funnel/files/rolebinding.yaml create mode 100644 helm/funnel/files/server-config.yaml create mode 100644 helm/funnel/files/serviceaccount.yaml create mode 100644 helm/funnel/files/worker-job.yaml create mode 100644 helm/funnel/files/worker-pv.yaml create mode 100644 helm/funnel/files/worker-pvc.yaml create mode 100644 helm/funnel/templates/NOTES.txt create mode 100644 helm/funnel/templates/_helpers.tpl create mode 100644 helm/funnel/templates/clusterrole.yaml create mode 100644 helm/funnel/templates/clusterrolebinding.yaml create mode 100644 helm/funnel/templates/job-resources.yaml create mode 100644 helm/funnel/templates/plugin-server.yaml create mode 100644 helm/funnel/templates/server-configmap.yaml create mode 100644 helm/funnel/templates/server-deployment.yaml create mode 100644 helm/funnel/templates/service.yaml create mode 100644 helm/funnel/templates/serviceaccount.yaml create mode 100644 helm/funnel/templates/tests/test-connection.yaml create mode 100644 helm/funnel/templates/worker-configmap.yaml create mode 100644 helm/funnel/values.yaml diff --git a/helm/backups/Chart.yaml b/helm/backups/Chart.yaml index 3d0ee4d02..8b3f6c778 100644 --- a/helm/backups/Chart.yaml +++ b/helm/backups/Chart.yaml @@ -15,11 +15,11 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.1 +version: 0.5.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. # Ref: https://github.com/calypr/backup-service/releases -appVersion: "1.13.0" +appVersion: "1.15.1" diff --git a/helm/backups/README.md b/helm/backups/README.md index 5a4aff027..48972b3c6 100644 --- a/helm/backups/README.md +++ b/helm/backups/README.md @@ -1,6 +1,6 @@ # backups -![Version: 0.4.0](https://img.shields.io/badge/Version-0.4.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.13.0](https://img.shields.io/badge/AppVersion-1.13.0-informational?style=flat-square) +![Version: 0.5.0](https://img.shields.io/badge/Version-0.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.15.0](https://img.shields.io/badge/AppVersion-1.15.0-informational?style=flat-square) A Helm chart for Kubernetes @@ -10,15 +10,14 @@ A Helm chart for Kubernetes |-----|------|---------|-------------| | backups.grip.graph | string | `"CALYPR"` | | | backups.grip.host | string | `"grip-service.default.svc.cluster.local"` | | -| backups.image | string | `"quay.io/ohsu-comp-bio/backup-service:latest"` | | +| backups.image | string | `"quay.io/ohsu-comp-bio/backup-service:main"` | | | backups.postgres.host | string | `"local-postgresql.default.svc.cluster.local"` | | | backups.postgres.secret | string | `"local-postgresql"` | | | backups.postgres.user | string | `"postgres"` | | | backups.s3.bucket | string | `"calypr-backups"` | | | backups.s3.dir | string | `"calypr-dev"` | | | backups.s3.endpoint | string | `"https://aced-storage.ohsu.edu"` | | -| backups.s3.secret.key | string | `"s3-credentials-key"` | | -| backups.s3.secret.secret | string | `"s3-credentials-secret"` | | +| backups.s3.secret | string | `"s3-credentials"` | | | backups.schedule | string | `"0 2 * * *"` | | ---------------------------------------------- diff --git a/helm/backups/templates/cronjob.yaml b/helm/backups/templates/cronjob.yaml index 36c67f888..da1cf61c3 100644 --- a/helm/backups/templates/cronjob.yaml +++ b/helm/backups/templates/cronjob.yaml @@ -56,7 +56,10 @@ spec: # S3 Access Key # TODO: This secret requires creation; it is not part of Gen3 by default. - - name: ACCESS_KEY + # + # MinIO client requires the Access Key to be set to AWS_ACCESS_KEY + # Ref: https://github.com/minio/minio-py/blob/7.2.20/minio/credentials/providers.py#L251 + - name: AWS_ACCESS_KEY valueFrom: secretKeyRef: name: "{{ .Values.backups.s3.secret }}" @@ -64,7 +67,10 @@ spec: # S3 Secret Key # TODO: This secret requires creation; it is not part of Gen3 by default. - - name: SECRET_KEY + # + # MinIO client requires the Secret Key to be set to AWS_SECRET_KEY + # Ref: https://github.com/minio/minio-py/blob/7.2.20/minio/credentials/providers.py#L258 + - name: AWS_SECRET_KEY valueFrom: secretKeyRef: name: "{{ .Values.backups.s3.secret }}" diff --git a/helm/backups/values.yaml b/helm/backups/values.yaml index 1408949a0..4fda85499 100644 --- a/helm/backups/values.yaml +++ b/helm/backups/values.yaml @@ -2,7 +2,7 @@ backups: # CronJob configuration (Daily at 2am) schedule: "0 2 * * *" - image: "quay.io/ohsu-comp-bio/backup-service:docs-contributing.md" + image: "quay.io/ohsu-comp-bio/backup-service:main" # GRIP grip: diff --git a/helm/funnel/.helmignore b/helm/funnel/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/helm/funnel/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/funnel/Chart.yaml b/helm/funnel/Chart.yaml new file mode 100644 index 000000000..5f391d054 --- /dev/null +++ b/helm/funnel/Chart.yaml @@ -0,0 +1,37 @@ +apiVersion: v2 +name: funnel +description: A toolkit for distributed task execution ⚙️ +keywords: + - funnel + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.76 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: 2025-12-22 + +dependencies: + - name: postgresql + version: 18.1.15 + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled + + - name: mongodb + version: 13.9.4 + repository: https://charts.bitnami.com/bitnami + condition: mongodb.enabled diff --git a/helm/funnel/README.md b/helm/funnel/README.md new file mode 100644 index 000000000..e37281279 --- /dev/null +++ b/helm/funnel/README.md @@ -0,0 +1,182 @@ +# funnel + +![Version: 0.1.75](https://img.shields.io/badge/Version-0.1.75-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2025-12-22](https://img.shields.io/badge/AppVersion-2025--12--22-informational?style=flat-square) + +A toolkit for distributed task execution ⚙️ + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | mongodb | 13.9.4 | +| https://charts.bitnami.com/bitnami | postgresql | 18.1.15 | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| AWSBatch.DisableReconciler | bool | `true` | | +| AWSBatch.JobDefinition | string | `"funnel-job-def"` | | +| AWSBatch.JobQueue | string | `"funnel-job-queue"` | | +| AWSBatch.Key | string | `""` | | +| AWSBatch.ReconcileRate | string | `"10s"` | | +| AWSBatch.Region | string | `""` | | +| AWSBatch.Secret | string | `""` | | +| AmazonS3.AWSConfig.Key | string | `""` | | +| AmazonS3.AWSConfig.MaxRetries | int | `10` | | +| AmazonS3.AWSConfig.Secret | string | `""` | | +| AmazonS3.Disabled | bool | `false` | | +| AmazonS3.SSE.CustomerKeyFile | string | `""` | | +| AmazonS3.SSE.KMSKey | string | `""` | | +| BoltDB.Path | string | `"./funnel-work-dir/funnel.db"` | | +| Compute | string | `"kubernetes"` | | +| Database | string | `"postgres"` | | +| Datastore.CredentialsFile | string | `""` | | +| Datastore.Project | string | `""` | | +| DynamoDB.AWSConfig.Key | string | `""` | | +| DynamoDB.AWSConfig.Region | string | `""` | | +| DynamoDB.AWSConfig.Secret | string | `""` | | +| DynamoDB.TableBasename | string | `"funnel"` | | +| Elastic.IndexPrefix | string | `"funnel"` | | +| Elastic.URL | string | `"http://localhost:9200"` | | +| EventWriters[0] | string | `"postgres"` | | +| EventWriters[1] | string | `"log"` | | +| FTPStorage.Disabled | bool | `false` | | +| FTPStorage.Password | string | `"anonymous"` | | +| FTPStorage.Timeout | string | `"10s"` | | +| FTPStorage.User | string | `"anonymous"` | | +| GoogleStorage.CredentialsFile | string | `""` | | +| GoogleStorage.Disabled | bool | `false` | | +| GridEngine.Template | string | `"#!bin/bash\n#$ -N {{.TaskId}}\n#$ -o {{.WorkDir}}/funnel-stdout\n#$ -e {{.WorkDir}}/funnel-stderr\n#$ -l nodes=1\n{{if ne .Cpus 0 -}}\n{{printf \"#$ -pe mpi %d\" .Cpus}}\n{{- end}}\n{{if ne .RamGb 0.0 -}}\n{{printf \"#$ -l h_vmem=%.0fG\" .RamGb}}\n{{- end}}\n{{if ne .DiskGb 0.0 -}}\n{{printf \"#$ -l h_fsize=%.0fG\" .DiskGb}}\n{{- end}}\n\n{{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}}\n"` | | +| GridEngine.TemplateFile | string | `""` | | +| HTCondor.DisableReconciler | bool | `true` | | +| HTCondor.ReconcileRate | string | `"10s"` | | +| HTCondor.Template | string | `"universe = vanilla\ngetenv = True\nexecutable = {{.Executable}}\narguments = worker run --config {{.Config}} --task-id {{.TaskId}}\nlog = {{.WorkDir}}/condor-event-log\nerror = {{.WorkDir}}/funnel-stderr\noutput = {{.WorkDir}}/funnel-stdout\nshould_transfer_files = YES\nwhen_to_transfer_output = ON_EXIT_OR_EVICT\n{{if ne .Cpus 0 -}}\n{{printf \"request_cpus = %d\" .Cpus}}\n{{- end}}\n{{if ne .RamGb 0.0 -}}\n{{printf \"request_memory = %.0f GB\" .RamGb}}\n{{- end}}\n{{if ne .DiskGb 0.0 -}}\n{{printf \"request_disk = %.0f GB\" .DiskGb}}\n{{- end}}\n\nqueue\n"` | | +| HTCondor.TemplateFile | string | `""` | | +| HTTPStorage.Timeout | string | `"30s"` | | +| Kafka.Topic | string | `"funnel"` | | +| Kubernetes.DisableJobCleanup | bool | `false` | | +| Kubernetes.DisableReconciler | bool | `false` | | +| Kubernetes.Executor | string | `"kubernetes"` | | +| Kubernetes.ExecutorTemplate | string | `""` | | +| Kubernetes.JobsNamespace | string | `""` | | +| Kubernetes.Namespace | string | `""` | | +| Kubernetes.NodeSelector | object | `{}` | | +| Kubernetes.PVCTemplate | string | `""` | | +| Kubernetes.PVTemplate | string | `""` | | +| Kubernetes.ReconcileRate | string | `"10s"` | | +| Kubernetes.ServiceAccount | string | `""` | | +| Kubernetes.Tolerations | list | `[]` | | +| Kubernetes.WorkerTemplate | string | `""` | | +| LocalStorage.AllowedDirs[0] | string | `"./"` | | +| Logger.level | string | `"debug"` | | +| Logger.outputFile | string | `""` | | +| MongoDB.Addrs | list | `[]` | | +| MongoDB.Database | string | `"funnel"` | | +| MongoDB.Timeout.duration | string | `"300s"` | | +| MongoDB.Username | string | `"example"` | | +| Node.ID | string | `""` | | +| Node.Resources.Cpus | int | `0` | | +| Node.Resources.DiskGb | float | `0` | | +| Node.Resources.RamGb | float | `0` | | +| Node.Timeout.disabled | bool | `true` | | +| Node.UpdateRate | string | `"5s"` | | +| PBS.DisableReconciler | bool | `true` | | +| PBS.ReconcileRate | string | `"10s"` | | +| PBS.Template | string | `"#!bin/bash\n#PBS -N {{.TaskId}}\n#PBS -o {{.WorkDir}}/funnel-stdout\n#PBS -e {{.WorkDir}}/funnel-stderr\n{{if ne .Cpus 0 -}}\n{{printf \"#PBS -l nodes=1:ppn=%d\" .Cpus}}\n{{- end}}\n{{if ne .RamGb 0.0 -}}\n{{printf \"#PBS -l mem=%.0fgb\" .RamGb}}\n{{- end}}\n{{if ne .DiskGb 0.0 -}}\n{{printf \"#PBS -l file=%.0fgb\" .DiskGb}}\n{{- end}}\n\n{{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}}\n"` | | +| PBS.TemplateFile | string | `""` | | +| Postgres.AdminPassword | string | `"example"` | | +| Postgres.AdminUser | string | `"postgres"` | | +| Postgres.Database | string | `"funnel"` | | +| Postgres.Host | string | `"funnel-postgresql.default.svc.cluster.local"` | | +| Postgres.Password | string | `"example"` | | +| Postgres.Timeout.duration | string | `"300s"` | | +| Postgres.User | string | `"funnel"` | | +| RPCClient.MaxRetries | int | `10` | | +| RPCClient.ServerAddress | string | `"localhost:9090"` | | +| RPCClient.Timeout.duration | string | `"60s"` | | +| Scheduler.NodeInitTimeout.duration | string | `"300s"` | | +| Scheduler.NodePingTimeout.duration | string | `"60s"` | | +| Scheduler.ScheduleChunk | int | `10` | | +| Scheduler.ScheduleRate | string | `"1s"` | | +| Server.DisableHTTPCache | bool | `true` | | +| Server.HTTPPort | string | `"8000"` | | +| Server.HostName | string | `"funnel"` | | +| Server.RPCPort | string | `"9090"` | | +| Slurm.DisableReconciler | bool | `true` | | +| Slurm.ReconcileRate | string | `"10s"` | | +| Slurm.Template | string | `"#!/bin/bash\n#SBATCH --job-name {{.TaskId}}\n#SBATCH --ntasks 1\n#SBATCH --error {{.WorkDir}}/funnel-stderr\n#SBATCH --output {{.WorkDir}}/funnel-stdout\n{{if ne .Cpus 0 -}}\n{{printf \"#SBATCH --cpus-per-task %d\" .Cpus}}\n{{- end}}\n{{if ne .RamGb 0.0 -}}\n{{printf \"#SBATCH --mem %.0fGB\" .RamGb}}\n{{- end}}\n{{if ne .DiskGb 0.0 -}}\n{{printf \"#SBATCH --tmp %.0fGB\" .DiskGb}}\n{{- end}}\n\n{{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}}\n"` | | +| Slurm.TemplateFile | string | `""` | | +| Swift.AuthURL | string | `""` | | +| Swift.ChunkSizeBytes | int | `500000000` | | +| Swift.Disabled | bool | `false` | | +| Swift.Password | string | `""` | | +| Swift.RegionName | string | `""` | | +| Swift.TenantID | string | `""` | | +| Swift.TenantName | string | `""` | | +| Swift.UserName | string | `""` | | +| Worker.LeaveWorkDir | bool | `false` | | +| Worker.LogTailSize | int | `10000` | | +| Worker.LogUpdateRate | string | `"5s"` | | +| Worker.MaxParallelTransfers | int | `10` | | +| Worker.PollingRate | string | `"5s"` | | +| Worker.WorkDir | string | `"./funnel-work-dir"` | | +| backoffLimit | int | `1` | | +| completions | int | `1` | | +| image.initContainers[0].command[0] | string | `"cp"` | | +| image.initContainers[0].command[1] | string | `"/app/build/plugins/authorizer"` | | +| image.initContainers[0].command[2] | string | `"/opt/funnel/plugin-binaries/auth-plugin"` | | +| image.initContainers[0].image | string | `"quay.io/ohsu-comp-bio/funnel-plugins"` | | +| image.initContainers[0].name | string | `"plugins"` | | +| image.initContainers[0].pullPolicy | string | `"Always"` | | +| image.initContainers[0].tag | string | `"pr-1"` | | +| image.initContainers[0].volumeMounts[0].mountPath | string | `"/opt/funnel/plugin-binaries"` | | +| image.initContainers[0].volumeMounts[0].name | string | `"plugin-volume"` | | +| image.pullPolicy | string | `"Always"` | | +| image.repository | string | `"quay.io/ohsu-comp-bio/funnel"` | | +| labels.app | string | `"funnel"` | | +| mongodb.app | string | `"funnel-mongodb"` | | +| mongodb.architecture | string | `"standalone"` | | +| mongodb.auth.enabled | bool | `true` | | +| mongodb.auth.rootPassword | string | `"example"` | | +| mongodb.auth.rootUser | string | `"example"` | | +| mongodb.commonLabels.app | string | `"funnel-mongodb"` | | +| mongodb.enabled | bool | `false` | | +| mongodb.image.registry | string | `"public.ecr.aws"` | | +| mongodb.persistence.enabled | bool | `false` | | +| mongodb.persistence.size | string | `"1Gi"` | | +| postgresql.enabled | bool | `true` | | +| postgresql.global.postgresql.auth.database | string | `"funnel"` | | +| postgresql.global.postgresql.auth.password | string | `"example"` | | +| postgresql.global.postgresql.auth.postgresPassword | string | `"example"` | | +| postgresql.global.postgresql.auth.username | string | `"funnel"` | | +| postgresql.primary.persistence.enabled | bool | `false` | | +| rbac.create | bool | `true` | | +| replicaCount | int | `1` | | +| resources.limits.cpu | string | `"1000m"` | | +| resources.limits.ephemeral_storage | string | `"2048Mi"` | | +| resources.limits.memory | string | `"2048Mi"` | | +| resources.requests.cpu | string | `"100m"` | | +| resources.requests.ephemeral_storage | string | `"512Mi"` | | +| resources.requests.memory | string | `"512Mi"` | | +| service.httpPort | int | `8000` | | +| service.rpcPort | int | `9090` | | +| service.type | string | `"ClusterIP"` | | +| storage.accessMode | string | `"ReadWriteMany"` | | +| storage.className | string | `"s3-csi-sc"` | | +| storage.createStorageClass | bool | `true` | | +| storage.driver | string | `"aws-s3"` | | +| storage.provisioner | string | `"s3.csi.aws.com"` | | +| storage.size | string | `"10Mi"` | | +| volumeMounts[0].mountPath | string | `"/etc/config/funnel-server.yaml"` | | +| volumeMounts[0].name | string | `"funnel-server-config-volume"` | | +| volumeMounts[0].subPath | string | `"funnel-server.yaml"` | | +| volumeMounts[1].mountPath | string | `"/opt/funnel/plugin-binaries"` | | +| volumeMounts[1].name | string | `"plugin-volume"` | | +| volumes[0].configMap.name | string | `"funnel-server-config"` | | +| volumes[0].name | string | `"funnel-server-config-volume"` | | +| volumes[1].emptyDir | object | `{}` | | +| volumes[1].name | string | `"plugin-volume"` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/helm/funnel/dev.sh b/helm/funnel/dev.sh new file mode 100644 index 000000000..0b578dbfb --- /dev/null +++ b/helm/funnel/dev.sh @@ -0,0 +1,13 @@ +apk add git curl vim wget +wget https://go.dev/dl/go1.24.2.linux-arm64.tar.gz +tar -C /usr/local -xzf go1.24.2.linux-arm64.tar.gz +echo 'PATH=$PATH:/usr/local/go/bin:/root/go/bin' >> ~/.bashrc +source ~/.bashrc +git clone https://github.com/ohsu-comp-bio/funnel.git +git clone https://github.com/ohsu-comp-bio/funnel-plugins.git +rm -f /app/funnel +cd funnel && git checkout feature/plugins && go build . && go install . + + + + diff --git a/helm/funnel/example_dev_values.yaml b/helm/funnel/example_dev_values.yaml new file mode 100644 index 000000000..03bfa7550 --- /dev/null +++ b/helm/funnel/example_dev_values.yaml @@ -0,0 +1,37 @@ +image: + repository: quay.io/ohsu-comp-bio/funnel + tag: feature-plugins-cleanup + pullPolicy: Always + # Plugin Init Container + initContainer: + image: quay.io/ohsu-comp-bio/funnel-plugins + tag: feature-auth-proto-config + pullPolicy: Always + command: + - cp + - /app/build/plugins/authorizer + - /opt/funnel/plugin-binaries/auth-plugin + +Logger: + Level: debug + +GenericS3: + - Disabled: false + Endpoint: "s3.us-west-2.amazonaws.com" + Key: "you-bucket-key-goes-here" + Secret: "your-bucket-secret-goes-here" + Bucket: funnel-testing + Region: us-west-2 + +AmazonS3: + Disabled: true + +Plugins: + Path: plugin-binaries/auth-plugin + Params: + Host: http://funnel-plugin-test-server:8080/token?user= + +Kubernetes: + JobsNamespace: "funnel" + Namespace: "funnel" + ReconcileRate: 6s \ No newline at end of file diff --git a/helm/funnel/files/executor-job.yaml b/helm/funnel/files/executor-job.yaml new file mode 100644 index 000000000..ff57e5a0e --- /dev/null +++ b/helm/funnel/files/executor-job.yaml @@ -0,0 +1,85 @@ +# Task Executor +apiVersion: batch/v1 +kind: Job +metadata: + name: {{`{{.TaskId}}`}}-{{`{{.JobId}}`}} + namespace: {{`{{.JobsNamespace}}`}} + labels: + app: funnel-executor + job-name: {{`{{.TaskId}}-{{.JobId}}`}} +spec: + backoffLimit: {{ .Values.backoffLimit}} + completions: {{ .Values.completions }} + template: + spec: + # NodeSelectors + # https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes/#create-a-pod-that-gets-scheduled-to-your-chosen-node + {{` + {{- if .NodeSelector }} + nodeSelector: + {{- range $key, $value := .NodeSelector }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} + `}} + + # Tolerations + # https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + {{` + {{- if .Tolerations }} + tolerations: + {{- range .Tolerations }} + - key: {{ .Key }} + operator: {{ .Operator }} + effect: {{ .Effect }} + {{if .Value}}value: {{ .Value }}{{end}} + {{if .TolerationSeconds}}tolerationSeconds: {{ .TolerationSeconds }}{{end}} + {{- end }} + {{- end }} + `}} + + # SecurityContext + # https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + {{` + {{- if .securityContext }} + securityContext: + fsGroup: {{ .securityContext.fsGroup }} + runAsUser: {{ .securityContext.runAsUser }} + runAsGroup: {{ .securityContext.runAsGroup }} + supplementalGroups: {{ .securityContext.supplementalGroups }} + {{- end }} + `}} + + restartPolicy: OnFailure + serviceAccountName: {{`{{.ServiceAccountName}}`}} + containers: + - name: funnel-worker-{{`{{.TaskId}}`}} + image: {{`{{.Image}}`}} + imagePullPolicy: Always + command: {{`{{.Command}}`}} + workingDir: {{`{{.Workdir}}`}} + resources: + requests: + cpu: {{ .Values.resources.requests.cpu }} + memory: {{ .Values.resources.requests.memory }} + ephemeral-storage: {{ .Values.resources.requests.ephemeral_storage }} + + volumeMounts: + {{` + {{- if .NeedsPVC }} + {{range $idx, $item := .Volumes}} + - name: funnel-storage-{{$.TaskId}} + mountPath: {{$item.ContainerPath}} + subPath: {{$.TaskId}}{{$item.ContainerPath}} + {{end}} + {{- end }} + `}} + + volumes: + {{` + {{- if .NeedsPVC }} + - name: funnel-storage-{{.TaskId}} + persistentVolumeClaim: + claimName: funnel-worker-pvc-{{.TaskId}} + {{- end }} + `}} \ No newline at end of file diff --git a/helm/funnel/files/role.yaml b/helm/funnel/files/role.yaml new file mode 100644 index 000000000..ef27a85ef --- /dev/null +++ b/helm/funnel/files/role.yaml @@ -0,0 +1,39 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: funnel-worker-sa-{{`{{.Namespace}}`}}-{{`{{.TaskId}}`}}-role + namespace: {{`{{.Namespace}}`}} + labels: + app: funnel + taskId: {{`{{.TaskId}}`}} +rules: +# Job management permissions +- apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] +- apiGroups: ["batch"] + resources: ["jobs/status"] + verbs: ["get", "update", "patch"] + +# Pod management permissions (jobs create pods) +- apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "delete"] +- apiGroups: [""] + resources: ["pods/status", "pods/log"] + verbs: ["get", "list", "watch"] + +# ConfigMap and Secret access (if needed for job configuration) +- apiGroups: [""] + resources: ["configmaps", "secrets"] + verbs: ["get", "list"] + +# PVC management (if using persistent volumes) +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create", "get", "list", "watch", "delete"] + +# Events (for job status reporting) +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] diff --git a/helm/funnel/files/rolebinding.yaml b/helm/funnel/files/rolebinding.yaml new file mode 100644 index 000000000..fa287a897 --- /dev/null +++ b/helm/funnel/files/rolebinding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: funnel-worker-sa-{{`{{.Namespace}}`}}-{{`{{.TaskId}}`}}-binding + namespace: {{`{{.Namespace}}`}} + labels: + app: funnel + taskId: {{`{{.TaskId}}`}} +subjects: +- kind: ServiceAccount + name: {{`{{.ServiceAccountName}}`}} + namespace: {{`{{.Namespace}}`}} +roleRef: + kind: Role + name: funnel-worker-sa-{{`{{.Namespace}}`}}-{{`{{.TaskId}}`}}-role + apiGroup: rbac.authorization.k8s.io diff --git a/helm/funnel/files/server-config.yaml b/helm/funnel/files/server-config.yaml new file mode 100644 index 000000000..db30ba2f5 --- /dev/null +++ b/helm/funnel/files/server-config.yaml @@ -0,0 +1,193 @@ +Compute: {{ .Values.Compute }} + +Kubernetes: + Executor: {{ .Values.Kubernetes.Executor }} + DisableReconciler: {{ .Values.Kubernetes.DisableReconciler }} + DisableJobCleanup: {{ .Values.Kubernetes.DisableJobCleanup }} + ReconcileRate: {{ .Values.Kubernetes.ReconcileRate }} + Namespace: {{ .Release.Namespace }} + JobsNamespace: {{ .Values.Kubernetes.JobsNamespace }} + ServiceAccount: {{ .Values.Kubernetes.ServiceAccount }} + + NodeSelector: + {{- if .Values.Kubernetes.NodeSelector }} + {{- range $key, $value := .Values.Kubernetes.NodeSelector }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} + + Tolerations: + {{- if .Values.Kubernetes.Tolerations }} + {{- range .Values.Kubernetes.Tolerations }} + - Key: {{ .Key }} + Operator: {{ .Operator }} + {{- if .Value }} + Value: {{ .Value }} + {{- end }} + Effect: {{ .Effect }} + {{- if .TolerationSeconds }} + TolerationSeconds: {{ .TolerationSeconds }} + {{- end }} + {{- end }} + {{- end }} + + # Worker Job + WorkerTemplate: | +{{ tpl (.Files.Get "files/worker-job.yaml") . | indent 4 }} + + # Executor Job + ExecutorTemplate: | +{{ tpl (.Files.Get "files/executor-job.yaml") . | indent 4 }} + + # PV template + PVTemplate: | +{{ tpl (.Files.Get "files/worker-pv.yaml") . | indent 4 }} + + # PVC template + PVCTemplate: | +{{ tpl (.Files.Get "files/worker-pvc.yaml") . | indent 4 }} + + # ServiceAccount template + ServiceAccountTemplate: | +{{ tpl (.Files.Get "files/serviceaccount.yaml") . | indent 4 }} + + RoleTemplate: | +{{ tpl (.Files.Get "files/role.yaml") . | indent 4 }} + + RoleBindingTemplate: | +{{ tpl (.Files.Get "files/rolebinding.yaml") . | indent 4 }} + +Database: {{ .Values.Database }} + +EventWriters: + {{- range .Values.EventWriters }} + - {{ . }} + {{- end }} + +Logger: + level: {{ .Values.Logger.level }} + outputFile: {{ .Values.Logger.outputFile }} + +Server: + HostName: "{{ .Values.Server.HostName }}" + HTTPPort: "{{ .Values.Server.HTTPPort }}" + RPCPort: "{{ .Values.Server.RPCPort }}" + DisableHTTPCache: {{ .Values.Server.DisableHttpCache }} + +RPCClient: + ServerAddress: {{ .Values.RPCClient.ServerAddress }} + Timeout: + duration: {{ .Values.RPCClient.Timeout.duration }} + MaxRetries: {{ .Values.RPCClient.MaxRetries }} + +Scheduler: + ScheduleRate: {{ .Values.Scheduler.ScheduleRate }} + ScheduleChunk: {{ .Values.Scheduler.ScheduleChunk }} + NodePingTimeout: + duration: {{ .Values.Scheduler.NodePingTimeout.duration }} + NodeInitTimeout: + duration: {{ .Values.Scheduler.NodeInitTimeout.duration }} + +Node: + ID: {{ .Values.Node.Id }} + Timeout: + disabled: {{ .Values.Node.Timeout.disabled }} + UpdateRate: {{ .Values.Node.UpdateRate }} + Resources: + Cpus: {{ .Values.Node.Resources.cpus }} + RamGb: {{ .Values.Node.Resources.ramGb }} + DiskGb: {{ .Values.Node.Resources.diskGb }} + +Worker: + WorkDir: {{ .Values.Worker.WorkDir }} + PollingRate: {{ .Values.Worker.PollingRate }} + LogUpdateRate: {{ .Values.Worker.LogUpdateRate }} + LogTailSize: {{ .Values.Worker.LogTailSize }} + LeaveWorkDir: {{ .Values.Worker.LeaveWorkDir }} + MaxParallelTransfers: {{ .Values.Worker.MaxParallelTransfers }} + +BoltDB: + Path: {{ .Values.BoltDB.Path }} + +AmazonS3: + Disabled: {{ .Values.AmazonS3.Disabled }} + AWSConfig: + MaxRetries: {{ .Values.AmazonS3.AWSConfig.MaxRetries }} + Key: {{ .Values.AmazonS3.AWSConfig.Key }} + Secret: {{ .Values.AmazonS3.AWSConfig.Secret }} + SSE: + CustomerKeyFile: {{ .Values.AmazonS3.SSE.CustomerKeyFile }} + KMSKey: {{ .Values.AmazonS3.SSE.KMSKey }} + +DynamoDB: + TableBasename: {{ .Values.DynamoDB.TableBasename }} + AWSConfig: + Region: {{ .Values.DynamoDB.AWSConfig.Region }} + Key: {{ .Values.DynamoDB.AWSConfig.Key }} + Secret: {{ .Values.DynamoDB.AWSConfig.Secret }} + +Elastic: + IndexPrefix: {{ .Values.Elastic.IndexPrefix }} + URL: {{ .Values.Elastic.Url }} + +Datastore: + Project: {{ .Values.Datastore.Project }} + CredentialsFile: {{ .Values.Datastore.CredentialsFile }} + +MongoDB: + Addrs: + {{- if .Values.MongoDB.Addrs }} + {{- range .Values.MongoDB.Addrs }} + - {{ . }} + {{- end }} + {{- else }} + - {{ .Release.Name }}-mongodb.{{ .Release.Namespace }}.svc.cluster.local + {{- end }} + Database: {{ .Values.MongoDB.Database }} + Timeout: + duration: {{ .Values.MongoDB.Timeout.duration }} + Username: {{ .Values.MongoDB.Username }} + Password: {{ .Values.MongoDB.Password }} + +Postgres: + Host: {{ .Values.Postgres.Host }} + Database: {{ .Values.Postgres.Database }} + User: {{ .Values.Postgres.User }} + Password: {{ .Values.Postgres.Password }} + AdminUser: {{ .Values.Postgres.AdminUser }} + AdminPassword: {{ .Values.Postgres.AdminPassword }} + Timeout: + duration: {{ .Values.Postgres.Timeout.duration }} + +Kafka: + Servers: + {{- if .Values.Kafka.Servers }} + {{- range .Values.Kafka.Servers }} + - {{ . }} + {{- end }} + {{- else }} + - "" + {{- end }} + Topic: {{ .Values.Kafka.Topic }} + +GenericS3: + {{- range .Values.GenericS3 }} + - Disabled: {{ .Disabled }} + Endpoint: {{ .Endpoint }} + Key: {{ .Key }} + Secret: {{ .Secret }} + Bucket: {{ .Bucket }} + Region: {{ .Region }} + KmsKeyID: {{ .KmsKeyID }} + {{- end }} + +{{- if .Values.Plugins}} +Plugins: + Path: {{ .Values.Plugins.Path }} + Params: + {{- if .Values.Plugins.Params }} + {{- range $key, $value := .Values.Plugins.Params }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} +{{- end}} diff --git a/helm/funnel/files/serviceaccount.yaml b/helm/funnel/files/serviceaccount.yaml new file mode 100644 index 000000000..2eaf90cc7 --- /dev/null +++ b/helm/funnel/files/serviceaccount.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + # Note: Only creating ServiceAccount here, not required to create new Role or RoleBinding + # TODO: Set defaults for case no custom values are being supplied from the TES Task Tags + name: {{`{{.ServiceAccountName}}`}} + namespace: {{`{{.Namespace}}`}} + labels: + app: funnel + taskId: {{`{{.TaskId}}`}} + # Only set if provided with Role ARN + {{`{{- if .IamRoleArn}}`}} + annotations: + eks.amazonaws.com/role-arn: {{`{{.IamRoleArn}}`}} + {{`{{- end}}`}} +spec: + # TODO: Should tokens be automatically mounted into pods? + automountServiceAccountToken: false diff --git a/helm/funnel/files/worker-job.yaml b/helm/funnel/files/worker-job.yaml new file mode 100644 index 000000000..f8c51c2e7 --- /dev/null +++ b/helm/funnel/files/worker-job.yaml @@ -0,0 +1,104 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{`{{.TaskId}}`}} + namespace: {{`{{.JobsNamespace}}`}} + labels: + app: funnel-worker + task-id: {{`{{.TaskId}}`}} + # Custom Labels support + {{` + {{- range $key, $value := .ExtraLabels }} + {{ $key }}: {{ $value }} + {{- end }} + {{- range $key, $value := .NodeSelector }} + {{ $key }}: {{ $value }} + {{- end }} + `}} +spec: + backoffLimit: {{ .Values.backoffLimit}} + completions: {{ .Values.completions }} + template: + metadata: + labels: + app: funnel-worker + task-id: {{`{{.TaskId}}`}} + spec: + # NodeSelectors + # https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes/#create-a-pod-that-gets-scheduled-to-your-chosen-node + {{` + {{- if .NodeSelector }} + nodeSelector: + {{- range $key, $value := .NodeSelector }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} + `}} + + # Tolerations + # https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + {{` + {{- if .Tolerations }} + tolerations: + {{- range .Tolerations }} + - key: {{ .Key }} + operator: {{ .Operator }} + effect: {{ .Effect }} + {{if .Value}}value: {{ .Value }}{{end}} + {{if .TolerationSeconds}}tolerationSeconds: {{ .TolerationSeconds }}{{end}} + {{- end }} + {{- end }} + `}} + + # SecurityContext + # https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + {{` + {{- if .securityContext }} + securityContext: + fsGroup: {{ .securityContext.fsGroup }} + runAsUser: {{ .securityContext.runAsUser }} + runAsGroup: {{ .securityContext.runAsGroup }} + supplementalGroups: {{ .securityContext.supplementalGroups }} + {{- end }} + `}} + + serviceAccountName: {{`{{.ServiceAccountName}}`}} + restartPolicy: OnFailure + containers: + - name: funnel-worker-{{`{{.TaskId}}`}} + # TODO: Make the image + tag configurable + image: {{`{{.Image}}`}} + imagePullPolicy: Always + args: + - "worker" + - "run" + - "--config" + - "/etc/config/funnel-worker.yaml" + - "--taskID" + - {{`{{.TaskId}}`}} + resources: + requests: + cpu: {{ .Values.resources.requests.cpu }} + memory: {{ .Values.resources.requests.memory }} + ephemeral-storage: {{ .Values.resources.requests.ephemeral_storage }} + volumeMounts: + - name: config-volume + mountPath: /etc/config + {{` + {{- if .NeedsPVC }} + - name: funnel-storage-{{.TaskId}} + mountPath: /opt/funnel/funnel-work-dir/{{.TaskId}} + subPath: {{.TaskId}} + {{- end }} + `}} + volumes: + - name: config-volume + configMap: + name: funnel-worker-config-{{`{{.TaskId}}`}} + {{` + {{- if .NeedsPVC }} + - name: funnel-storage-{{.TaskId}} + persistentVolumeClaim: + claimName: funnel-worker-pvc-{{.TaskId}} + {{- end }} + `}} diff --git a/helm/funnel/files/worker-pv.yaml b/helm/funnel/files/worker-pv.yaml new file mode 100644 index 000000000..e0fba3a60 --- /dev/null +++ b/helm/funnel/files/worker-pv.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: funnel-worker-pv-{{`{{.TaskId}}`}} + labels: + app: funnel + taskId: {{`{{.TaskId}}`}} +spec: + storageClassName: "" # Required for static provisioning + capacity: + storage: "10Mi" + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + mountOptions: + - allow-delete + - allow-overwrite + - region={{`{{.Region}}`}} + - file-mode=0755 + {{`{{- if .KmsKeyID}}`}} + - sse aws:kms + - sse-kms-key-id={{`{{.KmsKeyID}}`}} + {{`{{- end}}`}} + csi: + driver: s3.csi.aws.com + volumeHandle: s3-csi-{{`{{.TaskId}}`}} + volumeAttributes: + bucketName: {{`{{.Bucket}}`}} + claimRef: + namespace: {{`{{.Namespace}}`}} + name: funnel-worker-pvc-{{`{{.TaskId}}`}} diff --git a/helm/funnel/files/worker-pvc.yaml b/helm/funnel/files/worker-pvc.yaml new file mode 100644 index 000000000..d8fccb1b4 --- /dev/null +++ b/helm/funnel/files/worker-pvc.yaml @@ -0,0 +1,17 @@ +# Worker/Executor PVC +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: funnel-worker-pvc-{{`{{.TaskId}}`}} + namespace: {{`{{.Namespace}}`}} + labels: + app: funnel + taskId: {{`{{.TaskId}}`}} +spec: + storageClassName: "" # Required for static provisioning + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Mi + volumeName: funnel-worker-pv-{{`{{.TaskId}}`}} diff --git a/helm/funnel/templates/NOTES.txt b/helm/funnel/templates/NOTES.txt new file mode 100644 index 000000000..34e155afb --- /dev/null +++ b/helm/funnel/templates/NOTES.txt @@ -0,0 +1,3 @@ +To access the Funnel service locally, use: + kubectl --namespace {{ .Release.Namespace }} port-forward svc/{{ include "funnel.fullname" . }} 8000:8000 + echo "Visit http://127.0.0.1:8000" diff --git a/helm/funnel/templates/_helpers.tpl b/helm/funnel/templates/_helpers.tpl new file mode 100644 index 000000000..a16e840ba --- /dev/null +++ b/helm/funnel/templates/_helpers.tpl @@ -0,0 +1,67 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "funnel.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "funnel.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "funnel.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "funnel.labels" -}} +helm.sh/chart: {{ include "funnel.chart" . }} +{{ include "funnel.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.labels }} +{{- range $key, $value := . }} +{{ $key }}: {{ $value | quote }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "funnel.selectorLabels" -}} +app.kubernetes.io/name: {{ include "funnel.name" . }} +app.kubernetes.io/instance: funnel +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "funnel.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "funnel.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/helm/funnel/templates/clusterrole.yaml b/helm/funnel/templates/clusterrole.yaml new file mode 100644 index 000000000..4669eca2b --- /dev/null +++ b/helm/funnel/templates/clusterrole.yaml @@ -0,0 +1,65 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: funnel-cluster-role-{{ .Release.Namespace }} +rules: + - apiGroups: [""] + resources: + - "configmaps" + - "pods" + - "pods/log" + - "persistentvolumes" + - "persistentvolumeclaims" + - "secrets" + - "serviceaccounts" + - "events" + - "pods/status" + verbs: + - "get" + - "list" + - "watch" + - "create" + - "update" + - "patch" + - "delete" + + - apiGroups: ["batch", "extensions"] + resources: + - "jobs" + - "jobs/status" # Previously corrected + verbs: + - "get" + - "list" + - "watch" + - "create" + - "update" + - "patch" + - "delete" + + - apiGroups: ["extensions", "apps"] + resources: + - "deployments" + verbs: + - "get" + - "list" + - "watch" + - "create" + - "update" + - "patch" + - "delete" + + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +{{- end }} diff --git a/helm/funnel/templates/clusterrolebinding.yaml b/helm/funnel/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..2fd4a87e4 --- /dev/null +++ b/helm/funnel/templates/clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: funnel-cluster-rolebinding-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: funnel-sa-{{ .Release.Namespace }} + namespace: {{ .Release.Namespace }} + {{- if and .Values.Kubernetes.JobsNamespace (ne .Values.Kubernetes.JobsNamespace .Values.Kubernetes.Namespace) }} + - kind: ServiceAccount + name: funnel-sa-{{ .Release.Namespace }} + namespace: {{ .Values.Kubernetes.JobsNamespace }} + {{- end }} +roleRef: + kind: ClusterRole + name: funnel-cluster-role-{{ .Release.Namespace }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/helm/funnel/templates/job-resources.yaml b/helm/funnel/templates/job-resources.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/helm/funnel/templates/plugin-server.yaml b/helm/funnel/templates/plugin-server.yaml new file mode 100644 index 000000000..747772255 --- /dev/null +++ b/helm/funnel/templates/plugin-server.yaml @@ -0,0 +1,66 @@ +{{ if .Values.deployTestServer }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: funnel-plugin-test-server + namespace: {{ .Release.Namespace }} + labels: + app: funnel-plugin-test-server +spec: + replicas: 1 + selector: + matchLabels: + app: funnel-plugin-test-server + template: + metadata: + labels: + app: funnel-plugin-test-server + spec: + containers: + - name: test-server + image: quay.io/ohsu-comp-bio/funnel-plugins-test-server:testing + imagePullPolicy: Always + ports: + - containerPort: 8080 + volumeMounts: + - name: funnel-plugin-test-users + mountPath: /app/example-users.csv + subPath: example-users.csv + command: ["/bin/sh", "-c"] + args: ["while true; do sleep 3600; done"] + volumes: + - name: funnel-plugin-test-users + configMap: + name: funnel-plugin-test-users + +--- +apiVersion: v1 +kind: Service +metadata: + name: funnel-plugin-test-server + namespace: {{ .Release.Namespace }} +spec: + selector: + app: funnel-plugin-test-server + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: funnel-plugin-test-users + namespace: {{ .Release.Namespace }} +data: + example-users.csv: |- + user,key,secret,GET,PUT,DELETE + example,key1,secret1,1,1,1 + foo,key2,secret2,1,0,1 + bar,key3,secret3,0,0,0 + foobar,key4,secret4,0,1,1 + alpha,key5,secret5,1,1,0 + beta,key6,secret6,1,0,0 + gamma,key7,secret7,0,1,0 + delta,key8,secret8,0,0,1 +{{ end }} diff --git a/helm/funnel/templates/server-configmap.yaml b/helm/funnel/templates/server-configmap.yaml new file mode 100644 index 000000000..11c1a7499 --- /dev/null +++ b/helm/funnel/templates/server-configmap.yaml @@ -0,0 +1,15 @@ +# This is the config for the Funnel Server +# +# The Worker config uses the template in `config/kubernetes-configmap-template.yaml` +# to dynamically create a new ConfigMap for each task (in `compute/kubernetes/backend.go`). +# +# This is done to support per-user task configurations (e.g. S3 credentials). +apiVersion: v1 +kind: ConfigMap +metadata: + name: funnel-server-config + namespace: {{ .Release.Namespace }} + +data: + funnel-server.yaml: |- +{{ tpl (.Files.Get "files/server-config.yaml") . | indent 4 }} diff --git a/helm/funnel/templates/server-deployment.yaml b/helm/funnel/templates/server-deployment.yaml new file mode 100644 index 000000000..0186403f5 --- /dev/null +++ b/helm/funnel/templates/server-deployment.yaml @@ -0,0 +1,96 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.name | default "funnel-server" }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "funnel.labels" . | nindent 4 }} +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 # This ensures only one pod is unavailable during the update process + maxSurge: 0 # No new pod is created until the old one is fully terminated + selector: + matchLabels: + {{- include "funnel.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "funnel.labels" . | nindent 8 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/server-configmap.yaml") . | sha256sum }} + checksum/values: {{ .Values | quote | sha256sum }} + spec: + serviceAccountName: funnel-sa-{{ .Release.Namespace }} + {{- if .Values.image.initContainers }} + initContainers: + {{- range .Values.image.initContainers }} + - name: {{ .name | default "initcontainer" }} + image: "{{ .image }}:{{ .tag }}" + imagePullPolicy: {{ .pullPolicy | default "IfNotPresent" }} + {{- if .command }} + command: + {{- range .command }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .args }} + args: + {{- range .args }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .volumeMounts }} + volumeMounts: + {{- toYaml .volumeMounts | nindent 10 }} + {{- end }} + {{- if .env }} + env: + {{- toYaml .env | nindent 10 }} + {{- end }} + {{- end }} + {{- end }} + containers: + - name: funnel + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: Always + command: + - 'funnel' + - 'server' + - 'run' + - '--config' + - '/etc/config/funnel-server.yaml' + resources: + requests: + cpu: {{ .Values.resources.requests.cpu }} + memory: {{ .Values.resources.requests.memory }} + limits: + cpu: {{ .Values.resources.limits.cpu }} + memory: {{ .Values.resources.limits.memory }} + ports: + - name: http + containerPort: 8000 + - name: rpc + containerPort: 9090 + readinessProbe: + httpGet: + path: /healthz + port: 8000 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /healthz + port: 8000 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 3 + volumeMounts: + {{- toYaml .Values.volumeMounts | nindent 10 }} + + volumes: + {{- toYaml .Values.volumes | nindent 6 }} diff --git a/helm/funnel/templates/service.yaml b/helm/funnel/templates/service.yaml new file mode 100644 index 000000000..8780e71fa --- /dev/null +++ b/helm/funnel/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: funnel + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + ports: + - name: http + protocol: TCP + port: 8000 + targetPort: 8000 + - name: rpc + protocol: TCP + port: 9090 + targetPort: 9090 + selector: + {{- include "funnel.selectorLabels" . | nindent 4 }} diff --git a/helm/funnel/templates/serviceaccount.yaml b/helm/funnel/templates/serviceaccount.yaml new file mode 100644 index 000000000..203935c07 --- /dev/null +++ b/helm/funnel/templates/serviceaccount.yaml @@ -0,0 +1,22 @@ +{{- if .Values.rbac.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: funnel-sa-{{ .Release.Namespace }} + namespace: {{ .Values.Kubernetes.Namespace }} + labels: + {{- include "funnel.labels" . | nindent 4 }} +automountServiceAccountToken: true +{{- if and .Values.Kubernetes.JobsNamespace (ne .Values.Kubernetes.JobsNamespace .Values.Kubernetes.Namespace) }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: funnel-sa-{{ .Release.Namespace }} + namespace: {{ .Values.Kubernetes.JobsNamespace }} + labels: + {{- include "funnel.labels" . | nindent 4 }} +automountServiceAccountToken: true +{{- end }} +{{- end }} diff --git a/helm/funnel/templates/tests/test-connection.yaml b/helm/funnel/templates/tests/test-connection.yaml new file mode 100644 index 000000000..cb2d151cc --- /dev/null +++ b/helm/funnel/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "funnel.fullname" . }}-test-connection" + labels: + {{- include "funnel.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "funnel.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/helm/funnel/templates/worker-configmap.yaml b/helm/funnel/templates/worker-configmap.yaml new file mode 100644 index 000000000..5974512c6 --- /dev/null +++ b/helm/funnel/templates/worker-configmap.yaml @@ -0,0 +1,22 @@ +# This is the config containing the templates used by the Funnel Worker +apiVersion: v1 +kind: ConfigMap +metadata: + name: funnel-worker-templates + namespace: {{ .Release.Namespace }} + +data: + +{{ (.Files.Glob "files/worker-pv.yaml").AsConfig | indent 2 }} + +{{ (.Files.Glob "files/worker-pvc.yaml").AsConfig | indent 2 }} + +{{ (.Files.Glob "files/worker-job.yaml").AsConfig | indent 2 }} + +{{ (.Files.Glob "files/executor-job.yaml").AsConfig | indent 2 }} + +{{ (.Files.Glob "files/serviceaccount.yaml").AsConfig | indent 2 }} + +{{ (.Files.Glob "files/role.yaml").AsConfig | indent 2 }} + +{{ (.Files.Glob "files/rolebinding.yaml").AsConfig | indent 2 }} diff --git a/helm/funnel/values.yaml b/helm/funnel/values.yaml new file mode 100644 index 000000000..fd13c6dc4 --- /dev/null +++ b/helm/funnel/values.yaml @@ -0,0 +1,526 @@ +# Kubernetes-specifc Settings +replicaCount: 1 + +image: + repository: quay.io/ohsu-comp-bio/funnel + pullPolicy: Always + initContainers: + # Plugin Init Container + - name: plugins + image: quay.io/ohsu-comp-bio/funnel-plugins + tag: pr-1 + pullPolicy: Always + command: + - cp + - /app/build/plugins/authorizer + - /opt/funnel/plugin-binaries/auth-plugin + volumeMounts: + - name: plugin-volume + mountPath: /opt/funnel/plugin-binaries + +labels: + app: funnel + +# Default Funnel Server (Task Database) +# Note: This is a non-persistent database for testing/demo purposes. +postgresql: + enabled: true + global: + postgresql: + auth: + database: funnel + username: funnel + password: example + postgresPassword: example + primary: + persistence: + enabled: false + +mongodb: + enabled: false + app: funnel-mongodb + commonLabels: + app: funnel-mongodb + image: + registry: public.ecr.aws + architecture: standalone + auth: + enabled: true + rootUser: example + rootPassword: example + persistence: + enabled: false + size: 1Gi + +rbac: + create: true + +# Backoff/Completions for Worker + Executor Jobs +backoffLimit: 1 +completions: 1 + +# Resource Requests/Limits for Worker + Executor Jobs +resources: + requests: + cpu: 100m + memory: 512Mi + ephemeral_storage: 512Mi + limits: + cpu: 1000m + memory: 2048Mi + ephemeral_storage: 2048Mi + +# AWS STS Source + Region for S3 CSI Driver +authenticationSource: pod +stsRegion: us-east-1 + +# Kubernetes Service Settings +service: + type: ClusterIP + httpPort: 8000 + rpcPort: 9090 + +# Funnel Worker + Executor storage (S3 bucket) +storage: + driver: aws-s3 + size: 10Mi + accessMode: ReadWriteMany + className: s3-csi-sc + provisioner: s3.csi.aws.com + createStorageClass: true + +# Funnel Default Settings (with BoltDB replaced with MongoDB) +# +# These are passed into `files/server-config.yaml` and made available to the deployment via `server-configmap.yaml` +# +# - Ref: https://github.com/ohsu-comp-bio/funnel/blob/master/config/default-config.yaml + +# The name of the active server database backend +# Available backends: boltdb, badger, datastore, dynamodb, elastic, mongodb +Database: postgres + +# The name of the active compute backend +# Available backends: local, htcondor, slurm, pbs, gridengine, manual, aws-batch, kubernetes +Compute: kubernetes + +# The name of the active event writer backend(s). +# Available backends: log, boltdb, badger, datastore, dynamodb, elastic, mongodb, kafka +EventWriters: + - postgres + - log + +Logger: + # Logging levels: debug, info, error + level: debug + # Write logs to this path. If empty, logs are written to stderr. + outputFile: "" + +Server: + # Hostname of the Funnel server. + HostName: funnel + + # Port used for HTTP communication and the web dashboard. + HTTPPort: "8000" + + # Port used for RPC communication. + RPCPort: "9090" + + # Require basic authentication for the server APIs using a password. + # If used, make sure to properly restrict access to the config file + # (e.g. chmod 600 funnel.config.yml) + # BasicAuth: + # - User: user1 + # Password: abc123 + # - User: user2 + # Password: foobar + + # Include a "Cache-Control: no-store" HTTP header in Get/List responses + # to prevent caching by intermediary services. + DisableHTTPCache: true + +RPCClient: + # RPC server address + ServerAddress: localhost:9090 + + # Credentials for Basic authentication for the server APIs using a password. + # If used, make sure to properly restrict access to the config file + # (e.g. chmod 600 funnel.config.yml) + # User: funnel + # Password: abc123 + + # connection timeout. + Timeout: + duration: 60s + + # The maximum number of times that a request will be retried for failures. + # Time between retries follows an exponential backoff starting at 5 seconds + # up to 1 minute + MaxRetries: 10 + +# The scheduler is used for the Manual compute backend. +Scheduler: + # How often to run a scheduler iteration. + ScheduleRate: 1s + # How many tasks to schedule in one iteration. + ScheduleChunk: 10 + # How long to wait between updates before marking a node dead. + NodePingTimeout: + duration: 60s + # How long to wait for a node to start, before marking the node dead. + NodeInitTimeout: + duration: 300s + +Node: + # If empty, a node ID will be automatically generated. + ID: "" + + # If the node has been idle for longer than the timeout, it will shut down. + # -1 means there is no timeout. 0 means timeout immediately after the first task. + Timeout: + disabled: true + + # A Node will automatically try to detect what resources are available to it. + # Defining Resources in the Node configuration overrides this behavior. + Resources: + # CPUs available. + Cpus: 0 + + # RAM available, in GB. + RamGb: 0.0 + + # Disk space available, in GB. + DiskGb: 0.0 + + # For low-level tuning. + # How often to sync with the Funnel server. + UpdateRate: 5s + +Worker: + # Files created during processing will be written in this directory. + WorkDir: ./funnel-work-dir + + # For low-level tuning. + # How often to poll for cancel signals + PollingRate: 5s + + # For low-level tuning. + # How often to send stdout/err task log updates to the Funnel server. + # Setting this to 0 will result in these fields being updated a single time + # after the executor exits. + LogUpdateRate: 5s + + # Max bytes to store for stdout/err in the task log. + LogTailSize: 10000 # 10 KB + + # Normally the worker deletes its working directory after executing. + # This option disables that behavior. + LeaveWorkDir: false + + # Limit the number of concurrent downloads/uploads + MaxParallelTransfers: 10 + +#------------------------------------------------------------------------------- +# Databases and/or Event Writers/Handlers +#------------------------------------------------------------------------------- + +BoltDB: + # Path to the database file + Path: ./funnel-work-dir/funnel.db + +DynamoDB: + # Basename to use for dynamodb tables + TableBasename: funnel + AWSConfig: + # AWS region + Region: "" + # AWS Access key ID + Key: "" + # AWS Secret Access Key + Secret: "" + +Elastic: + # Prefix to use for indexes (task, events, nodes) + IndexPrefix: funnel + # URL of the elasticsearch server. + URL: http://localhost:9200 + +# Google Cloud Datastore task database. +Datastore: + Project: "" + # Path to account credentials file. + # Optional. If possible, credentials will be automatically discovered + # from the environment. + CredentialsFile: "" + +MongoDB: + # Addrs holds the addresses for the seed servers. + # K8s Note: These are computed dynamically in templates (in `server-configmap.yaml`) so that users + # don't have to manually set them. + Addrs: [] + # Database is the database name used within MongoDB to store funnel data. + Database: funnel + # Timeout is the amount of time to wait for a server to respond when + # first connecting and on follow up operations in the session. If + # timeout is zero, the call may block forever waiting for a connection + # to be established. + Timeout: + duration: 300s + # Username and Password inform the credentials for the initial authentication + # done on the database defined by the Database field. + Username: example + +Postgres: + Host: funnel-postgresql.default.svc.cluster.local + Database: funnel + User: funnel + Password: example + AdminUser: postgres + AdminPassword: example + Timeout: + duration: 300s + +Kafka: + Topic: funnel + +#------------------------------------------------------------------------------- +# Compute Backends +#------------------------------------------------------------------------------- + +HTCondor: + # Turn off task state reconciler. When enabled, Funnel communicates with the HPC + # scheduler to find tasks that are stuck in a queued state or errored and + # updates the task state accordingly. + DisableReconciler: true + # ReconcileRate is how often the compute backend compares states in Funnel's backend + # to those reported by the backend + ReconcileRate: 10s + TemplateFile: "" + Template: | + universe = vanilla + getenv = True + executable = {{.Executable}} + arguments = worker run --config {{.Config}} --task-id {{.TaskId}} + log = {{.WorkDir}}/condor-event-log + error = {{.WorkDir}}/funnel-stderr + output = {{.WorkDir}}/funnel-stdout + should_transfer_files = YES + when_to_transfer_output = ON_EXIT_OR_EVICT + {{if ne .Cpus 0 -}} + {{printf "request_cpus = %d" .Cpus}} + {{- end}} + {{if ne .RamGb 0.0 -}} + {{printf "request_memory = %.0f GB" .RamGb}} + {{- end}} + {{if ne .DiskGb 0.0 -}} + {{printf "request_disk = %.0f GB" .DiskGb}} + {{- end}} + + queue + +PBS: + # Turn off task state reconciler. When enabled, Funnel communicates with the HPC + # scheduler to find tasks that are stuck in a queued state or errored and + # updates the task state accordingly. + DisableReconciler: true + # ReconcileRate is how often the compute backend compares states in Funnel's backend + # to those reported by the backend + ReconcileRate: 10s + TemplateFile: "" + Template: | + #!bin/bash + #PBS -N {{.TaskId}} + #PBS -o {{.WorkDir}}/funnel-stdout + #PBS -e {{.WorkDir}}/funnel-stderr + {{if ne .Cpus 0 -}} + {{printf "#PBS -l nodes=1:ppn=%d" .Cpus}} + {{- end}} + {{if ne .RamGb 0.0 -}} + {{printf "#PBS -l mem=%.0fgb" .RamGb}} + {{- end}} + {{if ne .DiskGb 0.0 -}} + {{printf "#PBS -l file=%.0fgb" .DiskGb}} + {{- end}} + + {{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}} + +GridEngine: + TemplateFile: "" + Template: | + #!bin/bash + #$ -N {{.TaskId}} + #$ -o {{.WorkDir}}/funnel-stdout + #$ -e {{.WorkDir}}/funnel-stderr + #$ -l nodes=1 + {{if ne .Cpus 0 -}} + {{printf "#$ -pe mpi %d" .Cpus}} + {{- end}} + {{if ne .RamGb 0.0 -}} + {{printf "#$ -l h_vmem=%.0fG" .RamGb}} + {{- end}} + {{if ne .DiskGb 0.0 -}} + {{printf "#$ -l h_fsize=%.0fG" .DiskGb}} + {{- end}} + + {{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}} + +Slurm: + # Turn off task state reconciler. When enabled, Funnel communicates with the HPC + # scheduler to find tasks that are stuck in a queued state or errored and + # updates the task state accordingly. + DisableReconciler: true + # ReconcileRate is how often the compute backend compares states in Funnel's backend + # to those reported by the backend + ReconcileRate: 10s + TemplateFile: "" + Template: | + #!/bin/bash + #SBATCH --job-name {{.TaskId}} + #SBATCH --ntasks 1 + #SBATCH --error {{.WorkDir}}/funnel-stderr + #SBATCH --output {{.WorkDir}}/funnel-stdout + {{if ne .Cpus 0 -}} + {{printf "#SBATCH --cpus-per-task %d" .Cpus}} + {{- end}} + {{if ne .RamGb 0.0 -}} + {{printf "#SBATCH --mem %.0fGB" .RamGb}} + {{- end}} + {{if ne .DiskGb 0.0 -}} + {{printf "#SBATCH --tmp %.0fGB" .DiskGb}} + {{- end}} + + {{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}} + +# AWSBatch describes the configuration for the AWS Batch compute backend. +AWSBatch: + # Turn off task state reconciler. When enabled, Funnel communicates with AWS Batch + # to find tasks that never started and updates task state accordingly. + DisableReconciler: true + # ReconcileRate is how often the compute backend compares states in Funnel's backend + # to those reported by AWS Batch + ReconcileRate: 10s + # JobDefinition can be either a name or the Amazon Resource Name (ARN). + JobDefinition: "funnel-job-def" + # JobQueue can be either a name or the Amazon Resource Name (ARN). + JobQueue: "funnel-job-queue" + # AWS region of the specified job queue and to create the job definition in + Region: "" + Key: "" + Secret: "" + +# Kubernetes describes the configuration for the Kubernetes compute backend. +Kubernetes: + # The executor used to execute tasks. Available executors: docker, kubernetes + Executor: "kubernetes" + # Turn off task state reconciler. When enabled, Funnel communicates with Kubernetes + # to find tasks that are stuck in a queued state or errored and + # updates the task state accordingly. + DisableReconciler: false + DisableJobCleanup: false + # ReconcileRate is how often the compute backend compares states in Funnel's backend + # to those reported by the backend + ReconcileRate: 10s + # Kubernetes Namespace to spawn jobs within + Namespace: "" + # Kubernetes Namespace to spawn jobs within + JobsNamespace: "" + # Kubernetes ServiceAccount to use for the job + ServiceAccount: "" + # Master batch job template. See: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#job-v1-batch + WorkerTemplate: "" + # NodeSelector (scheduling) + NodeSelector: {} + # Tolerations (scheduling) + Tolerations: [] + + # Job template used for executing the tasks. + ExecutorTemplate: "" + PVTemplate: "" + PVCTemplate: "" + +#------------------------------------------------------------------------------- +# Storage +#------------------------------------------------------------------------------- + +# If possible, credentials will be automatically discovered +# from the environment. + +# Local file system. +LocalStorage: + # Whitelist of local directory paths which Funnel is allowed to access. + AllowedDirs: + - ./ + +# HTTPStorage is used to download public files on the web via a GET request. +HTTPStorage: + # Timeout for http(s) GET requests. + Timeout: 30s + +AmazonS3: + Disabled: false + # The maximum number of times that a request will be retried for failures. + AWSConfig: + MaxRetries: 10 + # AWS Access key ID + Key: "" + # AWS Secret Access Key + Secret: "" + # Server Side Encryption (SSE) settings + SSE: + # Customer Provided Key + # ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html + CustomerKeyFile: "" + # KMS Key + # ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html + KMSKey: "" + +# Configure storage backends for S3 providers such as Minio and/or Ceph +# GenericS3: +# - Disabled: true +# Endpoint: "" +# Key: "" +# Secret: "" +# KmsKeyID: "" + +GoogleStorage: + Disabled: false + # Path to account credentials file. + # Optional. If possible, credentials will be automatically discovered + # from the environment. + CredentialsFile: "" +# Available backends: local, htcondor, slurm, pbs, gridengine, manual, aws-batch, kubernetes/Kube +Swift: + Disabled: false + UserName: "" + Password: "" + AuthURL: "" + TenantName: "" + TenantID: "" + RegionName: "" + # 500 MB + ChunkSizeBytes: 500000000 + +FTPStorage: + Disabled: false + Timeout: 10s + User: "anonymous" + Password: "anonymous" + +# Overrides the default volumes configured in the server deployment. +# If custom volumes are specified, they will fully replace the default values. +volumes: + - name: funnel-server-config-volume + configMap: + name: funnel-server-config + + - name: plugin-volume + emptyDir: {} # Shared volume + +# Overrides the default volumesMounts configured in the server deployment. +# If custom volumesMounts are specified, they will fully replace the default values. +volumeMounts: + - name: funnel-server-config-volume + mountPath: /etc/config/funnel-server.yaml + subPath: funnel-server.yaml + + - name: plugin-volume + mountPath: /opt/funnel/plugin-binaries From d79925ae14816195b79a24fc28851c60e9bab157 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 6 Jan 2026 13:00:47 -0800 Subject: [PATCH 3/4] Update lint_test.yaml [no ci[ --- .github/workflows/lint_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint_test.yaml b/.github/workflows/lint_test.yaml index b22458e8d..8fb3770db 100644 --- a/.github/workflows/lint_test.yaml +++ b/.github/workflows/lint_test.yaml @@ -36,7 +36,7 @@ jobs: - name: Run chart-testing (lint) run: ct lint --config .github/ct.yaml - # TODO: add back in when we have tests + # TODO: add back in when tests are re-enabled # deploy-and-test-chart: # name: Deploy and Test Chart # timeout-minutes: 20 From 7d423ba9af484ba6b87e4d623fdfd0873fe548ab Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 6 Jan 2026 13:15:56 -0800 Subject: [PATCH 4/4] fix: Resolve linting errors --- helm/aws-es-proxy/values.yaml | 5 +++++ helm/funnel/Chart.yaml | 4 ++-- helm/funnel/values.yaml | 16 ++++++++-------- helm/grip/values.yaml | 5 +++++ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/helm/aws-es-proxy/values.yaml b/helm/aws-es-proxy/values.yaml index 11c1c200f..d4d413097 100644 --- a/helm/aws-es-proxy/values.yaml +++ b/helm/aws-es-proxy/values.yaml @@ -109,6 +109,11 @@ resources: # -- (string) The maximum amount of memory the container can use memory: 2Gi +# Added default to resolve nil pointer error with `ct` linting: +# <.Values.persistence.capacity>: nil pointer evaluating interface {}.capacity +persistence: + capacity: 1Gi + # -- (map) Kubernetes service information. service: # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". diff --git a/helm/funnel/Chart.yaml b/helm/funnel/Chart.yaml index 5f391d054..5d48fb078 100644 --- a/helm/funnel/Chart.yaml +++ b/helm/funnel/Chart.yaml @@ -23,14 +23,14 @@ version: 0.1.76 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 2025-12-22 +appVersion: "2025-12-22" dependencies: - name: postgresql version: 18.1.15 repository: https://charts.bitnami.com/bitnami condition: postgresql.enabled - + - name: mongodb version: 13.9.4 repository: https://charts.bitnami.com/bitnami diff --git a/helm/funnel/values.yaml b/helm/funnel/values.yaml index fd13c6dc4..2ef288bc2 100644 --- a/helm/funnel/values.yaml +++ b/helm/funnel/values.yaml @@ -209,8 +209,8 @@ Worker: # after the executor exits. LogUpdateRate: 5s - # Max bytes to store for stdout/err in the task log. - LogTailSize: 10000 # 10 KB + # Max bytes to store for stdout/err in the task log (10 KB) + LogTailSize: 10000 # Normally the worker deletes its working directory after executing. # This option disables that behavior. @@ -219,9 +219,9 @@ Worker: # Limit the number of concurrent downloads/uploads MaxParallelTransfers: 10 -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # Databases and/or Event Writers/Handlers -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- BoltDB: # Path to the database file @@ -282,9 +282,9 @@ Postgres: Kafka: Topic: funnel -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # Compute Backends -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- HTCondor: # Turn off task state reconciler. When enabled, Funnel communicates with the HPC @@ -437,9 +437,9 @@ Kubernetes: PVTemplate: "" PVCTemplate: "" -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # Storage -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # If possible, credentials will be automatically discovered # from the environment. diff --git a/helm/grip/values.yaml b/helm/grip/values.yaml index 772ea7c29..f34771227 100644 --- a/helm/grip/values.yaml +++ b/helm/grip/values.yaml @@ -49,6 +49,11 @@ resources: cpu: 0.2 memory: 256Mi +# Added default to resolve nil pointer error with `ct` linting: +# <.Values.persistence.capacity>: nil pointer evaluating interface {}.capacity +persistence: + capacity: 1Gi + autoscaling: enabled: false minReplicas: 1