From 4138130ba69508a19c1f6431970251d2e4c703ca Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Fri, 9 Feb 2024 16:19:30 -0500 Subject: [PATCH 01/29] Remove spaces that makes helm lint unhappy --- galaxy/templates/hook-cvmfs-fix.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/templates/hook-cvmfs-fix.yaml b/galaxy/templates/hook-cvmfs-fix.yaml index 1caa21ba..a91a76bf 100644 --- a/galaxy/templates/hook-cvmfs-fix.yaml +++ b/galaxy/templates/hook-cvmfs-fix.yaml @@ -1,5 +1,5 @@ {{- if and .Values.refdata.enabled (eq .Values.refdata.type "cvmfs") }} - # Include the code you want to run when both conditions are met +# Include the code you want to run when both conditions are met apiVersion: batch/v1 kind: Job metadata: From e0b960dadf8dc2495d6f6067f2a886dd21b1ef0b Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Fri, 9 Feb 2024 16:20:10 -0500 Subject: [PATCH 02/29] First pass at defining cron jobs in the values.yaml file. --- galaxy/templates/cronjob-maintenance.yaml | 46 ++++++++--------- galaxy/values.yaml | 63 ++++++++++++++++++++++- 2 files changed, 85 insertions(+), 24 deletions(-) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index 17b0219a..a9344494 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -1,47 +1,47 @@ +{{ if .Values.cronJobs -}} +{{ range $key, $cronjob := .Values.cronJobs }} +--- apiVersion: batch/v1 kind: CronJob metadata: - name: {{ include "galaxy.fullname" . }}-maintenance + name: {{ include "galaxy.fullname" $ }}-maintenance-{{ $key }} labels: - {{- include "galaxy.labels" . | nindent 4 }} + {{- include "galaxy.labels" $ | nindent 4 }} spec: - schedule: "0 2 * * *" + schedule: {{ $cronjob.schedule | quote }} jobTemplate: spec: template: spec: securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - {{- with .Values.nodeSelector }} + {{- toYaml $cronjob.securityContext | nindent 12 }} + {{- with $.nodeSelector }} nodeSelector: {{- toYaml . | nindent 16 }} {{- end }} containers: - - name: galaxy-maintenance - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} + - name: galaxy-maintenance-{{ $key }} + image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} # delete all tmp files older than walltime limit - command: - - find - - {{ .Values.persistence.mountPath }}/tmp - - '!' - - -newermt - - -{{ (index .Values "configs" "job_conf.yml" "runners" "k8s" "k8s_walltime_limit" | default 604800) }} seconds - - -type - - f - - -exec - - rm - - '{}' - - ; + command: {{ $cronjob.command }} + {{- if $cronjob.args }} + args: + {{- range $arg := $cronjob.args }} + - {{ tpl $arg $ }} + {{- end }} + {{- end }} volumeMounts: - name: galaxy-data - mountPath: {{ .Values.persistence.mountPath }} + mountPath: {{ $.Values.persistence.mountPath }} volumes: - name: galaxy-data - {{- if .Values.persistence.enabled }} + {{- if $.Values.persistence.enabled }} persistentVolumeClaim: - claimName: {{ template "galaxy.pvcname" . }} + claimName: {{ template "galaxy.pvcname" $ }} {{- else }} emptyDir: {} {{- end }} restartPolicy: OnFailure +{{- end }} +{{- end }} diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 9c85c0c0..ad8e3e25 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -267,6 +267,66 @@ extraEnv: [] # - name: EXAMPLE_ENV # value: MY_VALUE +cronJobs: + clean-tmp: + schedule: "0 2 * * *" + securityContext: + runAsUser: 0 + command: /bin/bash + args: + - "-c" + - "find" + - "{{ .Values.persistence.mountPath }}/tmp" + - "!" + - "-newermt" + - "-{{ (index .Values \"configs\" \"job_conf.yml\" \"runners\" \"k8s\" \"k8s_walltime_limit\" | default 604800) }} seconds" + - "-type" + - "f" + - "-exec" + - "rm" + - "{}" + - ";" + clean-jobs: + schedule: "10 2 * * *" + securityContext: + runAsUser: 0 + command: /usr/bin/tmpwatch + args: + - "-v" + - "--all" + - "--mtime" + - "--dirmtime" + - "7d" + - "{{ .Values.persistence.mountPath }}/jobs_directory" + cleanup-datasets: + schedule: "20 2 * * *" + securityContext: + runAsUser: 0 + command: /galaxy/server/.venv/bin/python + args: + - "/galaxy/server/scripts/cleanup_datasets/pgcleanup.py" + - "-c" + - "/galaxy/server/config/galaxy.yml" + - "-o" + - "7" + - "-l" + - "{{ .Values.persistence.mountPath }}/tmp" + - "-w" + - "128MB" + - "delete_userless_histories" + - "delete_exported_histories" + cleanup-tusd-store: + schedule: "30 2 * * *" + securityContext: + runAsUser: 0 + command: /usr/bin/tmpwatch + args: + - "-v" + - "--all" + - "--mtime" + - "--dirmtime" + - "7d" + - "{{ .Values.persistence.mountPath }}/tmp/tus_upload_store" ingress: #- Should ingress be enabled. Defaults to `true` enabled: true @@ -301,7 +361,8 @@ resources: memory: 7G ephemeral-storage: 10Gi -nodeSelector: {} +#nodeSelector: {} +nodeSelector: "cloud:google:com/gke-nodepool: default-pool" tolerations: [] From 9c84d17cfc3ce76d0df53d4590646e904e1ebdad Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Tue, 13 Feb 2024 08:07:27 -0500 Subject: [PATCH 03/29] Fix command formatting and quoting --- galaxy/templates/cronjob-maintenance.yaml | 7 +++- galaxy/values.yaml | 49 +++++++++++++---------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index a9344494..2ae3bde6 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -24,11 +24,14 @@ spec: image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" imagePullPolicy: {{ $.Values.image.pullPolicy }} # delete all tmp files older than walltime limit - command: {{ $cronjob.command }} + command: + {{- range $cmd := $cronjob.command }} + - {{ tpl $cmd $ | quote }} + {{- end}} {{- if $cronjob.args }} args: {{- range $arg := $cronjob.args }} - - {{ tpl $arg $ }} + - {{ tpl $arg $ | quote }} {{- end }} {{- end }} volumeMounts: diff --git a/galaxy/values.yaml b/galaxy/values.yaml index ad8e3e25..7dba17cd 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -272,14 +272,16 @@ cronJobs: schedule: "0 2 * * *" securityContext: runAsUser: 0 - command: /bin/bash + command: + - "/bin/bash" args: - "-c" - "find" - "{{ .Values.persistence.mountPath }}/tmp" - "!" - "-newermt" - - "-{{ (index .Values \"configs\" \"job_conf.yml\" \"runners\" \"k8s\" \"k8s_walltime_limit\" | default 604800) }} seconds" + - "-{{ (index .Values \"configs\" \"job_conf.yml\" \"runners\" \"k8s\" \"k8s_walltime_limit\" | default 604800) }}" + - "seconds" - "-type" - "f" - "-exec" @@ -290,7 +292,8 @@ cronJobs: schedule: "10 2 * * *" securityContext: runAsUser: 0 - command: /usr/bin/tmpwatch + command: + - "/usr/bin/tmpwatch" args: - "-v" - "--all" @@ -302,31 +305,33 @@ cronJobs: schedule: "20 2 * * *" securityContext: runAsUser: 0 - command: /galaxy/server/.venv/bin/python + command: + - "/galaxy/server/.venv/bin/python" args: - - "/galaxy/server/scripts/cleanup_datasets/pgcleanup.py" - - "-c" - - "/galaxy/server/config/galaxy.yml" - - "-o" - - "7" - - "-l" - - "{{ .Values.persistence.mountPath }}/tmp" - - "-w" - - "128MB" - - "delete_userless_histories" - - "delete_exported_histories" + - "/galaxy/server/scripts/cleanup_datasets/pgcleanup.py" + - "-c" + - "/galaxy/server/config/galaxy.yml" + - "-o" + - "7" + - "-l" + - "{{ .Values.persistence.mountPath }}/tmp" + - "-w" + - "128MB" + - "delete_userless_histories" + - "delete_exported_histories" cleanup-tusd-store: schedule: "30 2 * * *" securityContext: runAsUser: 0 - command: /usr/bin/tmpwatch + command: + - "/usr/bin/tmpwatch" args: - - "-v" - - "--all" - - "--mtime" - - "--dirmtime" - - "7d" - - "{{ .Values.persistence.mountPath }}/tmp/tus_upload_store" + - "-v" + - "--all" + - "--mtime" + - "--dirmtime" + - "7d" + - "{{ .Values.persistence.mountPath }}/tmp/tus_upload_store" ingress: #- Should ingress be enabled. Defaults to `true` enabled: true From d2941d7f78a1bbed894ffd0ae6844393932ea00b Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Tue, 13 Feb 2024 09:24:57 -0500 Subject: [PATCH 04/29] Remove nodeSelector --- galaxy/values.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 7dba17cd..c977e821 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -366,8 +366,7 @@ resources: memory: 7G ephemeral-storage: 10Gi -#nodeSelector: {} -nodeSelector: "cloud:google:com/gke-nodepool: default-pool" +nodeSelector: {} tolerations: [] From 95474e2b3e7327155c720adbf1ae8fb9e6b1ef0f Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Fri, 16 Feb 2024 13:21:22 -0500 Subject: [PATCH 05/29] Tmp cleanup and maintenance scripts are treated as special cases. --- galaxy/templates/cronjob-maintenance.yaml | 99 +++++++++++++++++++++++ galaxy/values.yaml | 78 +++++------------- 2 files changed, 118 insertions(+), 59 deletions(-) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index 2ae3bde6..2d1aa39b 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -1,3 +1,100 @@ +{{- if .Values.maintenance.tmp.enabled }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "galaxy.fullname" . }}-maintenance-clean-tmp + labels: + {{- include "galaxy.labels" . | nindent 4 }} +spec: + schedule: {{ .Values.maintenance.tmp.schedule | quote }} + jobTemplate: + spec: + template: + spec: + securityContext: + runAsUser: 0 + {{- toYaml .Values.securityContext | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 16 }} + {{- end }} + containers: + - name: galaxy-maintenance-clean-tmp + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + # delete all tmp files older than walltime limit + command: + - find + - {{ .Values.persistence.mountPath }}/tmp + - '!' + - -newermt + - "{{ .Values.maintenance.tmp.seconds }} seconds" + - -type + - f + - -exec + - rm + - '{}' + - ; + volumeMounts: + - name: galaxy-data + mountPath: {{ .Values.persistence.mountPath }} + volumes: + - name: galaxy-data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ template "galaxy.pvcname" . }} + {{- else }} + emptyDir: {} + {{- end }} + restartPolicy: OnFailure +{{- end }} +{{- if .Values.maintenance.script.enabled }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "galaxy.fullname" . }}-maintenance-script + labels: + {{- include "galaxy.labels" . | nindent 4 }} +spec: + schedule: {{ .Values.maintenance.script.schedule | quote }} + jobTemplate: + spec: + template: + spec: + securityContext: + runAsUser: 0 + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 16 }} + {{- end }} + containers: + - name: galaxy-maintenance + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + # delete all tmp files older than walltime limit + command: + - "/bin/bash" + - "-c" + args: + - "/galaxy/server/scripts/maintenance.sh" + - "--no-dry-run" + - "--days" + - {{ .Values.maintenance.script.days | quote }} + volumeMounts: + - name: galaxy-data + mountPath: {{ .Values.persistence.mountPath }} + volumes: + - name: galaxy-data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ template "galaxy.pvcname" . }} + {{- else }} + emptyDir: {} + {{- end }} + restartPolicy: OnFailure +{{- end }} {{ if .Values.cronJobs -}} {{ range $key, $cronjob := .Values.cronJobs }} --- @@ -13,8 +110,10 @@ spec: spec: template: spec: + {{- if $cronjob.securityContext }} securityContext: {{- toYaml $cronjob.securityContext | nindent 12 }} + {{- end}} {{- with $.nodeSelector }} nodeSelector: {{- toYaml . | nindent 16 }} diff --git a/galaxy/values.yaml b/galaxy/values.yaml index c977e821..fd821f04 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -267,71 +267,31 @@ extraEnv: [] # - name: EXAMPLE_ENV # value: MY_VALUE +#- User defined CronJobs cronJobs: - clean-tmp: - schedule: "0 2 * * *" + chown: + schedule: "0 * * * *" securityContext: runAsUser: 0 command: - - "/bin/bash" + - /usr/bin/chown args: - - "-c" - - "find" - - "{{ .Values.persistence.mountPath }}/tmp" - - "!" - - "-newermt" - - "-{{ (index .Values \"configs\" \"job_conf.yml\" \"runners\" \"k8s\" \"k8s_walltime_limit\" | default 604800) }}" - - "seconds" - - "-type" - - "f" - - "-exec" - - "rm" - - "{}" - - ";" - clean-jobs: + - "-R" + - "galaxy" + - "/galaxy/server/database" +#- Maintenance tasks that should be run periodically to prevent storage from being exhausted +maintenance: + #- Remove files in the tmp directory. + tmp: + enabled: true + schedule: "5 2 * * *" + seconds: "604800" # Default wall time for jobs + #- The maintenance script removes histories, datasets, and other Galaxy objects older than 7 days. + script: + enabled: true schedule: "10 2 * * *" - securityContext: - runAsUser: 0 - command: - - "/usr/bin/tmpwatch" - args: - - "-v" - - "--all" - - "--mtime" - - "--dirmtime" - - "7d" - - "{{ .Values.persistence.mountPath }}/jobs_directory" - cleanup-datasets: - schedule: "20 2 * * *" - securityContext: - runAsUser: 0 - command: - - "/galaxy/server/.venv/bin/python" - args: - - "/galaxy/server/scripts/cleanup_datasets/pgcleanup.py" - - "-c" - - "/galaxy/server/config/galaxy.yml" - - "-o" - - "7" - - "-l" - - "{{ .Values.persistence.mountPath }}/tmp" - - "-w" - - "128MB" - - "delete_userless_histories" - - "delete_exported_histories" - cleanup-tusd-store: - schedule: "30 2 * * *" - securityContext: - runAsUser: 0 - command: - - "/usr/bin/tmpwatch" - args: - - "-v" - - "--all" - - "--mtime" - - "--dirmtime" - - "7d" - - "{{ .Values.persistence.mountPath }}/tmp/tus_upload_store" + days: "7" + ingress: #- Should ingress be enabled. Defaults to `true` enabled: true From 93930d7d48c82b7baf62d237fb615e19950d1e71 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Fri, 16 Feb 2024 13:29:25 -0500 Subject: [PATCH 06/29] Update the name of the container used to run the maintenance script. --- galaxy/templates/cronjob-maintenance.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index 2d1aa39b..9df57386 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -70,7 +70,7 @@ spec: {{- toYaml . | nindent 16 }} {{- end }} containers: - - name: galaxy-maintenance + - name: galaxy-maintenance-script image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} # delete all tmp files older than walltime limit From 287cf133f440a03b37cc3549e91462c745cf7844 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Mon, 19 Feb 2024 12:12:38 -0500 Subject: [PATCH 07/29] Define all cron jobs in the values.yaml file again, but allows time durations to be parameterized. --- galaxy/templates/cronjob-maintenance.yaml | 102 +--------------------- galaxy/values.yaml | 46 +++++++--- 2 files changed, 35 insertions(+), 113 deletions(-) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index 9df57386..3f626e35 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -1,101 +1,3 @@ -{{- if .Values.maintenance.tmp.enabled }} ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: {{ include "galaxy.fullname" . }}-maintenance-clean-tmp - labels: - {{- include "galaxy.labels" . | nindent 4 }} -spec: - schedule: {{ .Values.maintenance.tmp.schedule | quote }} - jobTemplate: - spec: - template: - spec: - securityContext: - runAsUser: 0 - {{- toYaml .Values.securityContext | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 16 }} - {{- end }} - containers: - - name: galaxy-maintenance-clean-tmp - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - # delete all tmp files older than walltime limit - command: - - find - - {{ .Values.persistence.mountPath }}/tmp - - '!' - - -newermt - - "{{ .Values.maintenance.tmp.seconds }} seconds" - - -type - - f - - -exec - - rm - - '{}' - - ; - volumeMounts: - - name: galaxy-data - mountPath: {{ .Values.persistence.mountPath }} - volumes: - - name: galaxy-data - {{- if .Values.persistence.enabled }} - persistentVolumeClaim: - claimName: {{ template "galaxy.pvcname" . }} - {{- else }} - emptyDir: {} - {{- end }} - restartPolicy: OnFailure -{{- end }} -{{- if .Values.maintenance.script.enabled }} ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: {{ include "galaxy.fullname" . }}-maintenance-script - labels: - {{- include "galaxy.labels" . | nindent 4 }} -spec: - schedule: {{ .Values.maintenance.script.schedule | quote }} - jobTemplate: - spec: - template: - spec: - securityContext: - runAsUser: 0 - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 16 }} - {{- end }} - containers: - - name: galaxy-maintenance-script - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - # delete all tmp files older than walltime limit - command: - - "/bin/bash" - - "-c" - args: - - "/galaxy/server/scripts/maintenance.sh" - - "--no-dry-run" - - "--days" - - {{ .Values.maintenance.script.days | quote }} - volumeMounts: - - name: galaxy-data - mountPath: {{ .Values.persistence.mountPath }} - volumes: - - name: galaxy-data - {{- if .Values.persistence.enabled }} - persistentVolumeClaim: - claimName: {{ template "galaxy.pvcname" . }} - {{- else }} - emptyDir: {} - {{- end }} - restartPolicy: OnFailure -{{- end }} -{{ if .Values.cronJobs -}} {{ range $key, $cronjob := .Values.cronJobs }} --- apiVersion: batch/v1 @@ -116,13 +18,12 @@ spec: {{- end}} {{- with $.nodeSelector }} nodeSelector: - {{- toYaml . | nindent 16 }} + {{- toYaml . | nindent 12 }} {{- end }} containers: - name: galaxy-maintenance-{{ $key }} image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" imagePullPolicy: {{ $.Values.image.pullPolicy }} - # delete all tmp files older than walltime limit command: {{- range $cmd := $cronjob.command }} - {{ tpl $cmd $ | quote }} @@ -146,4 +47,3 @@ spec: {{- end }} restartPolicy: OnFailure {{- end }} -{{- end }} diff --git a/galaxy/values.yaml b/galaxy/values.yaml index fd821f04..454215eb 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -279,19 +279,41 @@ cronJobs: - "-R" - "galaxy" - "/galaxy/server/database" -#- Maintenance tasks that should be run periodically to prevent storage from being exhausted -maintenance: - #- Remove files in the tmp directory. - tmp: - enabled: true + maintenance: schedule: "5 2 * * *" - seconds: "604800" # Default wall time for jobs - #- The maintenance script removes histories, datasets, and other Galaxy objects older than 7 days. - script: - enabled: true - schedule: "10 2 * * *" - days: "7" - + extraSettings: + days: 7 + securityContext: + runAsUser: 0 + command: + - "/bin/bash" + - "-c" + args: + - "/galaxy/server/scripts/maintenance.sh" + - "--no-dry-run" + - "--days" + - "{{ .Values.cronJobs.maintenance.extraSettings.days }}" + #- Remove files from the tmp directory that are older than the allowable wall time for a job + tmp: + schedule: "15 2 * * *" + extraSettings: + seconds: 604800 + securityContext: + runAsUser: 0 + command: + - /usr/bin/find + args: + - "{{ .Values.persistence.mountPath }}/tmp" + - "!" + - "-newermt" + - "{{ .Values.cronJobs.tmp.extraSettings.seconds }}" + - "seconds" + - "-type" + - "f" + - "-exec" + - "rm" + - "{}" + - ";" ingress: #- Should ingress be enabled. Defaults to `true` enabled: true From 8225be5590c53c6657d50f4562f4fc2d81bf71ba Mon Sep 17 00:00:00 2001 From: nuwang <2070605+nuwang@users.noreply.github.com> Date: Wed, 21 Feb 2024 00:45:24 +0530 Subject: [PATCH 08/29] Use walltime limit for cleanup and other minor tweaks --- galaxy/templates/cronjob-maintenance.yaml | 4 ++-- galaxy/values.yaml | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index 3f626e35..3932f35d 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: {{ include "galaxy.fullname" $ }}-maintenance-{{ $key }} + name: {{ include "galaxy.fullname" $ }}-cron-{{ $key }} labels: {{- include "galaxy.labels" $ | nindent 4 }} spec: @@ -21,7 +21,7 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} containers: - - name: galaxy-maintenance-{{ $key }} + - name: galaxy-cron-{{ $key }} image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" imagePullPolicy: {{ $.Values.image.pullPolicy }} command: diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 454215eb..a049f74d 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -282,7 +282,7 @@ cronJobs: maintenance: schedule: "5 2 * * *" extraSettings: - days: 7 + days: '7' securityContext: runAsUser: 0 command: @@ -292,12 +292,12 @@ cronJobs: - "/galaxy/server/scripts/maintenance.sh" - "--no-dry-run" - "--days" - - "{{ .Values.cronJobs.maintenance.extraSettings.days }}" + - "{{ tpl .Values.cronJobs.maintenance.extraSettings.days $ }}" #- Remove files from the tmp directory that are older than the allowable wall time for a job - tmp: + tmpdir: schedule: "15 2 * * *" extraSettings: - seconds: 604800 + days: '{{ div (index .Values "configs" "job_conf.yml" "runners" "k8s" "k8s_walltime_limit" | default 604800) 7 }}' securityContext: runAsUser: 0 command: @@ -306,7 +306,7 @@ cronJobs: - "{{ .Values.persistence.mountPath }}/tmp" - "!" - "-newermt" - - "{{ .Values.cronJobs.tmp.extraSettings.seconds }}" + - "{{ tpl .Values.cronJobs.tmpdir.extraSettings.days $ }}" - "seconds" - "-type" - "f" From 8cc5d943e2239348dcfa1b9493e24610f7fd6b28 Mon Sep 17 00:00:00 2001 From: nuwang <2070605+nuwang@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:01:21 +0530 Subject: [PATCH 09/29] Change find units from seconds to days --- galaxy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index a049f74d..b54813f1 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -307,7 +307,7 @@ cronJobs: - "!" - "-newermt" - "{{ tpl .Values.cronJobs.tmpdir.extraSettings.days $ }}" - - "seconds" + - "days" - "-type" - "f" - "-exec" From 64320c88cbaecaff12d06d14df0301f8e3b85a0d Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Wed, 21 Feb 2024 12:09:28 -0500 Subject: [PATCH 10/29] Parameterize the nodeSelector --- galaxy/templates/cronjob-maintenance.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index 3932f35d..6fa86432 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -16,9 +16,12 @@ spec: securityContext: {{- toYaml $cronjob.securityContext | nindent 12 }} {{- end}} - {{- with $.nodeSelector }} + {{- if $cronjob.nodeSelector }} nodeSelector: - {{- toYaml . | nindent 12 }} + {{- toYaml $cronjob.nodeSelector | nindent 12 }} + {{- else if $.Values.nodeSelector }} + nodeSelector: + {{- toYaml $.Values.nodeSelector | nindent 12 }} {{- end }} containers: - name: galaxy-cron-{{ $key }} From 2bbaa3a875eec54d898f7d6d9b25ea688fc63e16 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Wed, 21 Feb 2024 12:34:40 -0500 Subject: [PATCH 11/29] Remove the chown cron job --- galaxy/values.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index b54813f1..8f26de53 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -269,16 +269,6 @@ extraEnv: [] #- User defined CronJobs cronJobs: - chown: - schedule: "0 * * * *" - securityContext: - runAsUser: 0 - command: - - /usr/bin/chown - args: - - "-R" - - "galaxy" - - "/galaxy/server/database" maintenance: schedule: "5 2 * * *" extraSettings: From df1fb39f8534f6a07f1878d9d2510e9a445088e8 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Wed, 21 Feb 2024 16:39:01 -0500 Subject: [PATCH 12/29] Reverting my revert --- galaxy/values.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 8f26de53..a3861876 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -287,7 +287,7 @@ cronJobs: tmpdir: schedule: "15 2 * * *" extraSettings: - days: '{{ div (index .Values "configs" "job_conf.yml" "runners" "k8s" "k8s_walltime_limit" | default 604800) 7 }}' + last_modified: '{{ index .Values "configs" "job_conf.yml" "runners" "k8s" "k8s_walltime_limit" | default 604800 }}' securityContext: runAsUser: 0 command: @@ -296,8 +296,7 @@ cronJobs: - "{{ .Values.persistence.mountPath }}/tmp" - "!" - "-newermt" - - "{{ tpl .Values.cronJobs.tmpdir.extraSettings.days $ }}" - - "days" + - "{{ tpl .Values.cronJobs.tmpdir.extraSettings.last_modified $ }} seconds ago" - "-type" - "f" - "-exec" From 247cbf5fb4992d13d7ee891415df17d688397663 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Wed, 28 Feb 2024 12:58:54 -0600 Subject: [PATCH 13/29] Add extraFileMappings for cron jobs --- galaxy/templates/cronjob-maintenance.yaml | 15 +++++++++++++++ galaxy/values.yaml | 13 ++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index 6fa86432..954304a5 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -40,6 +40,11 @@ spec: volumeMounts: - name: galaxy-data mountPath: {{ $.Values.persistence.mountPath }} + {{- range $key, $entry := $cronjob.extraFileMappings -}} + - name: {{ include "galaxy.getExtraFilesUniqueName" $key }} + mountPath: {{ $key }} + subPath: {{ include "galaxy.getFilenameFromPath" $key }} + {{- end }} volumes: - name: galaxy-data {{- if $.Values.persistence.enabled }} @@ -48,5 +53,15 @@ spec: {{- else }} emptyDir: {} {{- end }} + {{- range $key, $entry := $cronjob.extraFileMappings -}} + - name: {{ include "galaxy.getExtraFilesUniqueName" $key }} + {{- if $entry.useSecret }} + secret: + secretName: {{ printf "%s-%s" (include "galaxy.fullname" $) (include "galaxy.getExtraFilesUniqueName" $key) }} + {{- else }} + configMap: + name: {{ printf "%s-%s" (include "galaxy.fullname" $) (include "galaxy.getExtraFilesUniqueName" $key) }} + {{- end }} + {{- end }} restartPolicy: OnFailure {{- end }} diff --git a/galaxy/values.yaml b/galaxy/values.yaml index a3861876..1fc25660 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -267,8 +267,9 @@ extraEnv: [] # - name: EXAMPLE_ENV # value: MY_VALUE -#- User defined CronJobs +#- CronJobs to perform periodic maintenacne tasks cronJobs: + #- Runs the maintenance.sh script to clean up the Galaxy database. maintenance: schedule: "5 2 * * *" extraSettings: @@ -283,6 +284,16 @@ cronJobs: - "--no-dry-run" - "--days" - "{{ tpl .Values.cronJobs.maintenance.extraSettings.days $ }}" + extraFileMappings: + /galaxy/server/scripts/maintenance.sh: + tpl: true + content: | + {{ .Files.Get "files/source/maintenance.sh" }} + /galaxy/server/scripts/cleanup_datasets/cleanup_datasets.py: + tpl: true + content: | + {{ .Files.Get "files/source/cleanup_datasets.py" }} + #- Remove files from the tmp directory that are older than the allowable wall time for a job tmpdir: schedule: "15 2 * * *" From cf779636840176347aced3133fcde69faa6b0eca Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Wed, 28 Feb 2024 12:59:07 -0600 Subject: [PATCH 14/29] Start documenting cron jobs --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3c150f37..e023b3de 100644 --- a/README.md +++ b/README.md @@ -417,6 +417,10 @@ The Galaxy application can be horizontally scaled for the web, job, or workflow by setting the desired values of the `webHandlers.replicaCount`, `jobHandlers.replicaCount`, and `workflowHandlers.replicaCount` configuration options. +## Cron jobs + +Two Cron jobs are defined to clean up Galaxy's database and the `tmp` directory. By default these +jobs run at 02:05 (the database maintenance script) and 02:15 (`tmp` directyory cleanup). Users ## Galaxy versions Some changes introduced in the chart sometimes rely on changes in the Galaxy From 13e4f0bc4a6701b612aec883d095e2be271c3a0b Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Fri, 8 Mar 2024 11:31:35 -0500 Subject: [PATCH 15/29] Make the galaxy.yml file available in a configmap for the maintenance cronjob --- galaxy/templates/configmap-galaxy.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 galaxy/templates/configmap-galaxy.yaml diff --git a/galaxy/templates/configmap-galaxy.yaml b/galaxy/templates/configmap-galaxy.yaml new file mode 100644 index 00000000..e69de29b From 27fd304e4d1090d8688bc267925c29c3f0086c42 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Fri, 8 Mar 2024 12:11:17 -0500 Subject: [PATCH 16/29] Update galaxy/values.yaml Co-authored-by: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> --- galaxy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 1fc25660..dc252c7e 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -267,7 +267,7 @@ extraEnv: [] # - name: EXAMPLE_ENV # value: MY_VALUE -#- CronJobs to perform periodic maintenacne tasks +#- CronJobs to perform periodic maintenance tasks cronJobs: #- Runs the maintenance.sh script to clean up the Galaxy database. maintenance: From 42488ff883ed0ec00d80d7fd3fb9cf73a5c77a5c Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Sun, 10 Mar 2024 16:55:10 -0400 Subject: [PATCH 17/29] Add helper to calculate the postgres connection string --- galaxy/disabled/configmap-galaxy.yaml | 9 +++++++++ galaxy/templates/_helpers.tpl | 7 +++++++ galaxy/templates/configmap-galaxy.yaml | 0 3 files changed, 16 insertions(+) create mode 100644 galaxy/disabled/configmap-galaxy.yaml delete mode 100644 galaxy/templates/configmap-galaxy.yaml diff --git a/galaxy/disabled/configmap-galaxy.yaml b/galaxy/disabled/configmap-galaxy.yaml new file mode 100644 index 00000000..6f49a14c --- /dev/null +++ b/galaxy/disabled/configmap-galaxy.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-galaxy-config + labels: + {{- include "galaxy.labels" $ | nindent 4 }} +kind: ConfigMap +data: + galaxy.yml: | + {{- .Values.galaxy | toYaml | nindent 4 }} diff --git a/galaxy/templates/_helpers.tpl b/galaxy/templates/_helpers.tpl index 4d7bb779..2f817cae 100644 --- a/galaxy/templates/_helpers.tpl +++ b/galaxy/templates/_helpers.tpl @@ -79,6 +79,13 @@ Return the postgresql database name to use {{- end -}} {{- end -}} +{{/* +Generate the connection string needed to connect to a Postres database +*/}} +{{- define "galaxy-postgresql.connection-string" -}} +{{- printf "postgresql://%s:%s@%s/galaxy%s" .Values.postgresql.galaxyDatabaseUser (include "galaxy.galaxyDbPassword" .) (include "galaxy-postgresql.fullname" .) .Values.postgresql.galaxyConnectionParams -}} +{{- end -}} + {{/* Return the rabbitmq cluster to use */}} diff --git a/galaxy/templates/configmap-galaxy.yaml b/galaxy/templates/configmap-galaxy.yaml deleted file mode 100644 index e69de29b..00000000 From 0fafea87f60fb9719e2e26491013a0148954a36a Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Sun, 10 Mar 2024 16:55:46 -0400 Subject: [PATCH 18/29] Add cron job documentation and remove Galaxy versions section --- README.md | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e023b3de..77480d43 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,8 @@ jobHandlers: failureThreshhold: 3 ``` +# Additional Configurations + ## Extra File Mappings The `extraFileMappings` field can be used to inject files to arbitrary paths in the `nginx` deployment, as well as any of the `job`, `web`, or `workflow` handlers, and the `init` jobs. @@ -420,22 +422,22 @@ by setting the desired values of the `webHandlers.replicaCount`, ## Cron jobs Two Cron jobs are defined to clean up Galaxy's database and the `tmp` directory. By default these -jobs run at 02:05 (the database maintenance script) and 02:15 (`tmp` directyory cleanup). Users -## Galaxy versions - -Some changes introduced in the chart sometimes rely on changes in the Galaxy -container image, especially in relation to the Kubernetes runner. This table -keeps track of recommended Chart versions for particular Galaxy versions as -breaking changes are introduced. Otherwise, the Galaxy image and chart should be -independently upgrade-able. In other words, upgrading the Galaxy image from -`21.05` to `21.09` should be a matter of `helm upgrade my-galaxy cloudve/galaxy ---reuse-values --set image.tag=21.09`. - - -| Chart version | Galaxy version | Description | -| :------------------ | :--------------- | :-------------- | -| `5.0` | `22.05` | Needs at least container image 22.05 as Galaxy switched from uwsgi to gunicorn | -| `4.0` | `21.05` | Needs [Galaxy PR#11899](https://github.com/galaxyproject/galaxy/pull/11899) for eliminating the CVMFS. If running chart 4.0+ with Galaxy image `21.01` or below, use the CVMFS instead with `--set setupJob.downloadToolConfs.enabled=false --set cvmfs.repositories.cvmfs-gxy-cloud=cloud.galaxyproject.org --set cvmfs.galaxyPersistentVolumeClaims.cloud.storage=1Gi --set cvmfs.galaxyPersistentVolumeClaims.cloud.storageClassName=cvmfs-gxy-cloud --set cvmfs.galaxyPersistentVolumeClaims.cloud.mountPath=/cvmfs/cloud.galaxyproject.org` | +jobs run at 02:05 (the database maintenance script) and 02:15 (`tmp` directyory cleanup). Users can +change the times the cron jobs are run by changing the `schedule` field in the `values.yaml` file: + +```yaml +cronJobs: + maintenance: + schedule: "30 6 * * *" # Execute the cron job at 6:30 UTC +``` + +To disable a cron job after Galaxy has been deployed simply set the schedule to a date that +can never occur such as midnight on Februrary 30th: + + +```bash +helm upgrade galaxy -n galaxy galaxy/galaxy --reuse-values --set cronJobs.maintenance.schedule="0 0 30 2 *" +``` ## Funding From 497b5729c94691c8a4da4d31125c519a363432d6 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Tue, 12 Mar 2024 15:08:55 -0400 Subject: [PATCH 19/29] Add env definitions --- galaxy/templates/cronjob-maintenance.yaml | 55 ++++++++++++++++++----- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index 954304a5..be71e35d 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -27,6 +27,15 @@ spec: - name: galaxy-cron-{{ $key }} image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" imagePullPolicy: {{ $.Values.image.pullPolicy }} + {{- if or $cronjob.defaultEnv $cronjob.extraEnv }} + env: + {{- if $cronjob.defaultEnv }} + {{- include "galaxy.podEnvVars" $}} + {{- end }} + {{- if $cronjob.extraEnv }} + {{- $cronjob.extraEnv | toYaml | nindent 12 }} + {{- end }} + {{- end }} command: {{- range $cmd := $cronjob.command }} - {{ tpl $cmd $ | quote }} @@ -40,7 +49,7 @@ spec: volumeMounts: - name: galaxy-data mountPath: {{ $.Values.persistence.mountPath }} - {{- range $key, $entry := $cronjob.extraFileMappings -}} + {{- range $key, $entry := $cronjob.extraFileMappings }} - name: {{ include "galaxy.getExtraFilesUniqueName" $key }} mountPath: {{ $key }} subPath: {{ include "galaxy.getFilenameFromPath" $key }} @@ -53,15 +62,41 @@ spec: {{- else }} emptyDir: {} {{- end }} - {{- range $key, $entry := $cronjob.extraFileMappings -}} - - name: {{ include "galaxy.getExtraFilesUniqueName" $key }} - {{- if $entry.useSecret }} - secret: - secretName: {{ printf "%s-%s" (include "galaxy.fullname" $) (include "galaxy.getExtraFilesUniqueName" $key) }} - {{- else }} - configMap: - name: {{ printf "%s-%s" (include "galaxy.fullname" $) (include "galaxy.getExtraFilesUniqueName" $key) }} + {{- range $key, $entry := $cronjob.extraFileMappings }} + - name: {{ include "galaxy.getExtraFilesUniqueName" $key }} + {{- if $entry.useSecret }} + secret: + secretName: {{ printf "%s-%s" (include "galaxy.fullname" $) (include "galaxy.getExtraFilesUniqueName" $key) }} + {{- else }} + configMap: + name: {{ printf "%s-%s" (include "galaxy.fullname" $) (include "galaxy.getExtraFilesUniqueName" $key) }} + {{- end }} {{- end }} - {{- end }} restartPolicy: OnFailure +{{- if $cronjob.extraFileMappings }} +{{- range $name, $entry := $cronjob.extraFileMappings }} +--- +apiVersion: v1 +metadata: + # Extract the filename portion only + name: {{ printf "%s-%s" (include "galaxy.fullname" $) (include "galaxy.getExtraFilesUniqueName" $name) }} + labels: + {{- include "galaxy.labels" $ | nindent 4 }} +{{- if $entry.useSecret }} +kind: Secret +type: Opaque +stringData: +{{- else }} +kind: ConfigMap +data: +{{- end }} + {{- include "galaxy.getFilenameFromPath" $name | nindent 2 }}: | + {{- if $entry.tpl }} + {{- tpl (tpl $entry.content $) $ | nindent 4 }} + {{- else }} + {{- $entry.content | nindent 4 }} + {{- end }} +{{- end }} {{- end }} + +{{- end }} \ No newline at end of file From da6a97a73a51e5fb625345326c1edb1e8df6a64b Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Tue, 12 Mar 2024 15:11:04 -0400 Subject: [PATCH 20/29] Maintenace job should include default env vars --- galaxy/values.yaml | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index dc252c7e..36f80939 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -269,31 +269,22 @@ extraEnv: [] #- CronJobs to perform periodic maintenance tasks cronJobs: - #- Runs the maintenance.sh script to clean up the Galaxy database. + #- Runs the maintenance.sh script to purge items in the Galaxy database that + #- have been flagged as deleted. maintenance: schedule: "5 2 * * *" extraSettings: + #- Purge items older than this. days: '7' securityContext: runAsUser: 0 + defaultEnv: true command: - - "/bin/bash" - - "-c" - args: - "/galaxy/server/scripts/maintenance.sh" + args: - "--no-dry-run" - "--days" - "{{ tpl .Values.cronJobs.maintenance.extraSettings.days $ }}" - extraFileMappings: - /galaxy/server/scripts/maintenance.sh: - tpl: true - content: | - {{ .Files.Get "files/source/maintenance.sh" }} - /galaxy/server/scripts/cleanup_datasets/cleanup_datasets.py: - tpl: true - content: | - {{ .Files.Get "files/source/cleanup_datasets.py" }} - #- Remove files from the tmp directory that are older than the allowable wall time for a job tmpdir: schedule: "15 2 * * *" @@ -317,7 +308,6 @@ cronJobs: ingress: #- Should ingress be enabled. Defaults to `true` enabled: true - #- ingressClassName: nginx canary: enabled: true @@ -490,7 +480,8 @@ configs: interactivetools_base_path: "{{$host := index .Values.ingress.hosts 0}}{{$path := index $host.paths 0}}{{$path.path}}" id_secret: mulled_resolution_cache_lock_dir: "/galaxy/server/local/mulled_cache_lock" - database_connection: postgresql://unused:because@overridden_by_envvar + database_connection: |- + {{ include "galaxy-postgresql.connection-string" .}} integrated_tool_panel_config: "/galaxy/server/config/mutable/integrated_tool_panel.xml" sanitize_allowlist_file: "/galaxy/server/config/mutable/sanitize_allowlist.txt" tool_config_file: "/galaxy/server/config/tool_conf.xml{{if .Values.setupJob.downloadToolConfs.enabled}},{{ .Values.setupJob.downloadToolConfs.volume.mountPath }}/config/shed_tool_conf.xml{{end}}" From cdc2a989d5135e510798808036d9444cfa91c2c1 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Tue, 12 Mar 2024 17:14:05 -0400 Subject: [PATCH 21/29] Update galaxy/values.yaml Co-authored-by: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> --- galaxy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 36f80939..02eb18b2 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -289,7 +289,7 @@ cronJobs: tmpdir: schedule: "15 2 * * *" extraSettings: - last_modified: '{{ index .Values "configs" "job_conf.yml" "runners" "k8s" "k8s_walltime_limit" | default 604800 }}' + lastModified: '{{ index .Values "configs" "job_conf.yml" "runners" "k8s" "k8s_walltime_limit" | default 604800 }}' securityContext: runAsUser: 0 command: From 63bc997ec1ba41cdff7b0eb4f509b1c554c586c3 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Tue, 12 Mar 2024 17:16:28 -0400 Subject: [PATCH 22/29] Update galaxy/values.yaml Co-authored-by: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> --- galaxy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 02eb18b2..519e9a39 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -298,7 +298,7 @@ cronJobs: - "{{ .Values.persistence.mountPath }}/tmp" - "!" - "-newermt" - - "{{ tpl .Values.cronJobs.tmpdir.extraSettings.last_modified $ }} seconds ago" + - "{{ tpl .Values.cronJobs.tmpdir.extraSettings.lastModified $ }} seconds ago" - "-type" - "f" - "-exec" From d96979a35cda1b6ff402cba3570a9ea6dfd08286 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Tue, 12 Mar 2024 18:34:16 -0400 Subject: [PATCH 23/29] Parameterize Docker image for cron jobs --- galaxy/templates/cronjob-maintenance.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index be71e35d..a5009841 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -25,7 +25,11 @@ spec: {{- end }} containers: - name: galaxy-cron-{{ $key }} + {{- if $cronjob.image }} + image: {{ $cronjob.image.repository }}:{{ $cronjob.image.tag }} + {{- else }} image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" + {{- end }} imagePullPolicy: {{ $.Values.image.pullPolicy }} {{- if or $cronjob.defaultEnv $cronjob.extraEnv }} env: From 3af24cd85cf97acc0a1105607af037e225ce388a Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Wed, 13 Mar 2024 14:58:41 -0400 Subject: [PATCH 24/29] Allow mode (permissions) to be defined on extraFileMappings --- galaxy/templates/cronjob-maintenance.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index a5009841..c6366435 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -75,6 +75,9 @@ spec: configMap: name: {{ printf "%s-%s" (include "galaxy.fullname" $) (include "galaxy.getExtraFilesUniqueName" $key) }} {{- end }} + {{- if $entry.mode }} + defaultMode: {{ $entry.mode }} + {{- end }} {{- end }} restartPolicy: OnFailure {{- if $cronjob.extraFileMappings }} From 3fe57237af05e77ff1619cda06d2ee0d49a27acf Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Wed, 13 Mar 2024 17:33:23 -0400 Subject: [PATCH 25/29] Add example cron job --- galaxy/values.yaml | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 519e9a39..cfe2bd4e 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -305,6 +305,50 @@ cronJobs: - "rm" - "{}" - ";" + #- An example cron job that showcases all available features. + example: + #- Disable the job by scheduling it for a date that never occurs, I.E. Feb 30th + #- The job can still be triggered manually. + schedule: "0 0 30 2 *" + #- Include the set of default environment variables. See galaxy.podEnvVars + #- in the Helm chart's _helpers.tpl for the variables that will be defined. + defaultEnv: true + #- Define extra environment variables that will be available to the job + extraEnv: + - name: LOGFILE + value: /galaxy/server/database/example.log + #- Run the job as root (uid 0) + securityContext: + runAsUser: 0 + #- Specify an alternate Docker image for the CronJob container + image: + repository: ksuderman/galaxy-maintenance + tag: "0.7" + #- The command to be run + command: + - /usr/local/bin/example.sh + #- Command line arguments to be passed to the command, one per line. + args: + - "--option" + - "value" + #- Define extra files that will be mounted into the image. In this case we + #- mount a simple Bash script that will write the current environment + #- variables to persistent storage. + extraFileMappings: + #- Path were the file will be mounted + /usr/local/bin/example.sh: + #- Default permission on the file. In this case 'rwxr-xr-x' + mode: "0755" + #- Run the contents through the Helm `tpl` command + tpl: true + #- The contents of the file to be mounted. Can contain Helm template values + #- if `tpl` is set to true. + content: |- + #!/usr/bin/bash + echo {{ .Release.Name }} >> $LOGFILE + echo "$@" >> $LOGFILE + env >> $LOGFILE + ingress: #- Should ingress be enabled. Defaults to `true` enabled: true From 07fdbd4ba46e4e76172e8bae8a06174affc82683 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Wed, 13 Mar 2024 17:33:41 -0400 Subject: [PATCH 26/29] Additional documentation for cron jobs --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 77480d43..4d5aebbc 100644 --- a/README.md +++ b/README.md @@ -421,7 +421,7 @@ by setting the desired values of the `webHandlers.replicaCount`, ## Cron jobs -Two Cron jobs are defined to clean up Galaxy's database and the `tmp` directory. By default these +Two Cron jobs are defined by default. One to clean up Galaxy's database and one to clean up the `tmp` directory. By default, these jobs run at 02:05 (the database maintenance script) and 02:15 (`tmp` directyory cleanup). Users can change the times the cron jobs are run by changing the `schedule` field in the `values.yaml` file: @@ -430,7 +430,11 @@ cronJobs: maintenance: schedule: "30 6 * * *" # Execute the cron job at 6:30 UTC ``` - +or by specifying the `schedule` on the command line when instaling Galaxy: +```bash +# Schedule the maintenance job to run at 06:30 on the first day of each month +helm install galaxy -n galaxy galaxy/galaxy --set cronJobs.maintenance.schedule="30 6 1 * *" +``` To disable a cron job after Galaxy has been deployed simply set the schedule to a date that can never occur such as midnight on Februrary 30th: @@ -439,6 +443,55 @@ can never occur such as midnight on Februrary 30th: helm upgrade galaxy -n galaxy galaxy/galaxy --reuse-values --set cronJobs.maintenance.schedule="0 0 30 2 *" ``` +### Run a CronJob manually + +Cron jobs can be invoked manually with tools such as [OpenLens](https://github.com/MuhammedKalkan/OpenLens) +or from the command line with `kubectl` +```bash +kubectl create job --namespace --from cronjob/galaxy-cron-maintenance +``` +This will run the cron job regardless of the `schedule` that has been set. + +**Note:** the name of the cron job will be `{{ .Release.Name }}-cron-` where the `` +is the name (key) used in the `values.yaml` file. + +### CronJob configuration + +The following fields can be specified when defining cron jobs. + +| Name | Definition | Required | +|---|-------------------------------------------------------------------------------------------------------------------------------------------|----------| +| schedule | When the job will be run. Use tools such as [crontab.guru](https://crontab.guru) for assistance determining the proper schedule string | **Yes** | +| defaultEnv | `true` or `false`. See the `galaxy.podEnvVars` macro in `_helpers.tpl` for the list of variables that will be defined. Default is `false` | No | +| extraEnv | Define extra environment variables that will be available to the job | No | +| securityContext | Specifies a `securityContext` for the job. Typically used to set `runAsUser` | No | +| image | Specify the Docker container used to run the job | No | +| command | The command to run | **Yes** | +| args | Any command line arguments that should be passed to the `command` | No | +| extraFileMappings | Allow arbitrary files to be mounted from config maps | No | + +### Notes + +If specifying the Docker `image` both the `resposity` and `tag` MUST be specified. +```yaml + image: + repository: quay.io/my-organization/my-image + tag: "1.0" +``` + +The `extraFileMappings` block is similar to the global `extraFileMappings` except the file will only be mounted for that cron job. +The following fields can be specified for each file. + +| Name | Definition | Required | +|---|---|----------| +| mode | The file mode (permissions) assigned to the file | No | +| tpl | If set to `true` the file contents will be run through Helm's templating engine. Defaults to `false` | No | +| content | The contents of the file | **Yes** | + + +See the `example` cron job included in the `values.yaml` file for a full example. + + ## Funding - _Version 3+_: Galaxy Project, Genomics Virtual Laboratory (GVL) From b4cbdf6e76347f5f819ad197efa6cf3f58506f89 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Wed, 13 Mar 2024 21:56:49 -0400 Subject: [PATCH 27/29] Comment out the example cron job. Examples should not add arbitrary code to clusters. --- galaxy/values.yaml | 86 +++++++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index cfe2bd4e..fdc57bab 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -305,49 +305,49 @@ cronJobs: - "rm" - "{}" - ";" - #- An example cron job that showcases all available features. - example: - #- Disable the job by scheduling it for a date that never occurs, I.E. Feb 30th - #- The job can still be triggered manually. - schedule: "0 0 30 2 *" - #- Include the set of default environment variables. See galaxy.podEnvVars - #- in the Helm chart's _helpers.tpl for the variables that will be defined. - defaultEnv: true - #- Define extra environment variables that will be available to the job - extraEnv: - - name: LOGFILE - value: /galaxy/server/database/example.log - #- Run the job as root (uid 0) - securityContext: - runAsUser: 0 - #- Specify an alternate Docker image for the CronJob container - image: - repository: ksuderman/galaxy-maintenance - tag: "0.7" - #- The command to be run - command: - - /usr/local/bin/example.sh - #- Command line arguments to be passed to the command, one per line. - args: - - "--option" - - "value" - #- Define extra files that will be mounted into the image. In this case we - #- mount a simple Bash script that will write the current environment - #- variables to persistent storage. - extraFileMappings: - #- Path were the file will be mounted - /usr/local/bin/example.sh: - #- Default permission on the file. In this case 'rwxr-xr-x' - mode: "0755" - #- Run the contents through the Helm `tpl` command - tpl: true - #- The contents of the file to be mounted. Can contain Helm template values - #- if `tpl` is set to true. - content: |- - #!/usr/bin/bash - echo {{ .Release.Name }} >> $LOGFILE - echo "$@" >> $LOGFILE - env >> $LOGFILE +# #- An example cron job that showcases all available features. +# example: +# #- Disable the job by scheduling it for a date that never occurs, I.E. Feb 30th +# #- The job can still be triggered manually. +# schedule: "0 0 30 2 *" +# #- Include the set of default environment variables. See galaxy.podEnvVars +# #- in the Helm chart's _helpers.tpl for the variables that will be defined. +# defaultEnv: true +# #- Define extra environment variables that will be available to the job +# extraEnv: +# - name: LOGFILE +# value: /galaxy/server/database/example.log +# #- Run the job as root (uid 0) +# securityContext: +# runAsUser: 0 +# #- Specify an alternate Docker image for the CronJob container +# image: +# repository: ksuderman/galaxy-maintenance +# tag: "0.7" +# #- The command to be run +# command: +# - /usr/local/bin/example.sh +# #- Command line arguments to be passed to the command, one per line. +# args: +# - "--option" +# - "value" +# #- Define extra files that will be mounted into the image. In this case we +# #- mount a simple Bash script that will write the current environment +# #- variables to persistent storage. +# extraFileMappings: +# #- Path were the file will be mounted +# /usr/local/bin/example.sh: +# #- Default permission on the file. In this case 'rwxr-xr-x' +# mode: "0755" +# #- Run the contents through the Helm `tpl` command +# tpl: true +# #- The contents of the file to be mounted. Can contain Helm template values +# #- if `tpl` is set to true. +# content: |- +# #!/usr/bin/bash +# echo {{ .Release.Name }} >> $LOGFILE +# echo "$@" >> $LOGFILE +# env >> $LOGFILE ingress: #- Should ingress be enabled. Defaults to `true` From db82ef00fc90b406e326f6848ea4c2940830cbf4 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Sun, 17 Mar 2024 15:00:24 -0400 Subject: [PATCH 28/29] Run extraEnv values through the template engine --- galaxy/templates/cronjob-maintenance.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index c6366435..25a069b6 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -37,7 +37,10 @@ spec: {{- include "galaxy.podEnvVars" $}} {{- end }} {{- if $cronjob.extraEnv }} - {{- $cronjob.extraEnv | toYaml | nindent 12 }} + {{- range $env := $cronjob.extraEnv }} + - name: {{ $env.name }} + value: {{ tpl $env.value $ | quote }} + {{- end }} {{- end }} {{- end }} command: From 14f190de57584bc19a64bbfaf6eb772544354bef Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Wed, 17 Apr 2024 20:03:35 -0400 Subject: [PATCH 29/29] Add newline to the end of the file. --- galaxy/templates/cronjob-maintenance.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/templates/cronjob-maintenance.yaml b/galaxy/templates/cronjob-maintenance.yaml index 25a069b6..0ea57a5e 100644 --- a/galaxy/templates/cronjob-maintenance.yaml +++ b/galaxy/templates/cronjob-maintenance.yaml @@ -109,4 +109,4 @@ data: {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }}