From 62b4f0f90fac07fe2b3a3c45fbe6c96814e8145d Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Mon, 7 Jun 2021 14:53:35 -0400 Subject: [PATCH 001/134] Documentation of extraFileMappings --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 30513d89..76150767 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,24 @@ jobHandlers: failureThreshhold: 3 ``` +## Extra File Mappings + +The `extraFileMappings` field can be used to inject files to arbitrary paths in the `job`, `web`, or `workflow` handlers. The contents of the file can be specified directly in the `values.yml` file with the `content` attribute, or by specifying the path to the file to be injected with the `path` attribute. + +```yaml +extraFileMappings: + /galaxy/server/static/welcome.html: + applyToWeb: true + applyToJob: false + applyToWorkflow: false + content: | + + ... + /galaxy/server/config/object_store_conf.xml: + applyToWeb: true + path: files/config/object_store_conf.xml +``` + ## Setting parameters on the command line Specify each parameter using the `--set key=value[,key=value]` argument to From 6ff43391d6179a81d4639ab36918a4e536518e8a Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Mon, 7 Jun 2021 16:07:47 -0400 Subject: [PATCH 002/134] Added note about external files --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 76150767..35a8f4a5 100644 --- a/README.md +++ b/README.md @@ -276,6 +276,12 @@ extraFileMappings: path: files/config/object_store_conf.xml ``` +**NOTE** for security reasons Helm will not load files from outside the chart so the `path` must be a relative path to location inside the chart directory. This will change when [helm#3276](https://github.com/helm/helm/issues/3276) is resolved. In the interim files can be loaded from external locations by: + +1. Creating a symbolic link in the chart directory to the external file, or +2. using `--set-file` to specify the contents of the file. E.g: + `helm upgrade --install galaxy cloudve/galaxy -n galaxy --set-file extraFileMappings."/galaxy/server/static/welcome\.html".content=/home/user/data/welcome.html` + ## Setting parameters on the command line Specify each parameter using the `--set key=value[,key=value]` argument to From 50e2e6f4bee6a800fa06c21958ffa05ae039faff Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Thu, 8 Jul 2021 12:36:45 -0400 Subject: [PATCH 003/134] Add GTN webhook --- galaxy/templates/configmap-nginx.yaml | 4 +++- galaxy/values.yaml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/galaxy/templates/configmap-nginx.yaml b/galaxy/templates/configmap-nginx.yaml index 39d3996b..5a62ce03 100644 --- a/galaxy/templates/configmap-nginx.yaml +++ b/galaxy/templates/configmap-nginx.yaml @@ -61,7 +61,9 @@ data: add_header X-Frame-Options SAMEORIGIN; add_header X-Content-Type-Options nosniff; } - + location /training-material/ { + proxy_pass https://training.galaxyproject.org/training-material/; + } # end server } # end http diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 55e71f97..909103ea 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -194,6 +194,7 @@ ingress: - host: ~ paths: - path: "/galaxy" + - path: "/training-material" tls: [] # - secretName: chart-example-tls # hosts: From 4e27f15ed2ec1c526cccc66c6010e0804688e6c0 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Fri, 13 Aug 2021 19:16:32 -0400 Subject: [PATCH 004/134] Changes for not mounting entire NFS to each job friends with https://github.com/galaxyproject/galaxy/pull/12345 --- galaxy/values.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 8cfd4e97..02eb3105 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -273,11 +273,20 @@ configs: k8s: load: galaxy.jobs.runners.kubernetes:KubernetesJobRunner k8s_use_service_account: true - k8s_persistent_volume_claims: |- + k8s_data_volume_claim: |- {{ template "galaxy.pvcname" . -}}:{{ .Values.persistence.mountPath -}} + k8s_persistent_volume_claims: |- + {{ template "galaxy.pvcname" . -}}/cache:{{ .Values.persistence.mountPath -}}/cache, + {{- template "galaxy.pvcname" . -}}/config:{{ .Values.persistence.mountPath -}}/config, + {{- template "galaxy.pvcname" . -}}/deps:{{ .Values.persistence.mountPath -}}/deps, + {{- template "galaxy.pvcname" . -}}/object_store_cache:{{ .Values.persistence.mountPath -}}/object_store_cache, + {{- template "galaxy.pvcname" . -}}/tmp:{{ .Values.persistence.mountPath -}}/tmp, + {{- template "galaxy.pvcname" . -}}/tool-data:{{ .Values.persistence.mountPath -}}/tool-data, + {{- template "galaxy.pvcname" . -}}/tools:{{ .Values.persistence.mountPath -}}/tools, + {{- template "galaxy.pvcname" . -}}/tool_search_index:{{ .Values.persistence.mountPath -}}/tool_search_index {{- if .Values.cvmfs.enabled -}} {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims -}} - ,{{- template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc:{{ $entry.mountPath -}} + ,{{ template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc:{{ $entry.mountPath -}} {{- end -}} {{- end -}} {{- if .Values.initJob.downloadToolConfs.enabled -}} From bb53d371a72ba298a64f16d17328b26584608388 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Mon, 30 Aug 2021 15:47:42 -0400 Subject: [PATCH 005/134] Separate data and working dir pvcs --- galaxy/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 02eb3105..fb569a81 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -275,6 +275,8 @@ configs: k8s_use_service_account: true k8s_data_volume_claim: |- {{ template "galaxy.pvcname" . -}}:{{ .Values.persistence.mountPath -}} + k8s_working_volume_claim: |- + {{ template "galaxy.pvcname" . -}}:{{ .Values.persistence.mountPath -}} k8s_persistent_volume_claims: |- {{ template "galaxy.pvcname" . -}}/cache:{{ .Values.persistence.mountPath -}}/cache, {{- template "galaxy.pvcname" . -}}/config:{{ .Values.persistence.mountPath -}}/config, From 380430dd51ada4def02cafcdc7733d5bccc5adce Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Mon, 20 Dec 2021 19:29:32 +0530 Subject: [PATCH 006/134] Removed unnecessary mounts and made inputs read only --- galaxy/values.yaml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index fb569a81..fe0ef196 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -274,18 +274,14 @@ configs: load: galaxy.jobs.runners.kubernetes:KubernetesJobRunner k8s_use_service_account: true k8s_data_volume_claim: |- - {{ template "galaxy.pvcname" . -}}:{{ .Values.persistence.mountPath -}} + {{ template "galaxy.pvcname" . -}}:{{ .Values.persistence.mountPath -}}:r k8s_working_volume_claim: |- {{ template "galaxy.pvcname" . -}}:{{ .Values.persistence.mountPath -}} k8s_persistent_volume_claims: |- - {{ template "galaxy.pvcname" . -}}/cache:{{ .Values.persistence.mountPath -}}/cache, - {{- template "galaxy.pvcname" . -}}/config:{{ .Values.persistence.mountPath -}}/config, - {{- template "galaxy.pvcname" . -}}/deps:{{ .Values.persistence.mountPath -}}/deps, - {{- template "galaxy.pvcname" . -}}/object_store_cache:{{ .Values.persistence.mountPath -}}/object_store_cache, - {{- template "galaxy.pvcname" . -}}/tmp:{{ .Values.persistence.mountPath -}}/tmp, - {{- template "galaxy.pvcname" . -}}/tool-data:{{ .Values.persistence.mountPath -}}/tool-data, - {{- template "galaxy.pvcname" . -}}/tools:{{ .Values.persistence.mountPath -}}/tools, - {{- template "galaxy.pvcname" . -}}/tool_search_index:{{ .Values.persistence.mountPath -}}/tool_search_index + {{ template "galaxy.pvcname" . -}}/config:{{ .Values.persistence.mountPath -}}/config:r, + {{- template "galaxy.pvcname" . -}}/tmp:{{ .Values.persistence.mountPath -}}/tmp:rw, + {{- template "galaxy.pvcname" . -}}/tool-data:{{ .Values.persistence.mountPath -}}/tool-data:r, + {{- template "galaxy.pvcname" . -}}/tools:{{ .Values.persistence.mountPath -}}/tools:r {{- if .Values.cvmfs.enabled -}} {{- range $key, $entry := .Values.cvmfs.galaxyPersistentVolumeClaims -}} ,{{ template "galaxy.fullname" $ -}}-cvmfs-gxy-{{ $key }}-pvc:{{ $entry.mountPath -}} From 5b2d170ddffd5b9f99f76a3ffa79de662456e8d5 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Mon, 19 Dec 2022 17:21:27 +0000 Subject: [PATCH 007/134] Required extra steps to bring down deployment --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 1c155f0b..74912bc4 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,21 @@ To uninstall/delete the `my-galaxy` deployment, run: helm delete my-galaxy ``` +if you see that some RabbitMQ and Postgres elements remain after some 10 minutes or more, you should be able to issue: + +``` +kubectl delete RabbitmqCluster/my-galaxy-rabbitmq-server +kubectl delete statefulset/galaxy-my-galaxy-postgres +``` + +it might be needed to remove the finalizer from the RabbitmqCluster if the above doesn't seem to get rid of RabbitmqCluster, through a + +``` +kubectl edit RabbitmqCluster/my-galaxy-rabbitmq-server +``` + +Consider as well that if you set persistence to be enabled, Postgres and Galaxy will leave their PVCs behind, which you might want to delete or not depending on your use case. + ## Configuration The following table lists the configurable parameters of the Galaxy chart. The From 11bd8dfb82152efcb121d1216541d220ec56ce7d Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Mon, 19 Dec 2022 17:44:48 +0000 Subject: [PATCH 008/134] Add finalizer removal --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 74912bc4..5b185378 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,22 @@ it might be needed to remove the finalizer from the RabbitmqCluster if the above kubectl edit RabbitmqCluster/my-galaxy-rabbitmq-server ``` +and remove the finalizer in: + +``` +apiVersion: rabbitmq.com/v1beta1 +kind: RabbitmqCluster +metadata: + annotations: + meta.helm.sh/release-name: my-galaxy + meta.helm.sh/release-namespace: default + creationTimestamp: "2022-12-19T16:54:33Z" + deletionGracePeriodSeconds: 0 + deletionTimestamp: "2022-12-19T17:41:40Z" + finalizers: + - deletion.finalizers.rabbitmqclusters.rabbitmq.com +``` + Consider as well that if you set persistence to be enabled, Postgres and Galaxy will leave their PVCs behind, which you might want to delete or not depending on your use case. ## Configuration From 2f3d71293f3656d353c8bce7e6a5062926de0793 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Mon, 19 Dec 2022 17:47:49 +0000 Subject: [PATCH 009/134] Add removal of postgres secret --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b185378..cc9bc259 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ it might be needed to remove the finalizer from the RabbitmqCluster if the above kubectl edit RabbitmqCluster/my-galaxy-rabbitmq-server ``` -and remove the finalizer in: +remove the finalizer in: ``` apiVersion: rabbitmq.com/v1beta1 @@ -124,6 +124,12 @@ metadata: - deletion.finalizers.rabbitmqclusters.rabbitmq.com ``` +and remove the postgres secret: + +``` +kubectl delete secrets/standby.galaxy-galaxy-dev-postgres.credentials.postgresql.acid.zalan.do +``` + Consider as well that if you set persistence to be enabled, Postgres and Galaxy will leave their PVCs behind, which you might want to delete or not depending on your use case. ## Configuration From fd2beea011a95ee0292e7f68846dc710a351384f Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Mon, 19 Dec 2022 17:48:26 +0000 Subject: [PATCH 010/134] Fix deploy name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cc9bc259..636b9c65 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ metadata: and remove the postgres secret: ``` -kubectl delete secrets/standby.galaxy-galaxy-dev-postgres.credentials.postgresql.acid.zalan.do +kubectl delete secrets/standby.galaxy-my-galaxy-postgres.credentials.postgresql.acid.zalan.do ``` Consider as well that if you set persistence to be enabled, Postgres and Galaxy will leave their PVCs behind, which you might want to delete or not depending on your use case. From aed55c57794aecb537a3aa4364a96ef22f3e0016 Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Wed, 8 Feb 2023 14:00:59 -0500 Subject: [PATCH 011/134] Add ttl for jobs after finish; disable concurrent job limits --- galaxy/values.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 65fc2b24..a46130d1 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -359,6 +359,7 @@ configs: k8s_supplemental_group_id: "101" k8s_pull_policy: IfNotPresent k8s_cleanup_job: onsuccess + k8s_job_ttl_secs_after_finished: 90 k8s_pod_priority_class: >- {{ if .Values.jobs.priorityClass.enabled -}} {{- include "galaxy.fullname" . }}-job-priority @@ -383,11 +384,11 @@ configs: tpv_config_files: - https://raw.githubusercontent.com/galaxyproject/tpv-shared-database/main/tools.yml - lib/galaxy/jobs/rules/tpv_rules_local.yml - limits: - - type: registered_user_concurrent_jobs - value: 5 - - type: anonymous_user_concurrent_jobs - value: 2 +# limits: +# - type: registered_user_concurrent_jobs +# value: 5 +# - type: anonymous_user_concurrent_jobs +# value: 2 galaxy.yml: galaxy: galaxy_url_prefix: "{{ .Values.ingress.path }}" From f6bc105fd4857338d2c50f4e5f6309a4726b62f7 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Sun, 12 Feb 2023 15:09:13 -0500 Subject: [PATCH 012/134] Parameterize the name of the cvmfs storage class used. --- galaxy/templates/pvc-refdata.yaml | 2 +- galaxy/values.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/galaxy/templates/pvc-refdata.yaml b/galaxy/templates/pvc-refdata.yaml index 13765e33..1589769c 100644 --- a/galaxy/templates/pvc-refdata.yaml +++ b/galaxy/templates/pvc-refdata.yaml @@ -15,7 +15,7 @@ spec: volumeName: {{ include "galaxy.fullname" $ }}-refdata-gxy-pv {{- end }} {{- if eq $.Values.refdata.type "cvmfs" }} - storageClassName: {{ $.Release.Name }}-cvmfs + storageClassName: {{ tpl .Values.cvmfs.storageClassName }} {{- end }} --- {{- end }} diff --git a/galaxy/values.yaml b/galaxy/values.yaml index a46130d1..351b78c5 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -308,6 +308,7 @@ refdata: cvmfs: deploy: true + storageClassName: {{ $.Release.Name }}-cvmfs s3csi: deploy: false From 7d8802f91d23d61b5ff374d3f6ea6b2a1a86f148 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Sun, 12 Feb 2023 22:23:42 -0500 Subject: [PATCH 013/134] Fix typos --- galaxy/templates/pvc-refdata.yaml | 2 +- galaxy/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/templates/pvc-refdata.yaml b/galaxy/templates/pvc-refdata.yaml index 1589769c..75cf0715 100644 --- a/galaxy/templates/pvc-refdata.yaml +++ b/galaxy/templates/pvc-refdata.yaml @@ -15,7 +15,7 @@ spec: volumeName: {{ include "galaxy.fullname" $ }}-refdata-gxy-pv {{- end }} {{- if eq $.Values.refdata.type "cvmfs" }} - storageClassName: {{ tpl .Values.cvmfs.storageClassName }} + storageClassName: {{ tpl .Values.cvmfs.storageClassName . }} {{- end }} --- {{- end }} diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 351b78c5..e6028c20 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -308,7 +308,7 @@ refdata: cvmfs: deploy: true - storageClassName: {{ $.Release.Name }}-cvmfs + storageClassName: "{{ $.Release.Name }}-cvmfs" s3csi: deploy: false From ac7851263e23cd8f387b8d55bb270dac906f79f5 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Wed, 15 Feb 2023 17:34:40 -0500 Subject: [PATCH 014/134] Add a flag to enable the hook and parameterize the url --- galaxy/templates/configmap-nginx.yaml | 4 +++- galaxy/values.yaml | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/galaxy/templates/configmap-nginx.yaml b/galaxy/templates/configmap-nginx.yaml index 5a62ce03..686aa9af 100644 --- a/galaxy/templates/configmap-nginx.yaml +++ b/galaxy/templates/configmap-nginx.yaml @@ -61,9 +61,11 @@ data: add_header X-Frame-Options SAMEORIGIN; add_header X-Content-Type-Options nosniff; } +{{- if .Values.trainingHook.enabled }} location /training-material/ { - proxy_pass https://training.galaxyproject.org/training-material/; + proxy_pass {{ .Values.trainingHook.url }}; } +{{- end }} # end server } # end http diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 909103ea..8640d2a9 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -12,6 +12,10 @@ image: imagePullSecrets: [] +trainingHook: + enabled: true + url: https://training.galaxyproject.org/training-material/ + service: type: ClusterIP port: 8000 From 62557652c3c0cb3a2af4cf84ba5f4a697f5006a1 Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Tue, 21 Feb 2023 14:29:32 -0500 Subject: [PATCH 015/134] Disable docker for jobs --- galaxy/values.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 65fc2b24..a856e31b 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -374,7 +374,7 @@ configs: environments: tpv_dispatcher: container_monitor: false - docker_enabled: true + docker_enabled: false runner: dynamic type: python function: map_tool_to_destination @@ -506,7 +506,6 @@ jobs: default: params: container_monitor: false - docker_enabled: "true" docker_default_container_id: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" tmp_dir: "true" scheduling: From 4930d3fab12a2b716f8428b73315e2c416c1946e Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Thu, 23 Feb 2023 11:27:35 -0500 Subject: [PATCH 016/134] Docker does need to be generally enabled --- galaxy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index a856e31b..4aae6065 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -374,7 +374,7 @@ configs: environments: tpv_dispatcher: container_monitor: false - docker_enabled: false + docker_enabled: true runner: dynamic type: python function: map_tool_to_destination From 753f05dd0901abe4a7f0cfd5a01f217ddf87890a Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Mon, 6 Mar 2023 10:19:04 -0500 Subject: [PATCH 017/134] Upgrade Galaxy version to 23.0 --- galaxy/Chart.yaml | 2 +- galaxy/values.yaml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/galaxy/Chart.yaml b/galaxy/Chart.yaml index 1ce84cba..f50b3b73 100644 --- a/galaxy/Chart.yaml +++ b/galaxy/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: galaxy type: application version: 5.5.0 -appVersion: "22.05" +appVersion: "23.0" description: Chart for Galaxy, an open, web-based platform for accessible, reproducible, and transparent computational biomedical research. icon: https://galaxyproject.org/images/galaxy-logos/galaxy_project_logo_square.png dependencies: diff --git a/galaxy/values.yaml b/galaxy/values.yaml index a46130d1..4e6cbf06 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -6,7 +6,7 @@ fullnameOverride: "" image: repository: quay.io/galaxyproject/galaxy-min - tag: 22.05 + tag: "23.0" # Value must be quoted pullPolicy: IfNotPresent imagePullSecrets: [] @@ -399,7 +399,6 @@ configs: interactivetools_shorten_url: true interactivetools_base_path: "{{$host := index .Values.ingress.hosts 0}}{{$path := index $host.paths 0}}{{$path.path}}" id_secret: - logo_src: 'static/favicon.png' mulled_resolution_cache_lock_dir: "/galaxy/server/local/mulled_cache_lock" database_connection: postgresql://unused:because@overridden_by_envvar integrated_tool_panel_config: "/galaxy/server/config/mutable/integrated_tool_panel.xml" From 1ff91421cbf86c130f1a3766789792ca9b6df0c0 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 6 Mar 2023 21:13:32 +0000 Subject: [PATCH 018/134] Automatic Version Bumping from 5.5.0 to 5.5.1 --- galaxy/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Chart.yaml b/galaxy/Chart.yaml index 1ce84cba..bb7e1d71 100644 --- a/galaxy/Chart.yaml +++ b/galaxy/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: galaxy type: application -version: 5.5.0 +version: 5.5.1 appVersion: "22.05" description: Chart for Galaxy, an open, web-based platform for accessible, reproducible, and transparent computational biomedical research. icon: https://galaxyproject.org/images/galaxy-logos/galaxy_project_logo_square.png From 509611720cdf1139435d1b0545636169c41d0dea Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Wed, 25 Jan 2023 17:29:38 -0500 Subject: [PATCH 019/134] Use pod priority class for jobs --- galaxy/templates/_helpers.tpl | 2 +- galaxy/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/templates/_helpers.tpl b/galaxy/templates/_helpers.tpl index 463547f6..c91e7d7c 100644 --- a/galaxy/templates/_helpers.tpl +++ b/galaxy/templates/_helpers.tpl @@ -262,7 +262,7 @@ Define pod priority class */}} {{- define "galaxy.pod-priority-class" -}} {{- if .Values.jobs.priorityClass.existingClass -}} -{{- printf "%s" .Values.jobs.priorityClass.existingClass -}} +{{- tpl .Values.jobs.priorityClass.existingClass . -}} {{- else -}} {{- printf "%s-job-priority" (include "galaxy.fullname" .) -}} {{- end -}} diff --git a/galaxy/values.yaml b/galaxy/values.yaml index e6028c20..e9e810b2 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -363,7 +363,7 @@ configs: k8s_job_ttl_secs_after_finished: 90 k8s_pod_priority_class: >- {{ if .Values.jobs.priorityClass.enabled -}} - {{- include "galaxy.fullname" . }}-job-priority + {{- include "galaxy.pod-priority-class" . }} {{- end }} k8s_interactivetools_use_ssl: true k8s_interactivetools_ingress_annotations: | From dfe980c7f0e28f5fdadf136f9538f49fa76aa33a Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 7 Mar 2023 14:23:11 +0000 Subject: [PATCH 020/134] Automatic Version Bumping from 5.5.1 to 5.6.0 --- galaxy/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Chart.yaml b/galaxy/Chart.yaml index 760a4aa3..28b22692 100644 --- a/galaxy/Chart.yaml +++ b/galaxy/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: galaxy type: application -version: 5.5.1 +version: 5.6.0 appVersion: "23.0" description: Chart for Galaxy, an open, web-based platform for accessible, reproducible, and transparent computational biomedical research. icon: https://galaxyproject.org/images/galaxy-logos/galaxy_project_logo_square.png From bd47a72626d5efb872eb8dfb6f1f7e621f05b9e0 Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Tue, 7 Mar 2023 10:07:50 -0500 Subject: [PATCH 021/134] Move docker_enabled to a destination --- galaxy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 4aae6065..a0f59777 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -374,7 +374,6 @@ configs: environments: tpv_dispatcher: container_monitor: false - docker_enabled: true runner: dynamic type: python function: map_tool_to_destination @@ -537,6 +536,7 @@ jobs: k8s: runner: k8s params: + docker_enabled: "true" requests_cpu: "{cores}" requests_memory: "{mem}Gi" limits_cpu: "{cores}" From d5d43d19c38397963e4f659bd9e486ca737ecbfa Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 7 Mar 2023 15:46:45 +0000 Subject: [PATCH 022/134] Automatic Version Bumping from 5.6.0 to 5.7.0 --- galaxy/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Chart.yaml b/galaxy/Chart.yaml index 28b22692..152cf8a6 100644 --- a/galaxy/Chart.yaml +++ b/galaxy/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: galaxy type: application -version: 5.6.0 +version: 5.7.0 appVersion: "23.0" description: Chart for Galaxy, an open, web-based platform for accessible, reproducible, and transparent computational biomedical research. icon: https://galaxyproject.org/images/galaxy-logos/galaxy_project_logo_square.png From b5d26e57ac4db336ccd1557b2bcce5d044e184df Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Thu, 9 Mar 2023 09:42:08 -0500 Subject: [PATCH 023/134] Use S3 storage class name as defined in the values file. --- galaxy/templates/pvc-refdata.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy/templates/pvc-refdata.yaml b/galaxy/templates/pvc-refdata.yaml index 75cf0715..a537eb03 100644 --- a/galaxy/templates/pvc-refdata.yaml +++ b/galaxy/templates/pvc-refdata.yaml @@ -13,6 +13,7 @@ spec: storage: {{ .Values.refdata.pvc.size }} {{- if eq $.Values.refdata.type "s3csi" }} volumeName: {{ include "galaxy.fullname" $ }}-refdata-gxy-pv + storageClassName: {{ .Values.s3csi.storageClass.name }} {{- end }} {{- if eq $.Values.refdata.type "cvmfs" }} storageClassName: {{ tpl .Values.cvmfs.storageClassName . }} From 10b3426ec102af76df82e1d2a8426f05b0b146a2 Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Thu, 16 Mar 2023 16:29:33 -0400 Subject: [PATCH 024/134] Add a post-install hook to restart cvmfs csi pods --- galaxy/templates/configmap-cvmfs-fix.yaml | 16 +++++++++++ galaxy/templates/hook-cvmfs-fix.yaml | 35 +++++++++++++++++++++++ galaxy/templates/rbac-job.yaml | 2 +- 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 galaxy/templates/configmap-cvmfs-fix.yaml create mode 100644 galaxy/templates/hook-cvmfs-fix.yaml diff --git a/galaxy/templates/configmap-cvmfs-fix.yaml b/galaxy/templates/configmap-cvmfs-fix.yaml new file mode 100644 index 00000000..48f28cdd --- /dev/null +++ b/galaxy/templates/configmap-cvmfs-fix.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-configmap-cvmfs-fix + labels: + {{- include "galaxy.labels" . | nindent 4 }} +data: + cvmfs-fix.sh: | + sleep 10; + while [ "$(kubectl get pods -n {{ .Release.Namespace }} -l 'app=cvmfscis' -l 'component=nodeplugin' -o custom-columns=STATUS:.status.phase --no-headers)" != "Running" ]; do + echo "Waiting on nodeplugin pod to enter 'Running' status."; + sleep 1; + done && \ + echo "Deleting nodeplugin pods..." + kubectl get pods -n {{ .Release.Namespace }} -l 'app=cvmfscis' -l 'component=nodeplugin' -o name | xargs kubectl -n {{ .Release.Namespace }} delete && \ + echo "Deleted nodeplugin pods." diff --git a/galaxy/templates/hook-cvmfs-fix.yaml b/galaxy/templates/hook-cvmfs-fix.yaml new file mode 100644 index 00000000..12ab3cbd --- /dev/null +++ b/galaxy/templates/hook-cvmfs-fix.yaml @@ -0,0 +1,35 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ .Release.Name }}-post-install-cvmfs-fix-job" + labels: + {{- include "galaxy.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install + "helm.sh/hook-weight": "10" + "helm.sh/hook-delete-policy": hook-succeeded +spec: + ttlSecondsAfterFinished: 120 + template: + metadata: + name: "{{ .Release.Name }}-post-install-cvmfs-fix-job" + labels: + {{- include "galaxy.labels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "galaxy.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.securityContext | nindent 8 }} + restartPolicy: Never + containers: + - name: post-install-kubectl + image: bitnami/kubectl + command: + - "sh" + - "/script/cvmfs-fix.sh" + volumeMounts: + - name: kubectl-script + mountPath: "/script" + volumes: + - name: kubectl-script + configMap: + name: "{{ .Release.Name }}-configmap-cvmfs-fix" diff --git a/galaxy/templates/rbac-job.yaml b/galaxy/templates/rbac-job.yaml index 7095a309..f1180267 100644 --- a/galaxy/templates/rbac-job.yaml +++ b/galaxy/templates/rbac-job.yaml @@ -9,7 +9,7 @@ metadata: rules: - apiGroups: [""] resources: ["pods", "pods/log"] - verbs: ["get", "list", "watch"] + verbs: ["get", "list", "watch", "delete"] - apiGroups: ["batch", "extensions"] resources: ["jobs"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] From e73842265263abc3a878b0b565e2776709df05d2 Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Thu, 16 Mar 2023 20:10:48 -0400 Subject: [PATCH 025/134] Fix multi-node setup and setup --- galaxy/templates/configmap-cvmfs-fix.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/galaxy/templates/configmap-cvmfs-fix.yaml b/galaxy/templates/configmap-cvmfs-fix.yaml index 48f28cdd..d5d4c129 100644 --- a/galaxy/templates/configmap-cvmfs-fix.yaml +++ b/galaxy/templates/configmap-cvmfs-fix.yaml @@ -7,10 +7,12 @@ metadata: data: cvmfs-fix.sh: | sleep 10; - while [ "$(kubectl get pods -n {{ .Release.Namespace }} -l 'app=cvmfscis' -l 'component=nodeplugin' -o custom-columns=STATUS:.status.phase --no-headers)" != "Running" ]; do - echo "Waiting on nodeplugin pod to enter 'Running' status."; + status=`kubectl get pods -n {{ .Release.Namespace }} -l 'app=cvmfscsi' -l 'component=nodeplugin' -o custom-columns=STATUS:.status.phase --no-headers | sort | uniq | tr -d '\n'` + while [ "$status" != "Running" ]; do + echo "Waiting on nodeplugin pod to enter 'Running' status. Currently '$status'."; sleep 1; + status=`kubectl get pods -n {{ .Release.Namespace }} -l 'app=cvmfscsi' -l 'component=nodeplugin' -o custom-columns=STATUS:.status.phase --no-headers | sort | uniq | tr -d '\n'` done && \ echo "Deleting nodeplugin pods..." - kubectl get pods -n {{ .Release.Namespace }} -l 'app=cvmfscis' -l 'component=nodeplugin' -o name | xargs kubectl -n {{ .Release.Namespace }} delete && \ + kubectl get pods -n {{ .Release.Namespace }} -l 'app=cvmfscsi' -l 'component=nodeplugin' -o name | xargs kubectl -n {{ .Release.Namespace }} delete && \ echo "Deleted nodeplugin pods." From 81a5fe2c831ec00ca95d6827fe3902db0bdd51ed Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 17 Mar 2023 14:12:32 +0000 Subject: [PATCH 026/134] Automatic Version Bumping from 5.7.0 to 5.7.1 --- galaxy/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Chart.yaml b/galaxy/Chart.yaml index 152cf8a6..f9bc667a 100644 --- a/galaxy/Chart.yaml +++ b/galaxy/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: galaxy type: application -version: 5.7.0 +version: 5.7.1 appVersion: "23.0" description: Chart for Galaxy, an open, web-based platform for accessible, reproducible, and transparent computational biomedical research. icon: https://galaxyproject.org/images/galaxy-logos/galaxy_project_logo_square.png From 005f73ee804991620d02575c69e5dc6ce066cbfe Mon Sep 17 00:00:00 2001 From: Yegor Tokmakov Date: Fri, 24 Mar 2023 15:15:31 +0100 Subject: [PATCH 027/134] amqp protocol and port as variables --- galaxy/templates/_helpers.tpl | 4 ++-- galaxy/values.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/galaxy/templates/_helpers.tpl b/galaxy/templates/_helpers.tpl index c91e7d7c..762601d5 100644 --- a/galaxy/templates/_helpers.tpl +++ b/galaxy/templates/_helpers.tpl @@ -171,7 +171,7 @@ Creates the bash command for the handlers to wait for init scripts {{- define "galaxy.init-container-wait-command" -}} until [ -f /galaxy/server/config/mutable/db_init_done_{{$.Release.Revision}} ]; do echo "waiting for DB initialization"; sleep 1; done; {{- if $.Values.rabbitmq.enabled }} -until timeout 1 bash -c "echo > /dev/tcp/{{ template "galaxy-rabbitmq.fullname" $ }}/5672"; do echo "waiting for rabbitmq service"; sleep 1; done; +until timeout 1 bash -c "echo > /dev/tcp/{{ template "galaxy-rabbitmq.fullname" $ }}/{{.Values.rabbitmq.port}}"; do echo "waiting for rabbitmq service"; sleep 1; done; {{- end }} until [ -f /galaxy/server/config/mutable/init_mounts_done_{{$.Release.Revision}} ]; do echo "waiting for copying onto NFS"; sleep 1; done; {{- if .Values.setupJob.downloadToolConfs.enabled }} @@ -252,7 +252,7 @@ Define pod env vars name: {{ tpl .Values.rabbitmq.existingSecret . }} key: password - name: GALAXY_CONFIG_OVERRIDE_AMQP_INTERNAL_CONNECTION - value: amqp://$(GALAXY_RABBITMQ_USERNAME):$(GALAXY_RABBITMQ_PASSWORD)@{{ template "galaxy-rabbitmq.fullname" . }} + value: {{ .Values.rabbitmq.protocol }}://$(GALAXY_RABBITMQ_USERNAME):$(GALAXY_RABBITMQ_PASSWORD)@{{ template "galaxy-rabbitmq.fullname" . }}:{{ .Values.rabbitmq.port }} {{- end }} {{- end -}} diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 7e3dd7c4..a181407c 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -651,6 +651,8 @@ rabbitmq: deploy: true existingCluster: existingSecret: '{{ include "galaxy-rabbitmq.fullname" . }}-default-user' + protocol: amqp + port: 5672 nameOverride: rabbitmq extraSpec: {} terminationGracePeriodSeconds: 90 From eefbc054604ffb76a391b3e1d628cb46e0dc3e6f Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Tue, 28 Mar 2023 17:40:40 -0400 Subject: [PATCH 028/134] First pass at a YamlDoc like utility to generate documentation from comments in the values.yaml file. --- VALUES.md | 49 ++++++++++++++++ galaxy/values.yaml | 84 +++++++++++++++++++++------- scripts/values_table.py | 10 +++- scripts/yamldoc.py | 120 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 241 insertions(+), 22 deletions(-) create mode 100644 VALUES.md create mode 100644 scripts/yamldoc.py diff --git a/VALUES.md b/VALUES.md new file mode 100644 index 00000000..0585741d --- /dev/null +++ b/VALUES.md @@ -0,0 +1,49 @@ +| Key | Description | +| nameOverride | Partial override of the `galaxy.fullname`. The `.Release.Name` will be prepended to generate the fullname. | +| fullnameOverride | Fully override the `galaxy.fullname` | +| image.repository | Repository containing the Galaxy image. | +| image.tag | Galaxy Docker image tag (generally corresponds to the desired Galaxy version) | +| image.pullPolicy | Galaxy image [pull policy](https://kubernetes.io/docs/concepts/configuration/overview/#container-images) | +| imagePullSecrets | Secrets used to [access a Galaxy image from a private repository](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) | +| trainingHook.enabled | Enable the GTN webhook to link references to tools in tutorials to the corresponding tool panel in Galaxy. | +| trainingHook.url | The training material server used to service the training-material webhook. | +| service.type | The Galaxy service type | +| service.port | The port Galaxy is listening to | +| service.nodePort | The external port exposed on each node | +| metrics.enabled | Enable the metrics server. Defaults to `false` | +| serviceAccount.create | Specifies whether a service account should be created | +| serviceAccount.annotations | Annotations to add to the service account | +| serviceAccount.name | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | +| rbac.enabled | Does the cluster use role based access control. | +| securityContext.fsGroup | Security context and file system group used by jobs. | +| persistence | Configure the PVC used by Galaxy for local storage. | +| persistence.enabled | Persistence is enabled by default | +| persistence.name | Name of the PVC to create | +| persistence.storageClass | StorageClass for the PVC. Must support `ReadWriteMany`. | +| persistence.existingClaim | The name of an existing PVC to use for persistence. | +| setupJob | tasks to perform once after installation | +| setupJob.createDatabase | create the database | +| setupJob.securityContext.runAsUser | the setup jobs will run as this user | +| setupJob.securityContext.runAsGroup | the `runAsUser` will belong to this group. | +| setupJob.securityContext.fsGroup | the filesystem group | +| setupJob.downloadToolConfs.archives.startup | A tar.gz publicly accessible archive containing AT LEAST conf files and XML tool wrappers. Meant to be enough for Galaxy handlers to startup | +| setupJob.downloadToolConfs.archives.running | A tar.gz publicly accessible archive containing AT LEAST confs, tool wrappers, and scripts excluding test data. Meant to be enough for Galaxy handlers to run jobs. | +| setupJob.downloadToolConfs.archives.full | A tar.gz publicly accessible archive containing the full `tools` directory, including each tool's test data. Meant to be enough to run automated tool-tests, fully mimicking CVMFS setup | +| extraInitContainers | Allow users to specify extra init containers | +| ingress.enabled | Should ingress be enabled. Defaults to `true` | +| ingress.ingressClassName | | +| resources.requests | We recommend updating these based on the usage levels of the server. | +| postgresql.deploy | Whether to deploy the postgresl operator. In general, we recommend installing the operator globally in production. | +| postgresql.existingDatabase | hostname and port of an existing database to use. | +| refdata | Configuration block for reference data | +| refdata.enabled | Whether or not to mount cloud-hosted Galaxy reference data and tools. | +| refdata.type | `s3fs` or `cvmfs`, determines the CSI to use for mounting reference data. `cvmfs` is the default and recommended for the time being. | +| cvmfs | Configuration block if `cvmfs` is used as `refdata.type` | +| cvmfs.deploy | Deploy the Galaxy-CVMFS-CSI Helm Chart. This is an optional dependency, and for production scenarios it should be deployed separately as a cluster-wide resource | +| s3csi | Configuration block if `s3csi` is used as the `refdata.type` | +| s3csi.deploy | Deploy the CSI-S3 Helm Chart. This is an optional dependency, and for production scenarios it should be deployed separately as a cluster-wide resource. | +| useSecretConfigs | When this flag is set to true, all configs will be set in secrets, when it is set to false, all configs will be set in configmaps | +| configs | All config files will be relative to `/galaxy/server/config/` directory | +| configs.galaxy\.yml | Galaxy configuration. See the [Galaxy documentation](https://docs.galaxyproject.org/en/master/admin/config.html) for more information. | +| jobs | Additional dynamic rules to map into the container. | +| jobs.priorityClass.enabled | Assign a [priorityClass](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) to the dispatched jobs. | diff --git a/galaxy/values.yaml b/galaxy/values.yaml index a181407c..0d5a994d 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -1,19 +1,34 @@ # Default values for Galaxy. # Declare variables to be passed into your templates. +#- Partial override of the `galaxy.fullname`. The `.Release.Name` will be prepended to generate the fullname. nameOverride: "" +#- Fully override the `galaxy.fullname` fullnameOverride: "" image: + #- Repository containing the Galaxy image. repository: quay.io/galaxyproject/galaxy-min + #- Galaxy Docker image tag (generally corresponds to the desired Galaxy version) tag: "23.0" # Value must be quoted + #- Galaxy image [pull policy](https://kubernetes.io/docs/concepts/configuration/overview/#container-images) pullPolicy: IfNotPresent +#- Secrets used to [access a Galaxy image from a private repository](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) imagePullSecrets: [] +trainingHook: + #- Enable the GTN webhook to link references to tools in tutorials to the corresponding tool panel in Galaxy. + enabled: true + #- The training material server used to service the training-material webhook. + url: https://training.galaxyproject.org/training-material/ + service: + #- The Galaxy service type type: ClusterIP + #- The port Galaxy is listening to port: 8000 + #- The external port exposed on each node nodePort: 30700 workflowHandlers: @@ -141,6 +156,7 @@ celeryBeat: timeoutSeconds: 10 metrics: + #- Enable the metrics server. Defaults to `false` enabled: false annotations: {} podAnnotations: {} @@ -151,25 +167,32 @@ metrics: pullPolicy: IfNotPresent serviceAccount: - # Specifies whether a service account should be created + #- Specifies whether a service account should be created create: true - # Annotations to add to the service account + #- Annotations to add to the service account annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template + #- The name of the service account to use. + #- If not set and create is true, a name is generated using the fullname template name: "" rbac: + #- Does the cluster use role based access control. enabled: true securityContext: + #- Security context and file system group used by jobs. fsGroup: 101 +#- Configure the PVC used by Galaxy for local storage. persistence: + #- Persistence is enabled by default enabled: true + #- Name of the PVC to create name: galaxy-pvc annotations: {} + #- StorageClass for the PVC. Must support `ReadWriteMany`. storageClass: "" + #- The name of an existing PVC to use for persistence. existingClaim: null accessMode: ReadWriteMany size: 5Gi @@ -184,11 +207,16 @@ extraVolumeMounts: [] # - name: shared-data # mountPath: /mnt/project/shared-data +#- tasks to perform once after installation setupJob: + #- create the database createDatabase: true securityContext: + #- the setup jobs will run as this user runAsUser: 101 + #- the `runAsUser` will belong to this group. runAsGroup: 101 + #- the filesystem group fsGroup: 101 ttlSecondsAfterFinished: 300 downloadToolConfs: @@ -198,18 +226,19 @@ setupJob: subPath: cvmfsclone # on galaxy-data volume (avoid config, tools, lib, etc...) mountPath: /cvmfs/cloud.galaxyproject.org archives: - # A tar.gz publicly accessible archive containing AT LEAST conf files and XML tool wrappers. - # Meant to be enough for Galaxy handlers to startup + #- A tar.gz publicly accessible archive containing AT LEAST conf files and XML tool wrappers. + #- Meant to be enough for Galaxy handlers to startup startup: https://storage.googleapis.com/cloud-cvmfs/startup.tar.gz - # A tar.gz publicly accessible archive containing AT LEAST confs, tool wrappers, and scripts - # excluding test data. - # Meant to be enough for Galaxy handlers to run jobs. + #- A tar.gz publicly accessible archive containing AT LEAST confs, tool wrappers, and scripts + #- excluding test data. + #- Meant to be enough for Galaxy handlers to run jobs. running: https://storage.googleapis.com/cloud-cvmfs/partial.tar.gz - # A tar.gz publicly accessible archive containing the full `tools` directory, - # including each tool's test data. - # Meant to be enough to run automated tool-tests, fully mimicking CVMFS setup + #- A tar.gz publicly accessible archive containing the full `tools` directory, + #- including each tool's test data. + #- Meant to be enough to run automated tool-tests, fully mimicking CVMFS setup full: https://storage.googleapis.com/cloud-cvmfs/contents.tar.gz +#- Allow users to specify extra init containers extraInitContainers: [] # - name: my-first-container # applyToJob: true @@ -239,7 +268,9 @@ extraEnv: [] # value: MY_VALUE ingress: + #- Should ingress be enabled. Defaults to `true` enabled: true + #- ingressClassName: nginx canary: enabled: true @@ -253,13 +284,14 @@ ingress: - host: ~ paths: - path: "/galaxy" + - path: "/training-material" tls: [] # - secretName: chart-example-tls # hosts: # - chart-example.local resources: - # We recommend updating these based on the usage levels of the server + #- We recommend updating these based on the usage levels of the server. requests: cpu: 100m memory: 1G @@ -277,10 +309,10 @@ affinity: {} postgresql: enabled: true - # Whether to deploy the postgresl operator. - # In general, we recommend installing the operator globally in production. + #- Whether to deploy the postgresl operator. + #- In general, we recommend installing the operator globally in production. deploy: true - # hostname and port of an existing database to use. + #- hostname and port of an existing database to use. existingDatabase: galaxyDatabaseUser: galaxydbuser galaxyConnectionParams: "?sslmode=require" @@ -300,17 +332,25 @@ postgresql: # matchLabels: # label-key: label-value +#- Configuration block for reference data refdata: + #- Whether or not to mount cloud-hosted Galaxy reference data and tools. enabled: true + #- `s3fs` or `cvmfs`, determines the CSI to use for mounting reference data. + #-`cvmfs` is the default and recommended for the time being. type: cvmfs pvc: size: 10Gi +#- Configuration block if `cvmfs` is used as `refdata.type` cvmfs: + #- Deploy the Galaxy-CVMFS-CSI Helm Chart. This is an optional dependency, and for production scenarios it should be deployed separately as a cluster-wide resource deploy: true storageClassName: "{{ $.Release.Name }}-cvmfs" +#- Configuration block if `s3csi` is used as the `refdata.type` s3csi: + #- Deploy the CSI-S3 Helm Chart. This is an optional dependency, and for production scenarios it should be deployed separately as a cluster-wide resource. deploy: false images: csi: cloudve/csi-s3:0.31.3 @@ -329,11 +369,11 @@ s3csi: usePrefix: true prefix: /galaxy/v1/data.galaxyproject.org -# When this flag is set to true, all configs will be set in secrets, -# when it is set to false, all configs will be set in configmaps +#- When this flag is set to true, all configs will be set in secrets, +#- when it is set to false, all configs will be set in configmaps useSecretConfigs: false -# All files will be relative to `/galaxy/server/config/` directory +#- All config files will be relative to `/galaxy/server/config/` directory configs: job_conf.yml: runners: @@ -389,6 +429,9 @@ configs: # value: 5 # - type: anonymous_user_concurrent_jobs # value: 2 + + #- Galaxy configuration. See the [Galaxy documentation](https://docs.galaxyproject.org/en/master/admin/config.html) + #- for more information. galaxy.yml: galaxy: galaxy_url_prefix: "{{ .Values.ingress.path }}" @@ -493,9 +536,10 @@ configs: tool_conf.xml: | {{- (.Files.Get "files/configs/tool_conf.xml") }} -# Additional dynamic rules to map into the container. +#- Additional dynamic rules to map into the container. jobs: priorityClass: + #- Assign a [priorityClass](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) to the dispatched jobs. enabled: true existingClass: "" rules: diff --git a/scripts/values_table.py b/scripts/values_table.py index e3f9903f..2da51487 100644 --- a/scripts/values_table.py +++ b/scripts/values_table.py @@ -49,6 +49,8 @@ "jobs.", "refdata.deploy", "refdata.enabled", + "cvmfs", + "s3csi", "refdata.", "setupJob.", "ingress.", @@ -75,10 +77,12 @@ "postgresql.", "s3csi.", "tusd.", + "rabbitmq", + "trainingHook", ] # Entries that should be ignored. -ignored = [ ] +ignored = [ 'startupProbe', 'readinesProbe', 'livenessProbe' ] longest_key = -1 longest_desc = -1 @@ -120,6 +124,8 @@ def print_table(): numbered_order[full_key] = i for key in sorted(set(key_list), key=lambda d: numbered_order[d]): print_table_row(key) + # for key in README_ORDER: + # print_table_row(key) # Prints a single row in the table. def print_table_row(key): @@ -156,7 +162,7 @@ def parse_value(key, value, label=None): record_key('postgresql.enabled') return - if label in ignored: + if label in ignored or key in ignored: # print(f"Ignoring {label}") return diff --git a/scripts/yamldoc.py b/scripts/yamldoc.py new file mode 100644 index 00000000..ec996b5d --- /dev/null +++ b/scripts/yamldoc.py @@ -0,0 +1,120 @@ +# This is NOT a YAML parser. This script does the bare minimum to match +# comments with keys but may get it wrong. Output from this script MUST be +# curated by a human. + +import sys +import argparse + +# Character string used to indicate the comment is a YamlDoc comment. +chars = '#-' + + +class LineProvider: + ''' + A simple iterator type class. Mostly needed because occasionally we need + to push a line back after searching for the end of a multi-line string. + Also collapses lines that end with '\' into a single line. + ''' + def __init__(self, inputPath: str): + with open(inputPath) as f: + self._lines = f.read().splitlines() + self._current = 0 + self._previous = None + + def hasNext(self): + return self._current < len(self._lines) + + + def getNext(self): + self._previous = self._current + line = self._collect_line() + return line + + def back(self): + if self._previous is not None: + self._current = self._previous + else: + self._current -= 1 + + def _next_line(self): + line = self._lines[self._current].rstrip(' ') + self._current += 1 + return line + + def _collect_line(self): + segments = [] + collecting = True + while collecting and self.hasNext(): + line = self._next_line() + if line.endswith('\\'): + line = line.rstrip('\\') + else: + collecting = False + segments.append(line) + return ' '.join(segments) + + + +def get_line_indent(line: str) -> int: + return len(line) - len(line.lstrip()) + + +def print_table_row(path: list, key: str, comments: list): + key = key.replace('.', '\.') + if len(path) == 0: + full_path = key + else: + full_path = f"{'.'.join(path)}.{key}" + print(f"| {full_path} | { ' '.join(comments)} |") + + +def main(filename: str, chars: str): + lp = LineProvider(filename) + path = [] + doc_lines = [] + indent_stack = [ 0 ] + key = '' + print("| Key | Description |") + while lp.hasNext(): + line = lp.getNext().rstrip() + if line.lstrip().startswith(chars): + doc_lines.append(line.replace(chars, '').strip()) + continue + + if len(line.strip()) == 0 or line.lstrip().startswith('#'): + continue + indent = get_line_indent(line) + if indent > indent_stack[-1]: + indent_stack.append(indent) + path.append(key) + elif indent < indent_stack[-1]: + while indent != indent_stack[-1]: + indent_stack.pop() + path.pop() + + if ':' in line: + kv = line.split(':') + key = kv[0].strip() + if len(doc_lines) > 0: + print_table_row(path, key, doc_lines) + doc_lines = [] + # Skip over multi-line strings + if line.endswith('|') or line.endswith('|-') or line.endswith('>-'): + while lp.hasNext() and get_line_indent(line) >= indent: + line = lp.getNext() + # The above loop finds the first line that is NOT part of the + # multi-line string so we need to put that line back into the + # buffer. + lp.back() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="yamldoc", + description="Generate a Markdown table with descriptions of the YAML keys in a Helm Chart's vales.yaml file.", + epilog="Copyright 2023 The Galaxy Project" + ) + parser.add_argument('filename', nargs=1) + parser.add_argument('-c', '--chars', default="#-") + args = parser.parse_args() + main(args.filename[0], args.chars) \ No newline at end of file From ac52769a6349dff30b33b9ea522b5ba4eab90e4e Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Tue, 28 Mar 2023 17:51:21 -0400 Subject: [PATCH 029/134] Fix table header in output --- scripts/yamldoc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/yamldoc.py b/scripts/yamldoc.py index ec996b5d..99cb9890 100644 --- a/scripts/yamldoc.py +++ b/scripts/yamldoc.py @@ -75,6 +75,7 @@ def main(filename: str, chars: str): indent_stack = [ 0 ] key = '' print("| Key | Description |") + print("|-----|-------------|") while lp.hasNext(): line = lp.getNext().rstrip() if line.lstrip().startswith(chars): From 156eb1f5019ee95f32f9eb38535c9932520b290e Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Tue, 28 Mar 2023 17:51:37 -0400 Subject: [PATCH 030/134] New values table --- VALUES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/VALUES.md b/VALUES.md index 0585741d..bbae4835 100644 --- a/VALUES.md +++ b/VALUES.md @@ -1,4 +1,5 @@ | Key | Description | +|-----|-------------| | nameOverride | Partial override of the `galaxy.fullname`. The `.Release.Name` will be prepended to generate the fullname. | | fullnameOverride | Fully override the `galaxy.fullname` | | image.repository | Repository containing the Galaxy image. | From 32dc5a1e11e94e041f61e8897e1ffdb732a8767c Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Mon, 10 Apr 2023 15:36:15 -0400 Subject: [PATCH 031/134] Add a section for Expression Tools and the param_value_from_file tool --- galaxy/files/configs/tool_conf.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/galaxy/files/configs/tool_conf.xml b/galaxy/files/configs/tool_conf.xml index 6870427f..840aa27e 100644 --- a/galaxy/files/configs/tool_conf.xml +++ b/galaxy/files/configs/tool_conf.xml @@ -38,6 +38,9 @@ +
+ +