From 3f4302c8842311247b49ca9f7bd6d626c82ae1ac Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 12 Jun 2025 13:40:56 -0700 Subject: [PATCH 01/17] add grip indexing to initdb job --- helm/grip/templates/post-install.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/helm/grip/templates/post-install.yaml b/helm/grip/templates/post-install.yaml index e79ebaed8..fd0917aeb 100644 --- a/helm/grip/templates/post-install.yaml +++ b/helm/grip/templates/post-install.yaml @@ -32,4 +32,16 @@ spec: echo '}' >> /tmp/request.json curl -X POST --header "Content-Type: application/json" --data @/tmp/request.json http://{{ .Values.serviceName }}:8201/v1/graph/{{.Values.graphName}}/jsonschema + vertex_labels=$(curl -s -X GET "http://{{.Values.serviceName}}:8201/v1/graph/{{.Values.graphName}}__schema__/label" | sed -n 's/.*"vertexLabels":\[\([^]]*\)\].*/\1/p' | sed 's/"//g' | tr ',' '\n') + + echo "$vertex_labels" | while read -r label; do + JSON_PAYLOAD="{\"field\": \"auth_resource_path\"}" + curl -s -X POST \ + "http://{{.Values.serviceName}}:8201/v1/graph/CALYPR/index/${label}" \ + -H 'Content-Type: application/json' \ + -d "${JSON_PAYLOAD}" + done + + + restartPolicy: OnFailure From 764c8ec3a7ad05a14bbcfc029affd674d6818ba8 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 13 Jun 2025 11:17:35 -0700 Subject: [PATCH 02/17] indexd dev updates --- helm/indexd/indexd-settings/local_settings.py | 4 + helm/indexd/templates/deployment.yaml | 18 ++-- helm/indexd/templates/pre-install.yaml | 96 ------------------- .../gen3.nginx.conf/indexd-service.conf | 40 +------- 4 files changed, 16 insertions(+), 142 deletions(-) delete mode 100644 helm/indexd/templates/pre-install.yaml diff --git a/helm/indexd/indexd-settings/local_settings.py b/helm/indexd/indexd-settings/local_settings.py index 42d74247e..30fc5f906 100644 --- a/helm/indexd/indexd-settings/local_settings.py +++ b/helm/indexd/indexd-settings/local_settings.py @@ -62,4 +62,8 @@ ) ) +rbac = environ.get("RBAC", "false").lower() == "true" +if rbac: + CONFIG["RBAC"] = True + settings = {"config": CONFIG, "auth": AUTH} diff --git a/helm/indexd/templates/deployment.yaml b/helm/indexd/templates/deployment.yaml index 497d4f456..e969db4cc 100644 --- a/helm/indexd/templates/deployment.yaml +++ b/helm/indexd/templates/deployment.yaml @@ -52,13 +52,13 @@ spec: secretKeyRef: name: indexd-dbcreds key: host - optional: false + optional: false - name: PGUSER valueFrom: secretKeyRef: name: indexd-dbcreds key: username - optional: false + optional: false - name: PGPASSWORD valueFrom: secretKeyRef: @@ -93,15 +93,17 @@ spec: containerPort: 80 protocol: TCP livenessProbe: - httpGet: - path: /_status - port: http + exec: + command: + - ls + - / initialDelaySeconds: 30 periodSeconds: 10 readinessProbe: - httpGet: - path: /_status - port: http + exec: + command: + - ls + - / resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} diff --git a/helm/indexd/templates/pre-install.yaml b/helm/indexd/templates/pre-install.yaml deleted file mode 100644 index f6c2e3586..000000000 --- a/helm/indexd/templates/pre-install.yaml +++ /dev/null @@ -1,96 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - # - # A little helper jobs for creating new indexd - # users with basic-auth access to indexd update - # endpoints. - # Loads the list of users from env var. - name: indexd-userdb -spec: - backoffLimit: 0 - template: - metadata: - labels: - app: gen3job - spec: - automountServiceAccountToken: false - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - initContainers: - - name: wait-for-indexd - image: curlimages/curl:latest - command: ["/bin/sh","-c"] - args: ["while [ $(curl -sw '%{http_code}' http://indexd-service/index -o /dev/null) -ne 200 ]; do sleep 5; echo 'Waiting for indexd...'; done"] - containers: - - name: indexd - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - volumeMounts: - - name: "config-volume" - readOnly: true - mountPath: "/var/www/indexd/local_settings.py" - subPath: "local_settings.py" - env: - - name: PGHOST - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: host - optional: false - - name: PGUSER - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: username - optional: false - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: password - optional: false - - name: PGDB - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: database - optional: false - - name: DBREADY - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: dbcreated - optional: false - - name: FENCE_PASS - valueFrom: - secretKeyRef: - name: indexd-service-creds - key: fence - optional: false - - name: SHEEPDOG_PASS - valueFrom: - secretKeyRef: - name: indexd-service-creds - key: sheepdog - optional: false - imagePullPolicy: Always - command: ["/bin/bash" ] - args: - - "-c" - # Script always succeeds if it runs (echo exits with 0) - # indexd image does not include jq, so use python - - | - if command -v python &> /dev/null; then - echo 'python /indexd/bin/index_admin.py create --username "fence" --password "${FENCE_PASS}' - python /indexd/bin/index_admin.py create --username "fence" --password "${FENCE_PASS}" - echo 'python /indexd/bin/index_admin.py create --username "sheepdog" --password "${SHEEPDOG_PASS}' - python /indexd/bin/index_admin.py create --username "sheepdog" --password "${SHEEPDOG_PASS}" - else - echo 'poetry run python /indexd/bin/index_admin.py create --username "fence" --password "${FENCE_PASS}' - poetry run python /indexd/bin/index_admin.py create --username "fence" --password "${FENCE_PASS}" - echo 'poetry run python /indexd/bin/index_admin.py create --username "sheepdog" --password "${SHEEPDOG_PASS}' - poetry run python /indexd/bin/index_admin.py create --username "sheepdog" --password "${SHEEPDOG_PASS}" - fi - restartPolicy: Never \ No newline at end of file diff --git a/helm/revproxy/gen3.nginx.conf/indexd-service.conf b/helm/revproxy/gen3.nginx.conf/indexd-service.conf index a464afccb..5d5545a97 100644 --- a/helm/revproxy/gen3.nginx.conf/indexd-service.conf +++ b/helm/revproxy/gen3.nginx.conf/indexd-service.conf @@ -1,5 +1,3 @@ - - # GA4GH endpoint for DOS resolver and DRS server location /ga4gh/ { if ($csrf_check !~ ^ok-\S.+$) { return 403 "failed csrf check"; @@ -18,40 +16,6 @@ set $proxy_service "indexd"; set $upstream http://indexd-service$des_domain; - rewrite ^/index/(.*) /$1 break; - proxy_pass $upstream; - proxy_redirect http://$host/ https://$host/index/; - } - - location /index-admin/ { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } - set $authz_resource "/indexd_gateway"; - set $authz_method "access"; - set $authz_service "indexd_gateway"; - # be careful - sub-request runs in same context as this request - auth_request /gen3-authz; - - # - # For some reason nginx breaks the proxy body - # if we try to set Authorization from a perl_set variable - # that samples the environment ... ugh! - # - set $indexd_password "Basic ${indexd_b64}"; - - # For testing: - #add_header Set-Cookie "X-Frickjack=${indexd_password};Path=/;Max-Age=600"; - set $proxy_service "indexd"; - set $upstream http://indexd-service$des_domain; - rewrite ^/index-admin/(.*) /$1 break; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For "$realip"; - proxy_set_header X-UserId "$userid"; - proxy_set_header X-SessionId "$session_id"; - proxy_set_header X-VisitorId "$visitor_id"; - proxy_set_header Authorization "$indexd_password"; - proxy_pass $upstream; - proxy_redirect http://$host/ https://$host/index-admin/; - } + proxy_redirect http://$host/ https://$host/; + } \ No newline at end of file From f05225aed8ffde5293dd0cf607a569093f6c0469 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 13 Jun 2025 11:32:46 -0700 Subject: [PATCH 03/17] bug fix --- helm/revproxy/gen3.nginx.conf/indexd-service.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/revproxy/gen3.nginx.conf/indexd-service.conf b/helm/revproxy/gen3.nginx.conf/indexd-service.conf index 5d5545a97..067b7d5de 100644 --- a/helm/revproxy/gen3.nginx.conf/indexd-service.conf +++ b/helm/revproxy/gen3.nginx.conf/indexd-service.conf @@ -17,5 +17,5 @@ set $proxy_service "indexd"; set $upstream http://indexd-service$des_domain; proxy_pass $upstream; - proxy_redirect http://$host/ https://$host/; + proxy_redirect http://$host/ https://$host/index/; } \ No newline at end of file From a4c5e460d6ed2636fa64238560e035e6a9387886 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 13 Jun 2025 11:49:29 -0700 Subject: [PATCH 04/17] revert change --- helm/revproxy/gen3.nginx.conf/indexd-service.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/revproxy/gen3.nginx.conf/indexd-service.conf b/helm/revproxy/gen3.nginx.conf/indexd-service.conf index 067b7d5de..35614db3a 100644 --- a/helm/revproxy/gen3.nginx.conf/indexd-service.conf +++ b/helm/revproxy/gen3.nginx.conf/indexd-service.conf @@ -16,6 +16,7 @@ set $proxy_service "indexd"; set $upstream http://indexd-service$des_domain; + rewrite ^/index/(.*) /$1 break; proxy_pass $upstream; proxy_redirect http://$host/ https://$host/index/; } \ No newline at end of file From eb645301e4e621ece23259de9254e266993ab1a7 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 3 Jul 2025 15:53:41 +0000 Subject: [PATCH 05/17] update indexd nginx --- helm/indexd/templates/deployment.yaml | 14 +++---- .../gen3.nginx.conf/indexd-service.conf | 39 ++++++++++--------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/helm/indexd/templates/deployment.yaml b/helm/indexd/templates/deployment.yaml index e969db4cc..ac1698545 100644 --- a/helm/indexd/templates/deployment.yaml +++ b/helm/indexd/templates/deployment.yaml @@ -93,17 +93,15 @@ spec: containerPort: 80 protocol: TCP livenessProbe: - exec: - command: - - ls - - / + httpGet: + path: /_status + port: http initialDelaySeconds: 30 periodSeconds: 10 readinessProbe: - exec: - command: - - ls - - / + httpGet: + path: /_status + port: http resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} diff --git a/helm/revproxy/gen3.nginx.conf/indexd-service.conf b/helm/revproxy/gen3.nginx.conf/indexd-service.conf index 35614db3a..531ca99c0 100644 --- a/helm/revproxy/gen3.nginx.conf/indexd-service.conf +++ b/helm/revproxy/gen3.nginx.conf/indexd-service.conf @@ -1,22 +1,23 @@ - location /ga4gh/ { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } +location /ga4gh/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } - set $proxy_service "indexd"; - set $upstream http://indexd-service$des_domain; - proxy_pass $upstream; - proxy_redirect http://$host/ https://$host/; - } + set $proxy_service "indexd"; + set $upstream http://indexd-service$des_domain; - location /index/ { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/; +} + +location /index/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "indexd"; + set $upstream http://indexd-service$des_domain; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/index/; +} - set $proxy_service "indexd"; - set $upstream http://indexd-service$des_domain; - rewrite ^/index/(.*) /$1 break; - proxy_pass $upstream; - proxy_redirect http://$host/ https://$host/index/; - } \ No newline at end of file From 2875f617bb8edda2e7e005becbfbc5031bc62c54 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 8 Jul 2025 13:55:40 -0700 Subject: [PATCH 06/17] back out indexd revproxy change --- helm/grip/templates/deployment.yaml | 2 +- helm/revproxy/gen3.nginx.conf/indexd-service.conf | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/helm/grip/templates/deployment.yaml b/helm/grip/templates/deployment.yaml index 99ecf8a72..d66fe33bd 100644 --- a/helm/grip/templates/deployment.yaml +++ b/helm/grip/templates/deployment.yaml @@ -66,7 +66,7 @@ spec: - -w - graphql=gql-gen.so - -l - - graphql:graph=CALIPER + - graphql:graph=CALYPER env: - name: GOMAXPROCS value: "4" diff --git a/helm/revproxy/gen3.nginx.conf/indexd-service.conf b/helm/revproxy/gen3.nginx.conf/indexd-service.conf index 531ca99c0..20f9414a7 100644 --- a/helm/revproxy/gen3.nginx.conf/indexd-service.conf +++ b/helm/revproxy/gen3.nginx.conf/indexd-service.conf @@ -17,6 +17,9 @@ location /index/ { set $proxy_service "indexd"; set $upstream http://indexd-service$des_domain; + + rewrite ^/index/(.*) /$1 break; + proxy_pass $upstream; proxy_redirect http://$host/ https://$host/index/; } From 90da533eb062d54c77a7d8fab2e274d6637ad728 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 22 Sep 2025 12:48:34 -0700 Subject: [PATCH 07/17] add small changes --- helm/fence/templates/useryaml-job.yaml | 1 - helm/grip/templates/deployment.yaml | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/helm/fence/templates/useryaml-job.yaml b/helm/fence/templates/useryaml-job.yaml index 6adb96c4d..c6375af47 100644 --- a/helm/fence/templates/useryaml-job.yaml +++ b/helm/fence/templates/useryaml-job.yaml @@ -43,7 +43,6 @@ spec: - "-c" # Script always succeeds if it runs (echo exits with 0) - | - pip3 install SQLAlchemy==1.3.6 # can be removed once this is merged: https://github.com/uc-cdis/fence/pull/1096 fence-create sync --arborist http://arborist-service --yaml /var/www/fence/user.yaml restartPolicy: OnFailure diff --git a/helm/grip/templates/deployment.yaml b/helm/grip/templates/deployment.yaml index d66fe33bd..941af95b8 100644 --- a/helm/grip/templates/deployment.yaml +++ b/helm/grip/templates/deployment.yaml @@ -66,7 +66,9 @@ spec: - -w - graphql=gql-gen.so - -l - - graphql:graph=CALYPER + - graphql:graph=CALYPR + - --log-level + - debug env: - name: GOMAXPROCS value: "4" From 0471bd1dcc2102bd9bdf023d4ebb0fe5c19ff96d Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 29 Sep 2025 15:03:19 -0700 Subject: [PATCH 08/17] add qdrant into deployment --- helm/gen3/Chart.yaml | 3 ++ .../gen3/templates/qdrant-api-key-secret.yaml | 26 +++++++++++++++ helm/gen3/values.yaml | 32 +++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 helm/gen3/templates/qdrant-api-key-secret.yaml diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index 2a133975a..0febf36ad 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -111,6 +111,9 @@ dependencies: version: 0.1.0 repository: "file://../viv" condition: viv.enabled +- name: qdrant + version: 1.15.4 + repository: "https://qdrant.github.io/qdrant-helm" # A chart can be either an 'application' or a 'library' chart. # diff --git a/helm/gen3/templates/qdrant-api-key-secret.yaml b/helm/gen3/templates/qdrant-api-key-secret.yaml new file mode 100644 index 000000000..24078f250 --- /dev/null +++ b/helm/gen3/templates/qdrant-api-key-secret.yaml @@ -0,0 +1,26 @@ +{{- /* Generate a random, persistent Qdrant API Key. +This logic lives in the parent chart to avoid being overwritten by 'helm dep update'. +*/}} +{{- $secretName := "qdrant-api-key-secret" -}} +{{- $secretKey := "api-key" -}} +{{- $existingSecret := (lookup "v1" "Secret" .Release.Namespace $secretName) -}} + +# Only render the Secret if qdrant is enabled in the values file +{{- if .Values.qdrant.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ .Release.Namespace }} + labels: + # Use a label helper from the 'common' subchart or define your own + {{- include "common.labels.standard" . | nindent 4 }} +type: Opaque +data: + # Use the lookup function to ensure persistence across upgrades + {{- if $existingSecret }} + {{ $secretKey }}: {{ index $existingSecret.data $secretKey }} + {{- else }} + {{ $secretKey }}: {{ randAlphaNum 32 | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index 282a68002..088acfaf2 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -290,3 +290,35 @@ postgresql: persistence: # -- (bool) Option to persist the dbs data. enabled: true + + +qdrant: + extraEnv: + - name: QDRANT__SERVICE__API_KEY + valueFrom: + secretKeyRef: + name: qdrant-api-key-secret # Name of the Kubernetes Secret from Step 1 + key: api-key + enabled: true + replicaCount: 1 + resources: + limits: + cpu: 4 + memory: 8Gi + requests: + cpu: 200m + memory: 256Mi + persistence: + enabled: true + storageClass: "standard" + size: 26Gi + accessModes: + - ReadWriteOnce + service: + type: ClusterIP + port: 6333 + grpcPort: 6334 + image: + repository: qdrant/qdrant + tag: v1.15 + pullPolicy: IfNotPresent \ No newline at end of file From d4643f4a84479f975c9e40bc4285312b84d27f30 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 3 Oct 2025 12:55:41 -0700 Subject: [PATCH 09/17] update gecko deployment template --- helm/gecko/templates/deployment.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/helm/gecko/templates/deployment.yaml b/helm/gecko/templates/deployment.yaml index 6c9271b16..658eea7e1 100644 --- a/helm/gecko/templates/deployment.yaml +++ b/helm/gecko/templates/deployment.yaml @@ -61,6 +61,16 @@ spec: env: {{- toYaml .Values.env | nindent 12 }} + - name: QDRANT_HOST + value: {{ printf "%s-qdrant" .Release.Name | quote }} + - name: QDRANT_PORT + value: "6334" + - name: QDRANT_API_KEY + valueFrom: + secretKeyRef: + name: {{ "qdrant-api-key-secret" }} + key: {{ "api-key" }} + optional: false - name: PGPASSWORD valueFrom: secretKeyRef: From cbe2c1dacd91285c5d2f5ba5cc29d4ebb4fb297b Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 6 Oct 2025 07:39:32 -0700 Subject: [PATCH 10/17] update gecko service to include swagger docs --- helm/revproxy/gen3.nginx.conf/gecko-service.conf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/helm/revproxy/gen3.nginx.conf/gecko-service.conf b/helm/revproxy/gen3.nginx.conf/gecko-service.conf index 32195a4c1..8bf534d9d 100644 --- a/helm/revproxy/gen3.nginx.conf/gecko-service.conf +++ b/helm/revproxy/gen3.nginx.conf/gecko-service.conf @@ -11,6 +11,7 @@ location /ExplorerConfig/health { proxy_pass $upstream; } + location /ExplorerConfig/list { proxy_connect_timeout 600s; proxy_send_timeout 600s; @@ -38,4 +39,18 @@ location /ExplorerConfig/ { rewrite ^/ExplorerConfig/(.*)$ /config/$1 break; proxy_pass $upstream$is_args$args; client_max_body_size 0; +} + +location ~* ^/(Vector|vector)/ { + proxy_connect_timeout 600s; + proxy_send_timeout 600s; + proxy_read_timeout 600s; + send_timeout 600s; + + set $proxy_service "gecko"; + set $upstream http://gecko-service.$namespace.svc.cluster.local; + + rewrite ^/(Vector|vector)/(.*)$ /vector/$2 break; + proxy_pass $upstream; + client_max_body_size 0; } \ No newline at end of file From d2a0a87defe421c93560c695913a54c35b9fac2d Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 16 Oct 2025 10:00:39 -0700 Subject: [PATCH 11/17] update helm to work with dir reader --- helm/gecko/templates/deployment.yaml | 13 +++++++++++++ helm/gecko/templates/qdrant-pv.yaml | 18 ++++++++++++++++++ helm/gen3/values.yaml | 2 +- helm/grip/templates/deployment.yaml | 4 ++++ helm/grip/templates/post-install.yaml | 4 ++-- helm/grip/templates/service.yaml | 5 +++++ .../gen3.nginx.conf/gecko-service.conf | 16 ++++++++++++++++ 7 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 helm/gecko/templates/qdrant-pv.yaml diff --git a/helm/gecko/templates/deployment.yaml b/helm/gecko/templates/deployment.yaml index 658eea7e1..d423e705e 100644 --- a/helm/gecko/templates/deployment.yaml +++ b/helm/gecko/templates/deployment.yaml @@ -46,6 +46,7 @@ spec: httpGet: path: /health port: 80 + resources: {{- toYaml .Values.resources | nindent 12 }} command: ["sh"] @@ -61,6 +62,18 @@ spec: env: {{- toYaml .Values.env | nindent 12 }} + - name: GRIP_GRAPH + valueFrom: + configMapKeyRef: + name: grip-values + key: graphName + - name: GRIP_HOST + valueFrom: + configMapKeyRef: + name: grip-values + key: serviceName + - name: GRIP_PORT + value: "8202" - name: QDRANT_HOST value: {{ printf "%s-qdrant" .Release.Name | quote }} - name: QDRANT_PORT diff --git a/helm/gecko/templates/qdrant-pv.yaml b/helm/gecko/templates/qdrant-pv.yaml new file mode 100644 index 000000000..6f60f79f8 --- /dev/null +++ b/helm/gecko/templates/qdrant-pv.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: qdrant-storage-pv + labels: + app.kubernetes.io/managed-by: "Helm" + annotations: + meta.helm.sh/release-name: {{ .Release.Name }} + meta.helm.sh/release-namespace: "default" +spec: + capacity: + storage: 26Gi + accessModes: + - ReadWriteOnce + storageClassName: "qdrant-manual-storage" + persistentVolumeReclaimPolicy: Retain + hostPath: + path: "/mnt/data/qdrant-local" diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index 088acfaf2..5fd9316c0 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -310,7 +310,7 @@ qdrant: memory: 256Mi persistence: enabled: true - storageClass: "standard" + storageClass: "qdrant-manual-storage" size: 26Gi accessModes: - ReadWriteOnce diff --git a/helm/grip/templates/deployment.yaml b/helm/grip/templates/deployment.yaml index 941af95b8..aabe90ae1 100644 --- a/helm/grip/templates/deployment.yaml +++ b/helm/grip/templates/deployment.yaml @@ -41,6 +41,10 @@ spec: - name: http containerPort: {{ .Values.service.port }} protocol: TCP + + - name: rpc + protocol: TCP + containerPort: 8202 livenessProbe: httpGet: path: /writer/_status diff --git a/helm/grip/templates/post-install.yaml b/helm/grip/templates/post-install.yaml index fd0917aeb..0ad298180 100644 --- a/helm/grip/templates/post-install.yaml +++ b/helm/grip/templates/post-install.yaml @@ -25,8 +25,8 @@ spec: done echo "Grip is online. Creating {{.Values.graphName}} Graph" curl -X POST 'http://{{ .Values.serviceName }}:8201/v1/graph/{{.Values.graphName}}' - echo "Adding {{.Values.graphName}} schema from https://raw.githubusercontent.com/bmeg/iceberg/7f6cfdb558d05370fc645b5ab894b98b38a01e1b/schemas/graph/graph-fhir.json" - curl -s https://raw.githubusercontent.com/bmeg/iceberg/7f6cfdb558d05370fc645b5ab894b98b38a01e1b/schemas/graph/graph-fhir.json -o /tmp/schema.json + echo "Adding {{.Values.graphName}} schema from https://raw.githubusercontent.com/bmeg/iceberg/9c916e834716ea725d956f529ae37a343b10f3e6/schemas/graph/graph-fhir.json" + curl -s https://raw.githubusercontent.com/bmeg/iceberg/9c916e834716ea725d956f529ae37a343b10f3e6/schemas/graph/graph-fhir.json -o /tmp/schema.json echo '{"graph":"{{.Values.graphName}}","data":' > /tmp/request.json cat /tmp/schema.json >> /tmp/request.json echo '}' >> /tmp/request.json diff --git a/helm/grip/templates/service.yaml b/helm/grip/templates/service.yaml index c87dbaca8..6de3e94a6 100644 --- a/helm/grip/templates/service.yaml +++ b/helm/grip/templates/service.yaml @@ -11,6 +11,11 @@ spec: targetPort: http protocol: TCP name: http + + - port: 8202 + targetPort: rpc + protocol: TCP + name: rpc selector: {{- include "grip.selectorLabels" . | nindent 4 }} diff --git a/helm/revproxy/gen3.nginx.conf/gecko-service.conf b/helm/revproxy/gen3.nginx.conf/gecko-service.conf index 8bf534d9d..3fa3168b2 100644 --- a/helm/revproxy/gen3.nginx.conf/gecko-service.conf +++ b/helm/revproxy/gen3.nginx.conf/gecko-service.conf @@ -53,4 +53,20 @@ location ~* ^/(Vector|vector)/ { rewrite ^/(Vector|vector)/(.*)$ /vector/$2 break; proxy_pass $upstream; client_max_body_size 0; +} + +location ~* ^/Dir { + proxy_connect_timeout 600s; + proxy_send_timeout 600s; + proxy_read_timeout 600s; + send_timeout 600s; + + set $proxy_service "gecko"; + set $upstream http://gecko-service.$namespace.svc.cluster.local; + + rewrite ^/Dir(.*)$ /dir$1 break; + proxy_pass $upstream; + client_max_body_size 0; + + } \ No newline at end of file From 2edbfb92720ffee1c73d7ff255732cc2e9cf0050 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 5 Nov 2025 15:27:07 -0800 Subject: [PATCH 12/17] add gecko db init for frontend --- helm/gecko/files/init-data/apps_page.json | 25 ++++ helm/gecko/files/init-data/file_summary.json | 25 ++++ helm/gecko/files/init-data/nav.json | 120 +++++++++++++++++++ helm/gecko/templates/db-init-job.yaml | 77 +++++++++++- helm/gecko/templates/job-db-init.yaml | 16 +++ helm/guppy/templates/guppy_config.yaml | 3 +- 6 files changed, 260 insertions(+), 6 deletions(-) create mode 100644 helm/gecko/files/init-data/apps_page.json create mode 100644 helm/gecko/files/init-data/file_summary.json create mode 100644 helm/gecko/files/init-data/nav.json create mode 100644 helm/gecko/templates/job-db-init.yaml diff --git a/helm/gecko/files/init-data/apps_page.json b/helm/gecko/files/init-data/apps_page.json new file mode 100644 index 000000000..ee206b969 --- /dev/null +++ b/helm/gecko/files/init-data/apps_page.json @@ -0,0 +1,25 @@ +{ + "appCards": [ + { + "title": "Explore BForePC", + "description": "Explore BForePC data in the CALYPR system", + "icon": "/icons/binoculars.svg", + "href": "/Explorer/BForePC", + "perms": "HTAN_INT-BForePC" + }, + { + "title": "Explore MMTERT SMMART Data", + "description": "Explore MMTERT SMMART data in the CALYPR system", + "icon": "/icons/binoculars.svg", + "href": "/Explorer/SMMART", + "perms": "cbds-smmart_labkey_demo" + }, + { + "title": "SMMART Report", + "description": "A clinical research study measuring tumor evolution and its resistance to therapy to deliver better cancer treatment. Overview of SMMART data processing", + "icon": "/icons/chart-bar.svg", + "href": "/SMCLICK", + "perms": "cbds-smmart_labkey_demo" + } + ] +} diff --git a/helm/gecko/files/init-data/file_summary.json b/helm/gecko/files/init-data/file_summary.json new file mode 100644 index 000000000..4fa207ead --- /dev/null +++ b/helm/gecko/files/init-data/file_summary.json @@ -0,0 +1,25 @@ +{ + "config": { + "document_reference_title": { + "title": "Title", + "field": "document_reference_title" + }, + "document_reference_size": { + "cellRenderFunction": "HumanReadableString", + "type": "string", + "title": "File Size", + "field": "document_reference_size" + }, + "document_reference_source_path": { + "title": "Source Path", + "field": "document_reference_source_path" + } + }, + "binslicePoints": [ + 0, 1048576, 524288000, 1073741824, 107374182400, 9007199254740991 + ], + "barChartColor": "#e9724d", + "defaultProject": "gdc-esca", + "idField": "document_reference_id", + "index": "document_reference" +} diff --git a/helm/gecko/files/init-data/nav.json b/helm/gecko/files/init-data/nav.json new file mode 100644 index 000000000..536e2c819 --- /dev/null +++ b/helm/gecko/files/init-data/nav.json @@ -0,0 +1,120 @@ +{ + "headerProps": { + "topBar": { + "items": [ + { + "href": "https://www.ohsu.edu/knight-cancer-institute", + "name": "CBDS", + "classNames": { + "root": "", + "label": "", + "button": "" + } + } + ], + "loginButtonVisibility": "hidden" + }, + "navigation": { + "classNames": { + "root": "bg-base-max text-primary opacity-100 hover:opacity-100", + "item": "py-2 px-4 hover:bg-base-lightest hover:text-base-contrast", + "navigationPanel": "bg-base-max text-primary" + }, + "logo": { + "src": "/icons/ohsu.svg", + "width": 52.5, + "height": 40, + "href": "/Apps", + "title": "CALYPR" + }, + "items": [ + { + "icon": "gen3:exploration", + "href": "/Explorer", + "name": "Exploration", + "tooltip": "The Exploration Page enables discovery of the data at the subject level and features a cohort builder.", + "title": "Explorer" + }, + { + "icon": "gen3:profile", + "href": "/Profile", + "name": "Profile", + "tooltip": "Create API keys for programmatic data access, and review your authorization privileges to datasets and services.", + "title": "Profile" + } + ] + }, + "leftnav": [ + { + "title": "Home", + "description": "Home Apps page", + "icon": "/icons/home.svg", + "href": "/Apps", + "perms": null + }, + { + "title": "Directory Structure", + "description": "Search for files via a tree based interactive search", + "icon": "/icons/binary-tree.svg", + "href": "/Miller", + "perms": null + }, + { + "title": "File Summary", + "description": "Overview of file system usage", + "icon": "/icons/file.svg", + "href": "/Filesummary", + "perms": null + }, + { + "title": "Project Discovery", + "description": "Explore project summaries of every project in CALYPR", + "icon": "/icons/compass.svg", + "href": "/Discovery", + "perms": null + }, + { + "title": "Image Viewer", + "description": "View available .ome.tif images using Avivator", + "icon": "/icons/layers-intersect.svg", + "href": "/AvailableImages", + "perms": null + }, + { + "title": "My Projects", + "description": "Identify the list of projects in which you have access", + "icon": "/icons/key.svg", + "href": "/MyProjects", + "perms": null + } + ] + }, + "footerProps": { + "classNames": { + "root": "", + "layout": "flex items-center justify-center" + }, + "rightSection": { + "columns": [ + { + "rows": [ + { + "Icon": { + "logo": "/icons/knight.svg", + "logolight": "/icons/knight_white.svg", + "width": 100, + "height": 47, + "description": "Knight Cancer Institute" + } + } + ] + } + ] + } + }, + "headerMetadata": { + "title": "CALYPR", + "content": "Cancer Analytics Platform", + "key": "calypr-main" + } +} diff --git a/helm/gecko/templates/db-init-job.yaml b/helm/gecko/templates/db-init-job.yaml index df7517add..c5e562b5f 100644 --- a/helm/gecko/templates/db-init-job.yaml +++ b/helm/gecko/templates/db-init-job.yaml @@ -19,19 +19,73 @@ spec: - | #!/bin/bash set -e + + INIT_DATA_PATH="/mnt/db-init-data" + echo "Waiting for database to be ready..." until psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "SELECT 1"; do echo "Database not ready yet, waiting..." sleep 5 done + echo "Database ready, initializing..." + psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" < Date: Tue, 11 Nov 2025 08:42:22 -0800 Subject: [PATCH 13/17] update initdb file --- helm/gecko/files/init-data/apps_page.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/gecko/files/init-data/apps_page.json b/helm/gecko/files/init-data/apps_page.json index ee206b969..fbac3978d 100644 --- a/helm/gecko/files/init-data/apps_page.json +++ b/helm/gecko/files/init-data/apps_page.json @@ -4,14 +4,14 @@ "title": "Explore BForePC", "description": "Explore BForePC data in the CALYPR system", "icon": "/icons/binoculars.svg", - "href": "/Explorer/BForePC", + "href": "/Explorer/HTAN_INT-BForePC", "perms": "HTAN_INT-BForePC" }, { "title": "Explore MMTERT SMMART Data", "description": "Explore MMTERT SMMART data in the CALYPR system", "icon": "/icons/binoculars.svg", - "href": "/Explorer/SMMART", + "href": "/Explorer/cbds-smmart_labkey_demo", "perms": "cbds-smmart_labkey_demo" }, { From 628992c4cd7cb4295a25f7c1563a94bf61239b58 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 6 Jan 2026 13:52:32 -0800 Subject: [PATCH 14/17] fix: Resolve ct linting errors --- helm/gen3/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index 5fd9316c0..7c21aad93 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -297,7 +297,7 @@ qdrant: - name: QDRANT__SERVICE__API_KEY valueFrom: secretKeyRef: - name: qdrant-api-key-secret # Name of the Kubernetes Secret from Step 1 + name: qdrant-api-key-secret # Name of the Kubernetes Secret from Step 1 key: api-key enabled: true replicaCount: 1 @@ -321,4 +321,4 @@ qdrant: image: repository: qdrant/qdrant tag: v1.15 - pullPolicy: IfNotPresent \ No newline at end of file + pullPolicy: IfNotPresent From 8d51b793cc5c4f4e39dd14804b7ac4f7830988e9 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 6 Jan 2026 13:57:01 -0800 Subject: [PATCH 15/17] fix: Update Helm charts Added a step to update Helm charts in the workflow. --- .github/workflows/lint_test.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/lint_test.yaml b/.github/workflows/lint_test.yaml index 8fb3770db..d25a13ade 100644 --- a/.github/workflows/lint_test.yaml +++ b/.github/workflows/lint_test.yaml @@ -17,6 +17,10 @@ jobs: with: version: v3.10.0 + - name: Update Helm charts + run: | + helm repo update + - uses: actions/setup-python@v4 with: python-version: '3.9' From d8b25b6ef162888cf6d50eeaf0ffcc614fb9f02c Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 6 Jan 2026 14:00:00 -0800 Subject: [PATCH 16/17] fix: Update lint_test.yaml --- .github/workflows/lint_test.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint_test.yaml b/.github/workflows/lint_test.yaml index d25a13ade..2bf831608 100644 --- a/.github/workflows/lint_test.yaml +++ b/.github/workflows/lint_test.yaml @@ -17,8 +17,11 @@ jobs: with: version: v3.10.0 - - name: Update Helm charts + - name: Add Helm repos run: | + helm repo add elastic https://helm.elastic.co + helm repo add bitnami https://charts.bitnami.com/bitnami + helm repo add qdrant https://qdrant.github.io/qdrant-helm helm repo update - uses: actions/setup-python@v4 From 2c86152cacdda936511b54055447d08cc9e3c189 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 20 Feb 2026 14:09:42 -0800 Subject: [PATCH 17/17] update configs --- helm/gecko/files/init-data/nav.json | 16 +--------------- helm/revproxy/templates/ingress_dev.yaml | 8 +++++--- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/helm/gecko/files/init-data/nav.json b/helm/gecko/files/init-data/nav.json index 536e2c819..522e167bd 100644 --- a/helm/gecko/files/init-data/nav.json +++ b/helm/gecko/files/init-data/nav.json @@ -66,26 +66,12 @@ "href": "/Filesummary", "perms": null }, - { - "title": "Project Discovery", - "description": "Explore project summaries of every project in CALYPR", - "icon": "/icons/compass.svg", - "href": "/Discovery", - "perms": null - }, { "title": "Image Viewer", "description": "View available .ome.tif images using Avivator", "icon": "/icons/layers-intersect.svg", "href": "/AvailableImages", "perms": null - }, - { - "title": "My Projects", - "description": "Identify the list of projects in which you have access", - "icon": "/icons/key.svg", - "href": "/MyProjects", - "perms": null } ] }, @@ -117,4 +103,4 @@ "content": "Cancer Analytics Platform", "key": "calypr-main" } -} +} \ No newline at end of file diff --git a/helm/revproxy/templates/ingress_dev.yaml b/helm/revproxy/templates/ingress_dev.yaml index df2ea60c8..1ebeac551 100644 --- a/helm/revproxy/templates/ingress_dev.yaml +++ b/helm/revproxy/templates/ingress_dev.yaml @@ -7,16 +7,18 @@ spec: {{- if .Values.global.dev }} tls: - secretName: gen3-certs + hosts: + - {{ default .Values.global.hostname .Values.hostname | quote }} # <--- This maps the cert to the domain {{- end }} rules: - - host: {{ default .Values.global.hostname .Values.hostname }} + - host: {{ default .Values.global.hostname .Values.hostname | quote }} http: paths: - path: / pathType: Prefix backend: service: - name: revproxy-service + name: revproxy-service port: number: 80 - {{- end }} +{{- end }} \ No newline at end of file