diff --git a/.github/workflows/lint_test.yaml b/.github/workflows/lint_test.yaml index 8fb3770db..2bf831608 100644 --- a/.github/workflows/lint_test.yaml +++ b/.github/workflows/lint_test.yaml @@ -17,6 +17,13 @@ jobs: with: version: v3.10.0 + - name: Add Helm repos + run: | + helm repo add elastic https://helm.elastic.co + helm repo add bitnami https://charts.bitnami.com/bitnami + helm repo add qdrant https://qdrant.github.io/qdrant-helm + helm repo update + - uses: actions/setup-python@v4 with: python-version: '3.9' diff --git a/helm/fence/templates/useryaml-job.yaml b/helm/fence/templates/useryaml-job.yaml index 6adb96c4d..c6375af47 100644 --- a/helm/fence/templates/useryaml-job.yaml +++ b/helm/fence/templates/useryaml-job.yaml @@ -43,7 +43,6 @@ spec: - "-c" # Script always succeeds if it runs (echo exits with 0) - | - pip3 install SQLAlchemy==1.3.6 # can be removed once this is merged: https://github.com/uc-cdis/fence/pull/1096 fence-create sync --arborist http://arborist-service --yaml /var/www/fence/user.yaml restartPolicy: OnFailure diff --git a/helm/gecko/files/init-data/apps_page.json b/helm/gecko/files/init-data/apps_page.json new file mode 100644 index 000000000..fbac3978d --- /dev/null +++ b/helm/gecko/files/init-data/apps_page.json @@ -0,0 +1,25 @@ +{ + "appCards": [ + { + "title": "Explore BForePC", + "description": "Explore BForePC data in the CALYPR system", + "icon": "/icons/binoculars.svg", + "href": "/Explorer/HTAN_INT-BForePC", + "perms": "HTAN_INT-BForePC" + }, + { + "title": "Explore MMTERT SMMART Data", + "description": "Explore MMTERT SMMART data in the CALYPR system", + "icon": "/icons/binoculars.svg", + "href": "/Explorer/cbds-smmart_labkey_demo", + "perms": "cbds-smmart_labkey_demo" + }, + { + "title": "SMMART Report", + "description": "A clinical research study measuring tumor evolution and its resistance to therapy to deliver better cancer treatment. Overview of SMMART data processing", + "icon": "/icons/chart-bar.svg", + "href": "/SMCLICK", + "perms": "cbds-smmart_labkey_demo" + } + ] +} diff --git a/helm/gecko/files/init-data/file_summary.json b/helm/gecko/files/init-data/file_summary.json new file mode 100644 index 000000000..4fa207ead --- /dev/null +++ b/helm/gecko/files/init-data/file_summary.json @@ -0,0 +1,25 @@ +{ + "config": { + "document_reference_title": { + "title": "Title", + "field": "document_reference_title" + }, + "document_reference_size": { + "cellRenderFunction": "HumanReadableString", + "type": "string", + "title": "File Size", + "field": "document_reference_size" + }, + "document_reference_source_path": { + "title": "Source Path", + "field": "document_reference_source_path" + } + }, + "binslicePoints": [ + 0, 1048576, 524288000, 1073741824, 107374182400, 9007199254740991 + ], + "barChartColor": "#e9724d", + "defaultProject": "gdc-esca", + "idField": "document_reference_id", + "index": "document_reference" +} diff --git a/helm/gecko/files/init-data/nav.json b/helm/gecko/files/init-data/nav.json new file mode 100644 index 000000000..522e167bd --- /dev/null +++ b/helm/gecko/files/init-data/nav.json @@ -0,0 +1,106 @@ +{ + "headerProps": { + "topBar": { + "items": [ + { + "href": "https://www.ohsu.edu/knight-cancer-institute", + "name": "CBDS", + "classNames": { + "root": "", + "label": "", + "button": "" + } + } + ], + "loginButtonVisibility": "hidden" + }, + "navigation": { + "classNames": { + "root": "bg-base-max text-primary opacity-100 hover:opacity-100", + "item": "py-2 px-4 hover:bg-base-lightest hover:text-base-contrast", + "navigationPanel": "bg-base-max text-primary" + }, + "logo": { + "src": "/icons/ohsu.svg", + "width": 52.5, + "height": 40, + "href": "/Apps", + "title": "CALYPR" + }, + "items": [ + { + "icon": "gen3:exploration", + "href": "/Explorer", + "name": "Exploration", + "tooltip": "The Exploration Page enables discovery of the data at the subject level and features a cohort builder.", + "title": "Explorer" + }, + { + "icon": "gen3:profile", + "href": "/Profile", + "name": "Profile", + "tooltip": "Create API keys for programmatic data access, and review your authorization privileges to datasets and services.", + "title": "Profile" + } + ] + }, + "leftnav": [ + { + "title": "Home", + "description": "Home Apps page", + "icon": "/icons/home.svg", + "href": "/Apps", + "perms": null + }, + { + "title": "Directory Structure", + "description": "Search for files via a tree based interactive search", + "icon": "/icons/binary-tree.svg", + "href": "/Miller", + "perms": null + }, + { + "title": "File Summary", + "description": "Overview of file system usage", + "icon": "/icons/file.svg", + "href": "/Filesummary", + "perms": null + }, + { + "title": "Image Viewer", + "description": "View available .ome.tif images using Avivator", + "icon": "/icons/layers-intersect.svg", + "href": "/AvailableImages", + "perms": null + } + ] + }, + "footerProps": { + "classNames": { + "root": "", + "layout": "flex items-center justify-center" + }, + "rightSection": { + "columns": [ + { + "rows": [ + { + "Icon": { + "logo": "/icons/knight.svg", + "logolight": "/icons/knight_white.svg", + "width": 100, + "height": 47, + "description": "Knight Cancer Institute" + } + } + ] + } + ] + } + }, + "headerMetadata": { + "title": "CALYPR", + "content": "Cancer Analytics Platform", + "key": "calypr-main" + } +} \ No newline at end of file diff --git a/helm/gecko/templates/db-init-job.yaml b/helm/gecko/templates/db-init-job.yaml index df7517add..c5e562b5f 100644 --- a/helm/gecko/templates/db-init-job.yaml +++ b/helm/gecko/templates/db-init-job.yaml @@ -19,19 +19,73 @@ spec: - | #!/bin/bash set -e + + INIT_DATA_PATH="/mnt/db-init-data" + echo "Waiting for database to be ready..." until psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "SELECT 1"; do echo "Database not ready yet, waiting..." sleep 5 done + echo "Database ready, initializing..." + psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" < /tmp/request.json cat /tmp/schema.json >> /tmp/request.json echo '}' >> /tmp/request.json curl -X POST --header "Content-Type: application/json" --data @/tmp/request.json http://{{ .Values.serviceName }}:8201/v1/graph/{{.Values.graphName}}/jsonschema + vertex_labels=$(curl -s -X GET "http://{{.Values.serviceName}}:8201/v1/graph/{{.Values.graphName}}__schema__/label" | sed -n 's/.*"vertexLabels":\[\([^]]*\)\].*/\1/p' | sed 's/"//g' | tr ',' '\n') + + echo "$vertex_labels" | while read -r label; do + JSON_PAYLOAD="{\"field\": \"auth_resource_path\"}" + curl -s -X POST \ + "http://{{.Values.serviceName}}:8201/v1/graph/CALYPR/index/${label}" \ + -H 'Content-Type: application/json' \ + -d "${JSON_PAYLOAD}" + done + + + restartPolicy: OnFailure diff --git a/helm/grip/templates/service.yaml b/helm/grip/templates/service.yaml index c87dbaca8..6de3e94a6 100644 --- a/helm/grip/templates/service.yaml +++ b/helm/grip/templates/service.yaml @@ -11,6 +11,11 @@ spec: targetPort: http protocol: TCP name: http + + - port: 8202 + targetPort: rpc + protocol: TCP + name: rpc selector: {{- include "grip.selectorLabels" . | nindent 4 }} diff --git a/helm/guppy/templates/guppy_config.yaml b/helm/guppy/templates/guppy_config.yaml index 6c018d42b..2d271b9a0 100644 --- a/helm/guppy/templates/guppy_config.yaml +++ b/helm/guppy/templates/guppy_config.yaml @@ -12,5 +12,6 @@ data: {{- end }} "auth_filter_field": {{ .Values.authFilterField | quote }}, "enable_encrypt_whitelist": {{ .Values.enableEncryptWhitelist | quote }}, - "encrypt_whitelist": {{ .Values.encryptWhitelist | quote }} + "encrypt_whitelist": {{ .Values.encryptWhitelist | quote }}, + "log_level": {{.Values.log_level | quote}} } \ No newline at end of file diff --git a/helm/indexd/indexd-settings/local_settings.py b/helm/indexd/indexd-settings/local_settings.py index 42d74247e..30fc5f906 100644 --- a/helm/indexd/indexd-settings/local_settings.py +++ b/helm/indexd/indexd-settings/local_settings.py @@ -62,4 +62,8 @@ ) ) +rbac = environ.get("RBAC", "false").lower() == "true" +if rbac: + CONFIG["RBAC"] = True + settings = {"config": CONFIG, "auth": AUTH} diff --git a/helm/indexd/templates/deployment.yaml b/helm/indexd/templates/deployment.yaml index 497d4f456..ac1698545 100644 --- a/helm/indexd/templates/deployment.yaml +++ b/helm/indexd/templates/deployment.yaml @@ -52,13 +52,13 @@ spec: secretKeyRef: name: indexd-dbcreds key: host - optional: false + optional: false - name: PGUSER valueFrom: secretKeyRef: name: indexd-dbcreds key: username - optional: false + optional: false - name: PGPASSWORD valueFrom: secretKeyRef: diff --git a/helm/indexd/templates/pre-install.yaml b/helm/indexd/templates/pre-install.yaml deleted file mode 100644 index f6c2e3586..000000000 --- a/helm/indexd/templates/pre-install.yaml +++ /dev/null @@ -1,96 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - # - # A little helper jobs for creating new indexd - # users with basic-auth access to indexd update - # endpoints. - # Loads the list of users from env var. - name: indexd-userdb -spec: - backoffLimit: 0 - template: - metadata: - labels: - app: gen3job - spec: - automountServiceAccountToken: false - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - initContainers: - - name: wait-for-indexd - image: curlimages/curl:latest - command: ["/bin/sh","-c"] - args: ["while [ $(curl -sw '%{http_code}' http://indexd-service/index -o /dev/null) -ne 200 ]; do sleep 5; echo 'Waiting for indexd...'; done"] - containers: - - name: indexd - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - volumeMounts: - - name: "config-volume" - readOnly: true - mountPath: "/var/www/indexd/local_settings.py" - subPath: "local_settings.py" - env: - - name: PGHOST - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: host - optional: false - - name: PGUSER - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: username - optional: false - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: password - optional: false - - name: PGDB - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: database - optional: false - - name: DBREADY - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: dbcreated - optional: false - - name: FENCE_PASS - valueFrom: - secretKeyRef: - name: indexd-service-creds - key: fence - optional: false - - name: SHEEPDOG_PASS - valueFrom: - secretKeyRef: - name: indexd-service-creds - key: sheepdog - optional: false - imagePullPolicy: Always - command: ["/bin/bash" ] - args: - - "-c" - # Script always succeeds if it runs (echo exits with 0) - # indexd image does not include jq, so use python - - | - if command -v python &> /dev/null; then - echo 'python /indexd/bin/index_admin.py create --username "fence" --password "${FENCE_PASS}' - python /indexd/bin/index_admin.py create --username "fence" --password "${FENCE_PASS}" - echo 'python /indexd/bin/index_admin.py create --username "sheepdog" --password "${SHEEPDOG_PASS}' - python /indexd/bin/index_admin.py create --username "sheepdog" --password "${SHEEPDOG_PASS}" - else - echo 'poetry run python /indexd/bin/index_admin.py create --username "fence" --password "${FENCE_PASS}' - poetry run python /indexd/bin/index_admin.py create --username "fence" --password "${FENCE_PASS}" - echo 'poetry run python /indexd/bin/index_admin.py create --username "sheepdog" --password "${SHEEPDOG_PASS}' - poetry run python /indexd/bin/index_admin.py create --username "sheepdog" --password "${SHEEPDOG_PASS}" - fi - restartPolicy: Never \ No newline at end of file diff --git a/helm/revproxy/gen3.nginx.conf/gecko-service.conf b/helm/revproxy/gen3.nginx.conf/gecko-service.conf index 32195a4c1..3fa3168b2 100644 --- a/helm/revproxy/gen3.nginx.conf/gecko-service.conf +++ b/helm/revproxy/gen3.nginx.conf/gecko-service.conf @@ -11,6 +11,7 @@ location /ExplorerConfig/health { proxy_pass $upstream; } + location /ExplorerConfig/list { proxy_connect_timeout 600s; proxy_send_timeout 600s; @@ -38,4 +39,34 @@ location /ExplorerConfig/ { rewrite ^/ExplorerConfig/(.*)$ /config/$1 break; proxy_pass $upstream$is_args$args; client_max_body_size 0; +} + +location ~* ^/(Vector|vector)/ { + proxy_connect_timeout 600s; + proxy_send_timeout 600s; + proxy_read_timeout 600s; + send_timeout 600s; + + set $proxy_service "gecko"; + set $upstream http://gecko-service.$namespace.svc.cluster.local; + + rewrite ^/(Vector|vector)/(.*)$ /vector/$2 break; + proxy_pass $upstream; + client_max_body_size 0; +} + +location ~* ^/Dir { + proxy_connect_timeout 600s; + proxy_send_timeout 600s; + proxy_read_timeout 600s; + send_timeout 600s; + + set $proxy_service "gecko"; + set $upstream http://gecko-service.$namespace.svc.cluster.local; + + rewrite ^/Dir(.*)$ /dir$1 break; + proxy_pass $upstream; + client_max_body_size 0; + + } \ No newline at end of file diff --git a/helm/revproxy/gen3.nginx.conf/indexd-service.conf b/helm/revproxy/gen3.nginx.conf/indexd-service.conf index a464afccb..20f9414a7 100644 --- a/helm/revproxy/gen3.nginx.conf/indexd-service.conf +++ b/helm/revproxy/gen3.nginx.conf/indexd-service.conf @@ -1,57 +1,26 @@ +location /ga4gh/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } - # GA4GH endpoint for DOS resolver and DRS server - location /ga4gh/ { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } + set $proxy_service "indexd"; + set $upstream http://indexd-service$des_domain; - set $proxy_service "indexd"; - set $upstream http://indexd-service$des_domain; - proxy_pass $upstream; - proxy_redirect http://$host/ https://$host/; - } + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/; +} - location /index/ { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } +location /index/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } - set $proxy_service "indexd"; - set $upstream http://indexd-service$des_domain; - rewrite ^/index/(.*) /$1 break; - proxy_pass $upstream; - proxy_redirect http://$host/ https://$host/index/; - } + set $proxy_service "indexd"; + set $upstream http://indexd-service$des_domain; - location /index-admin/ { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } - set $authz_resource "/indexd_gateway"; - set $authz_method "access"; - set $authz_service "indexd_gateway"; - # be careful - sub-request runs in same context as this request - auth_request /gen3-authz; + rewrite ^/index/(.*) /$1 break; - # - # For some reason nginx breaks the proxy body - # if we try to set Authorization from a perl_set variable - # that samples the environment ... ugh! - # - set $indexd_password "Basic ${indexd_b64}"; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/index/; +} - # For testing: - #add_header Set-Cookie "X-Frickjack=${indexd_password};Path=/;Max-Age=600"; - set $proxy_service "indexd"; - set $upstream http://indexd-service$des_domain; - rewrite ^/index-admin/(.*) /$1 break; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For "$realip"; - proxy_set_header X-UserId "$userid"; - proxy_set_header X-SessionId "$session_id"; - proxy_set_header X-VisitorId "$visitor_id"; - proxy_set_header Authorization "$indexd_password"; - - proxy_pass $upstream; - proxy_redirect http://$host/ https://$host/index-admin/; - } diff --git a/helm/revproxy/templates/ingress_dev.yaml b/helm/revproxy/templates/ingress_dev.yaml index df2ea60c8..1ebeac551 100644 --- a/helm/revproxy/templates/ingress_dev.yaml +++ b/helm/revproxy/templates/ingress_dev.yaml @@ -7,16 +7,18 @@ spec: {{- if .Values.global.dev }} tls: - secretName: gen3-certs + hosts: + - {{ default .Values.global.hostname .Values.hostname | quote }} # <--- This maps the cert to the domain {{- end }} rules: - - host: {{ default .Values.global.hostname .Values.hostname }} + - host: {{ default .Values.global.hostname .Values.hostname | quote }} http: paths: - path: / pathType: Prefix backend: service: - name: revproxy-service + name: revproxy-service port: number: 80 - {{- end }} +{{- end }} \ No newline at end of file