add grip indexing to initdb job

.github/workflows/lint_test.yaml

@@ -17,6 +17,13 @@ jobs:
         with:
           version: v3.10.0
 
+      - name: Add Helm repos
+        run: |
+          helm repo add elastic https://helm.elastic.co
+          helm repo add bitnami https://charts.bitnami.com/bitnami
+          helm repo add qdrant https://qdrant.github.io/qdrant-helm
+          helm repo update
+
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9'

helm/fence/templates/useryaml-job.yaml

@@ -43,7 +43,6 @@ spec:
           - "-c"
           # Script always succeeds if it runs (echo exits with 0)
           - |
-            pip3 install SQLAlchemy==1.3.6
             # can be removed once this is merged: https://github.com/uc-cdis/fence/pull/1096
             fence-create sync --arborist http://arborist-service --yaml /var/www/fence/user.yaml
       restartPolicy: OnFailure

helm/gecko/files/init-data/apps_page.json

@@ -0,0 +1,25 @@
+{
+  "appCards": [
+    {
+      "title": "Explore BForePC",
+      "description": "Explore BForePC data in the CALYPR system",
+      "icon": "/icons/binoculars.svg",
+      "href": "/Explorer/HTAN_INT-BForePC",
+      "perms": "HTAN_INT-BForePC"
+    },
+    {
+      "title": "Explore MMTERT SMMART Data",
+      "description": "Explore MMTERT SMMART data in the CALYPR system",
+      "icon": "/icons/binoculars.svg",
+      "href": "/Explorer/cbds-smmart_labkey_demo",
+      "perms": "cbds-smmart_labkey_demo"
+    },
+    {
+      "title": "SMMART Report",
+      "description": "A clinical research study measuring tumor evolution and its resistance to therapy to deliver better cancer treatment. Overview of SMMART data processing",
+      "icon": "/icons/chart-bar.svg",
+      "href": "/SMCLICK",
+      "perms": "cbds-smmart_labkey_demo"
+    }
+  ]
+}

helm/gecko/files/init-data/file_summary.json

@@ -0,0 +1,25 @@
+{
+  "config": {
+    "document_reference_title": {
+      "title": "Title",
+      "field": "document_reference_title"
+    },
+    "document_reference_size": {
+      "cellRenderFunction": "HumanReadableString",
+      "type": "string",
+      "title": "File Size",
+      "field": "document_reference_size"
+    },
+    "document_reference_source_path": {
+      "title": "Source Path",
+      "field": "document_reference_source_path"
+    }
+  },
+  "binslicePoints": [
+    0, 1048576, 524288000, 1073741824, 107374182400, 9007199254740991
+  ],
+  "barChartColor": "#e9724d",
+  "defaultProject": "gdc-esca",
+  "idField": "document_reference_id",
+  "index": "document_reference"
+}

helm/gecko/files/init-data/nav.json

@@ -0,0 +1,106 @@
+{
+  "headerProps": {
+    "topBar": {
+      "items": [
+        {
+          "href": "https://www.ohsu.edu/knight-cancer-institute",
+          "name": "CBDS",
+          "classNames": {
+            "root": "",
+            "label": "",
+            "button": ""
+          }
+        }
+      ],
+      "loginButtonVisibility": "hidden"
+    },
+    "navigation": {
+      "classNames": {
+        "root": "bg-base-max text-primary opacity-100 hover:opacity-100",
+        "item": "py-2 px-4 hover:bg-base-lightest hover:text-base-contrast",
+        "navigationPanel": "bg-base-max text-primary"
+      },
+      "logo": {
+        "src": "/icons/ohsu.svg",
+        "width": 52.5,
+        "height": 40,
+        "href": "/Apps",
+        "title": "CALYPR"
+      },
+      "items": [
+        {
+          "icon": "gen3:exploration",
+          "href": "/Explorer",
+          "name": "Exploration",
+          "tooltip": "The Exploration Page enables discovery of the data at the subject level and features a cohort builder.",
+          "title": "Explorer"
+        },
+        {
+          "icon": "gen3:profile",
+          "href": "/Profile",
+          "name": "Profile",
+          "tooltip": "Create API keys for programmatic data access, and review your authorization privileges to datasets and services.",
+          "title": "Profile"
+        }
+      ]
+    },
+    "leftnav": [
+      {
+        "title": "Home",
+        "description": "Home Apps page",
+        "icon": "/icons/home.svg",
+        "href": "/Apps",
+        "perms": null
+      },
+      {
+        "title": "Directory Structure",
+        "description": "Search for files via a tree based interactive search",
+        "icon": "/icons/binary-tree.svg",
+        "href": "/Miller",
+        "perms": null
+      },
+      {
+        "title": "File Summary",
+        "description": "Overview of file system usage",
+        "icon": "/icons/file.svg",
+        "href": "/Filesummary",
+        "perms": null
+      },
+      {
+        "title": "Image Viewer",
+        "description": "View available .ome.tif images using Avivator",
+        "icon": "/icons/layers-intersect.svg",
+        "href": "/AvailableImages",
+        "perms": null
+      }
+    ]
+  },
+  "footerProps": {
+    "classNames": {
+      "root": "",
+      "layout": "flex items-center justify-center"
+    },
+    "rightSection": {
+      "columns": [
+        {
+          "rows": [
+            {
+              "Icon": {
+                "logo": "/icons/knight.svg",
+                "logolight": "/icons/knight_white.svg",
+                "width": 100,
+                "height": 47,
+                "description": "Knight Cancer Institute"
+              }
+            }
+          ]
+        }
+      ]
+    }
+  },
+  "headerMetadata": {
+    "title": "CALYPR",
+    "content": "Cancer Analytics Platform",
+    "key": "calypr-main"
+  }
+}

helm/gecko/templates/db-init-job.yaml

@@ -19,19 +19,73 @@ spec:
             - |
               #!/bin/bash
               set -e
+
+              INIT_DATA_PATH="/mnt/db-init-data"
+
               echo "Waiting for database to be ready..."
               until psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "SELECT 1"; do
                 echo "Database not ready yet, waiting..."
                 sleep 5
               done
+
               echo "Database ready, initializing..."
+
               psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" <<EOF
               \c gecko_local
-              CREATE TABLE IF NOT EXISTS documents (
-                  name VARCHAR(255) PRIMARY KEY,
-                  content JSONB
-              );
+
+              CREATE SCHEMA IF NOT EXISTS config_schema;
+              CREATE OR REPLACE FUNCTION create_config_table(table_name TEXT)
+              RETURNS void AS '
+              BEGIN
+                  EXECUTE format(''
+                      CREATE TABLE IF NOT EXISTS config_schema.%I (
+                          name VARCHAR(255) PRIMARY KEY,
+                          content JSONB
+                      );
+                  '', table_name);
+              END;
+              ' LANGUAGE plpgsql;
+
+              SELECT create_config_table('explorer');
+              SELECT create_config_table('nav');
+              SELECT create_config_table('file_summary');
+              SELECT create_config_table('apps_page');
+
+              DROP FUNCTION create_config_table(TEXT);
               EOF
+
+              echo "Inserting initial configuration data from ConfigMap..."
+
+              # Insert NAV config (using $(cat FILE) to read JSON content)
+              psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "
+                INSERT INTO config_schema.nav (name, content)
+                VALUES (
+                  '1',
+                  '$(cat ${INIT_DATA_PATH}/nav.json)'::jsonb
+                )
+                ON CONFLICT (name) DO UPDATE SET content = EXCLUDED.content;
+              "
+
+              # Insert FILE_SUMMARY config
+              psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "
+                INSERT INTO config_schema.file_summary (name, content)
+                VALUES (
+                  '1',
+                  '$(cat ${INIT_DATA_PATH}/file_summary.json)'::jsonb
+                )
+                ON CONFLICT (name) DO UPDATE SET content = EXCLUDED.content;
+              "
+
+              # Insert APPS_PAGE config
+              psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "
+                INSERT INTO config_schema.apps_page (name, content)
+                VALUES (
+                  '1',
+                  '$(cat ${INIT_DATA_PATH}/apps_page.json)'::jsonb
+                )
+                ON CONFLICT (name) DO UPDATE SET content = EXCLUDED.content;
+              "
+
               echo "Database initialization complete."
           env:
             - name: PGHOST
@@ -66,11 +120,24 @@ spec:
                   optional: false
             - name: PGSSLMODE
               value: "disable"
+          volumeMounts:
+            # Mount the ConfigMap containing the JSON files
+            - name: db-init-data-volume
+              mountPath: /mnt/db-init-data
+              readOnly: true
+
+      # Define the volume that pulls content from the ConfigMap
+      volumes:
+        - name: db-init-data-volume
+          configMap:
+            name: {{ include "gecko.fullname" . }}-db-init-data
+            optional: false
+
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+      {{- end }}

helm/gecko/templates/deployment.yaml

@@ -46,6 +46,7 @@ spec:
             httpGet:
               path: /health
               port: 80
+
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           command: ["sh"]
@@ -61,6 +62,28 @@ spec:
 
           env:
             {{- toYaml .Values.env | nindent 12 }}
+            - name: GRIP_GRAPH
+              valueFrom:
+                configMapKeyRef:
+                  name: grip-values
+                  key: graphName
+            - name: GRIP_HOST
+              valueFrom:
+                configMapKeyRef:
+                  name: grip-values
+                  key: serviceName
+            - name: GRIP_PORT
+              value: "8202"
+            - name: QDRANT_HOST
+              value: {{ printf "%s-qdrant" .Release.Name | quote }}
+            - name: QDRANT_PORT
+              value: "6334"
+            - name: QDRANT_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{  "qdrant-api-key-secret" }}
+                  key: {{  "api-key" }}
+                  optional: false
             - name: PGPASSWORD
               valueFrom:
                 secretKeyRef:

helm/gecko/templates/job-db-init.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "gecko.fullname" . }}-db-init-data
+data:
+  # Reads content from the chart filesystem at files/init-data/nav.json
+  nav.json: |
+{{ .Files.Get "files/init-data/nav.json" | trim | nindent 4 }}
+
+  # Reads content from the chart filesystem at files/init-data/file_summary.json
+  file_summary.json: |
+{{ .Files.Get "files/init-data/file_summary.json" | trim | nindent 4 }}
+
+  # Reads content from the chart filesystem at files/init-data/apps_page.json
+  apps_page.json: |
+{{ .Files.Get "files/init-data/apps_page.json" | trim | nindent 4 }}

helm/gecko/templates/qdrant-pv.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: qdrant-storage-pv
+  labels:
+    app.kubernetes.io/managed-by: "Helm"
+  annotations:
+    meta.helm.sh/release-name: {{ .Release.Name }}
+    meta.helm.sh/release-namespace: "default"
+spec:
+  capacity:
+    storage: 26Gi
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: "qdrant-manual-storage"
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+    path: "/mnt/data/qdrant-local"

helm/gen3/Chart.yaml

@@ -111,6 +111,9 @@ dependencies:
   version: 0.1.0
   repository: "file://../viv"
   condition: viv.enabled
+- name: qdrant
+  version: 1.15.4
+  repository: "https://qdrant.github.io/qdrant-helm"
 
 # A chart can be either an 'application' or a 'library' chart.
 #

helm/gen3/templates/qdrant-api-key-secret.yaml

@@ -0,0 +1,26 @@
+{{- /* Generate a random, persistent Qdrant API Key.
+This logic lives in the parent chart to avoid being overwritten by 'helm dep update'.
+*/}}
+{{- $secretName := "qdrant-api-key-secret" -}}
+{{- $secretKey := "api-key" -}}
+{{- $existingSecret := (lookup "v1" "Secret" .Release.Namespace $secretName) -}}
+
+# Only render the Secret if qdrant is enabled in the values file
+{{- if .Values.qdrant.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    # Use a label helper from the 'common' subchart or define your own
+    {{- include "common.labels.standard" . | nindent 4 }} 
+type: Opaque
+data:
+  # Use the lookup function to ensure persistence across upgrades
+  {{- if $existingSecret }}
+  {{ $secretKey }}: {{ index $existingSecret.data $secretKey }}
+  {{- else }}
+  {{ $secretKey }}: {{ randAlphaNum 32 | b64enc | quote }}
+  {{- end }}
+{{- end }}