Feature/grip updates

etl.yaml

@@ -11,79 +11,31 @@ spec:
       configMap:
         name: gen3-credentials
 
-    - name: sheepdog-dbcreds
-      secret:
-        secretName: sheepdog-dbcreds
-
     - name: useryaml
       configMap:
         name: useryaml
 
-    - name: studies-mount
-      hostPath:
-        path: /Users/walsbr/aced/data_model/studies
-
-    - name: output-mount
-      hostPath:
-        path: /Users/walsbr/aced/data_model/output
 
   containers:
     - name: container-configmap
-      image: quay.io/ohsu-comp-bio/aced-etl:development
+      image: quay.io/ohsu-comp-bio/aced-etl:feature_parent-specimens
       imagePullPolicy: Always
       volumeMounts:
         # Gen3 credentials file
         - name: gen3-credentials
           mountPath: /creds/credentials.json
           subPath: credentials.json
 
-        # Sheepdog creds, secrets defined in helm/sheepdog/templates/deployment.yaml
-        - name: sheepdog-dbcreds
-          mountPath: /creds/sheepdog-creds
 
         # User yaml, configmap defined in helm/fence/templates/useryaml-job.yaml
         - name: useryaml
           mountPath: /creds/user.yaml
           subPath: useryaml
 
-        # studies metadata
-        - name: studies-mount
-          mountPath: /studies
-
-        # studies files
-        - name: output-mount
-          mountPath: /output
 
       command: ["/bin/bash" ]
       args:
         - "-c"
         - |
-          # Until we fix in helm env, setup well known postgres env variables
-          # TODO - set GLOBAL_HOSTNAME to value from values.yaml
-          cat  << EOF >> ~/.bashrc
-          export PGDB=`cat /creds/sheepdog-creds/database`
-          export PGPASSWORD=`cat /creds/sheepdog-creds/password`
-          export PGUSER=`cat /creds/sheepdog-creds/username`
-          export PGHOST=`cat /creds/sheepdog-creds/host`
-          export DBREADY=`cat /creds/sheepdog-creds/dbcreated`
-          export PGPORT=`cat /creds/sheepdog-creds/port`
-          EOF
-
-          # link back to mounted volumes
-          ln -s /studies /data_model/studies
-          ln -s /output /data_model/output
-
-          echo "Done setting up data_model"
-
-          # link back to mounted volumes
-          ln -s /studies /data_model/studies
-          ln -s /output /data_model/output
-
-          echo "Done setting up data_model"
-
-          # For development purposes, extends the lifespan of the pod to let us exec into it.
           while true; do sleep 30; done;
 
-  # TODO - read nodegroup from values.yaml
-  #nodeSelector:
-  #      eks.amazonaws.com/nodegroup: aced-commons-development-etl-node-group-large

helm/aws-es-proxy/templates/elasticsearch-pv.yaml

@@ -8,7 +8,7 @@ spec:
   storageClassName: local-storage
   volumeMode: Filesystem
   capacity:
-    storage: 30Gi # Adjust the size as needed
+    storage: {{ .Values.persistence.capacity | default "30Gi" }}
   accessModes:
     - ReadWriteOnce # Adjust the access mode as needed
   persistentVolumeReclaimPolicy: Retain

helm/common/templates/_capabilities.tpl

@@ -12,6 +12,20 @@ Return the target Kubernetes version
 {{- default (default .Capabilities.KubeVersion.Version .Values.kubeVersion) ((.Values.global).kubeVersion) -}}
 {{- end -}}
 
+{{/*
+Return true if the apiVersion is supported
+Usage:
+{{ include "common.capabilities.apiVersions.has" (dict "version" "batch/v1" "context" $) }}
+*/}}
+{{- define "common.capabilities.apiVersions.has" -}}
+{{- $providedAPIVersions := default .context.Values.apiVersions ((.context.Values.global).apiVersions) -}}
+{{- if and (empty $providedAPIVersions) (.context.Capabilities.APIVersions.Has .version) -}}
+    {{- true -}}
+{{- else if has .version $providedAPIVersions -}}
+    {{- true -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Return the appropriate apiVersion for poddisruptionbudget.
 */}}
@@ -36,6 +50,18 @@ Return the appropriate apiVersion for networkpolicy.
 {{- end -}}
 {{- end -}}
 
+{{/*
+Return the appropriate apiVersion for job.
+*/}}
+{{- define "common.capabilities.job.apiVersion" -}}
+{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
+{{- if and (not (empty $kubeVersion)) (semverCompare "<1.21-0" $kubeVersion) -}}
+{{- print "batch/v1beta1" -}}
+{{- else -}}
+{{- print "batch/v1" -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Return the appropriate apiVersion for cronjob.
 */}}
@@ -157,14 +183,12 @@ Return the appropriate apiVersion for Vertical Pod Autoscaler.
 */}}
 {{- define "common.capabilities.vpa.apiVersion" -}}
 {{- $kubeVersion := include "common.capabilities.kubeVersion" .context -}}
-{{- if and (not (empty $kubeVersion)) (semverCompare "<1.23-0" $kubeVersion) -}}
-{{- if .beta2 -}}
-{{- print "autoscaling/v2beta2" -}}
-{{- else -}}
-{{- print "autoscaling/v2beta1" -}}
-{{- end -}}
+{{- if and (not (empty $kubeVersion)) (semverCompare "<1.11-0" $kubeVersion) -}}
+{{- print "autoscaling/v1beta1" -}}
+{{- else if and (not (empty $kubeVersion)) (semverCompare "<1.25-0" $kubeVersion) -}}
+{{- print "autoscaling/v1beta2" -}}
 {{- else -}}
-{{- print "autoscaling/v2" -}}
+{{- print "autoscaling/v1" -}}
 {{- end -}}
 {{- end -}}
 

helm/common/templates/_errors.tpl

@@ -1,28 +1,86 @@
 {{/*
-Copyright Broadcom, Inc. All Rights Reserved.
-SPDX-License-Identifier: APACHE-2.0
-*/}}
-
-{{/* vim: set filetype=mustache: */}}
-{{/*
-Through error when upgrading using empty passwords values that must not be empty.
-
-Usage:
-{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}}
-{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}}
-{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }}
-
-Required password params:
-  - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error.
-  - context - Context - Required. Parent context.
-*/}}
-{{- define "common.errors.upgrade.passwords.empty" -}}
-  {{- $validationErrors := join "" .validationErrors -}}
-  {{- if and $validationErrors .context.Release.IsUpgrade -}}
-    {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}}
-    {{- $errorString = print $errorString "\n                 Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}}
-    {{- $errorString = print $errorString "\n                 Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}}
-    {{- $errorString = print $errorString "\n%s" -}}
-    {{- printf $errorString $validationErrors | fail -}}
+  Copyright Broadcom, Inc. All Rights Reserved.
+  SPDX-License-Identifier: APACHE-2.0
+  */}}
+
+  {{/* vim: set filetype=mustache: */}}
+  {{/*
+  Throw error when upgrading using empty passwords values that must not be empty.
+
+  Usage:
+  {{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}}
+  {{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}}
+  {{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }}
+
+  Required password params:
+    - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error.
+    - context - Context - Required. Parent context.
+  */}}
+  {{- define "common.errors.upgrade.passwords.empty" -}}
+    {{- $validationErrors := join "" .validationErrors -}}
+    {{- if and $validationErrors .context.Release.IsUpgrade -}}
+      {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}}
+      {{- $errorString = print $errorString "\n                 Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}}
+      {{- $errorString = print $errorString "\n                 Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}}
+      {{- $errorString = print $errorString "\n%s" -}}
+      {{- printf $errorString $validationErrors | fail -}}
+    {{- end -}}
+  {{- end -}}
+
+  {{/*
+  Throw error when original container images are replaced.
+  The error can be bypassed by setting the "global.security.allowInsecureImages" to true. In this case,
+  a warning message will be shown instead.
+
+  Usage:
+  {{ include "common.errors.insecureImages" (dict "images" (list .Values.path.to.the.imageRoot) "context" $) }}
+  */}}
+  {{- define "common.errors.insecureImages" -}}
+  {{- $relocatedImages := list -}}
+  {{- $replacedImages := list -}}
+  {{- $retaggedImages := list -}}
+  {{- $globalRegistry := ((.context.Values.global).imageRegistry) -}}
+  {{- $originalImages := .context.Chart.Annotations.images -}}
+  {{- range .images -}}
+    {{- $registryName := default .registry $globalRegistry -}}
+    {{- $fullImageNameNoTag := printf "%s/%s" $registryName .repository -}}
+    {{- $fullImageName := printf "%s:%s" $fullImageNameNoTag .tag -}}
+    {{- if not (contains $fullImageNameNoTag $originalImages) -}}
+      {{- if not (contains $registryName $originalImages) -}}
+        {{- $relocatedImages = append $relocatedImages $fullImageName  -}}
+      {{- else if not (contains .repository $originalImages) -}}
+        {{- $replacedImages = append $replacedImages $fullImageName  -}}
+      {{- end -}}
+    {{- end -}}
+    {{- if not (contains (printf "%s:%s" .repository .tag) $originalImages) -}}
+      {{- $retaggedImages = append $retaggedImages $fullImageName  -}}
+    {{- end -}}
+  {{- end -}}
+
+  {{- if and (or (gt (len $relocatedImages) 0) (gt (len $replacedImages) 0)) (((.context.Values.global).security).allowInsecureImages) -}}
+    {{- print "\n\n⚠ SECURITY WARNING: Verifying original container images was skipped. Please note this Helm chart was designed, tested, and validated on multiple platforms using a specific set of Bitnami and Tanzu Application Catalog containers. Substituting other containers is likely to cause degraded security and performance, broken chart features, and missing environment variables.\n" -}}
+  {{- else if (or (gt (len $relocatedImages) 0) (gt (len $replacedImages) 0)) -}}
+    {{- $errorString := "Original containers have been substituted for unrecognized ones. Deploying this chart with non-standard containers is likely to cause degraded security and performance, broken chart features, and missing environment variables." -}}
+    {{- $errorString = print $errorString "\n\nUnrecognized images:" -}}
+    {{- range (concat $relocatedImages $replacedImages) -}}
+      {{- $errorString = print $errorString "\n  - " . -}}
+    {{- end -}}
+    {{- if or (contains "docker.io/bitnami/" $originalImages) (contains "docker.io/bitnamiprem/" $originalImages) -}}
+      {{- $errorString = print "\n\n⚠ ERROR: " $errorString -}}
+      {{- $errorString = print $errorString "\n\nIf you are sure you want to proceed with non-standard containers, you can skip container image verification by setting the global parameter 'global.security.allowInsecureImages' to true." -}}
+      {{- $errorString = print $errorString "\nFurther information can be obtained at https://github.com/bitnami/charts/issues/30850" -}}
+      {{- print $errorString | fail -}}
+    {{- else if gt (len $replacedImages) 0 -}}
+      {{- $errorString = print "\n\n⚠ WARNING: " $errorString -}}
+      {{- print $errorString -}}
+    {{- end -}}
+  {{- else if gt (len $retaggedImages) 0 -}}
+    {{- $warnString := "\n\n⚠ WARNING: Original containers have been retagged. Please note this Helm chart was tested, and validated on multiple platforms using a specific set of Tanzu Application Catalog containers. Substituting original image tags could cause unexpected behavior." -}}
+    {{- $warnString = print $warnString "\n\nRetagged images:" -}}
+    {{- range $retaggedImages -}}
+      {{- $warnString = print $warnString "\n  - " . -}}
+    {{- end -}}
+    {{- print $warnString -}}
+  {{- end -}}
   {{- end -}}
-{{- end -}}
+  

helm/gecko/Chart.yaml

@@ -0,0 +1,33 @@
+apiVersion: v2
+name: gecko
+description: A Helm chart for caliper gecko
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "master"
+
+dependencies:
+- name: common
+  version: 0.1.10
+  repository: file://../common
+- name: postgresql
+  version: 11.9.13
+  repository: "https://charts.bitnami.com/bitnami"
+  condition: postgres.separate

helm/gecko/templates/NOTES.txt

@@ -0,0 +1 @@
+{{ .Chart.Name }} has been deployed successfully.

helm/gecko/templates/_helpers.tpl

@@ -0,0 +1,81 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "gecko.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "gecko.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "gecko.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "gecko.labels" -}}
+{{- if .Values.commonLabels }}
+    {{- with .Values.commonLabels }}
+    {{- toYaml . }}
+    {{- end }}
+{{- else }}
+  {{- (include "common.commonLabels" .)}}
+{{- end }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "gecko.selectorLabels" -}}
+{{- if .Values.selectorLabels }}
+    {{- with .Values.selectorLabels }}
+    {{- toYaml . }}
+    {{- end }}
+{{- else }}
+  {{- (include "common.selectorLabels" .)}}
+{{- end }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "gecko.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "gecko.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+ Postgres Password lookup
+*/}}
+{{- define "gecko.postgres.password" -}}
+{{- $localpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}}
+{{- if $localpass }}
+{{- default (index $localpass.data "postgres-password" | b64dec) }}
+{{- else }}
+{{- default .Values.postgres.password }}
+{{- end }}
+{{- end }}
+