-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: expand default metadata fields, make values.yaml concise throug…
…h template functions and yaml anchors (#1935) - Adds more metadata fields to be used by pathoplexus to values.yaml - Configure a single abstract default organism in values.yaml that is reused by concrete ones to make values.yaml shorter (only what's differing between organisms needs to be duplicated). Inheritance is currently done via yaml anchor, aliases and merge keys - Make values.yaml more compact by generating the previously separate a) ingest, b) preprocessing and c) inputFields configs from schema.metadata. - Use default values in values.yaml to make it more compact, e.g. string is the default type, so one less line for most metadata fields to define. - Introduce "header: Other" as the default header so as not to show fields under no header Without the compactification, we'd be looking at multiple thousands of lines of yaml, which is not nice to work with (already now VS code and vim complain about the size)
- Loading branch information
1 parent
698b836
commit 1b7497a
Showing
19 changed files
with
1,205 additions
and
1,472 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| {{- define "loculus.ingestRename" -}} | ||
| {{- $metadata := . }} | ||
| {{- $ingestRename := dict }} | ||
| {{- range $field := $metadata }} | ||
| {{- if hasKey $field "ingest" }} | ||
| {{- $_ := set $ingestRename (index $field "ingest") (index $field "name") }} | ||
| {{- end }} | ||
| {{- end }} | ||
| {{- $output := dict "rename" $ingestRename }} | ||
| {{- toYaml $output }} | ||
| {{- end -}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| {{- define "loculus.inputFields" -}} | ||
| {{- $data := . }} | ||
| {{- $metadata := $data.metadata }} | ||
| {{- $extraFields := $data.extraInputFields }} | ||
| {{- $TO_KEEP := list "name" "displayName" "definition" "guidance" "example" "required" }} | ||
|
|
||
|
|
||
| {{- $fieldsDict := dict }} | ||
| {{- $index := 0 }} | ||
|
|
||
| {{- /* Add fields with position "first" to the dict */}} | ||
| {{- range $field := $extraFields }} | ||
| {{- if eq $field.position "first" }} | ||
| {{- $_ := set $fieldsDict (printf "%03d" $index) $field }} | ||
| {{- $index = add $index 1 }} | ||
| {{- end }} | ||
| {{- end }} | ||
|
|
||
| {{- /* Add filtered metadata fields to the dict */}} | ||
| {{- range $field := $metadata }} | ||
| {{- if not (hasKey $field "noInput") }} | ||
| {{- $_ := set $fieldsDict (printf "%03d" $index) $field }} | ||
| {{- $index = add $index 1 }} | ||
| {{- end }} | ||
| {{- end }} | ||
|
|
||
| {{- /* Add fields with position "last" to the dict */}} | ||
| {{- range $field := $extraFields }} | ||
| {{- if eq $field.position "last" }} | ||
| {{- $_ := set $fieldsDict (printf "%03d" $index) $field }} | ||
| {{- $index = add $index 1 }} | ||
| {{- end }} | ||
| {{- end }} | ||
|
|
||
| {{- /* Iterate over sorted index to get list of values (sorted by key) */}} | ||
| {{- $inputFields := list }} | ||
| {{- range $k:= keys $fieldsDict | sortAlpha }} | ||
| {{- $toAdd := dict }} | ||
| {{- range $k, $v := (index $fieldsDict $k) }} | ||
| {{- if has $k $TO_KEEP }} | ||
| {{- $_ := set $toAdd $k $v }} | ||
| {{- end }} | ||
| {{- end }} | ||
| {{- $inputFields = append $inputFields $toAdd }} | ||
| {{- end }} | ||
|
|
||
| {{- toYaml $inputFields }} | ||
| {{- end -}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| {{- define "loculus.preprocessingSpecs" -}} | ||
| {{- $metadata := . }} | ||
| {{- $specs := dict }} | ||
|
|
||
| {{- range $field := $metadata }} | ||
| {{- $name := index $field "name" }} | ||
| {{- $spec := dict "function" "identity" "inputs" (dict "input" $name) }} | ||
|
|
||
| {{- if hasKey $field "type" }} | ||
| {{- $type := index $field "type" }} | ||
| {{- if eq $type "int" }} | ||
| {{- $_ := set $spec "args" (dict "type" "int") }} | ||
| {{- else if eq $type "float" }} | ||
| {{- $_ := set $spec "args" (dict "type" "float") }} | ||
| {{- end }} | ||
| {{- end }} | ||
|
|
||
| {{- if hasKey $field "preprocessing" }} | ||
| {{- $preprocessing := index $field "preprocessing" }} | ||
| {{- if eq (typeOf $preprocessing) "string" }} | ||
| {{- $_ := set $spec "inputs" (dict "input" $preprocessing) }} | ||
| {{- else }} | ||
| {{- if hasKey $preprocessing "function" }} | ||
| {{- $_ := set $spec "function" (index $preprocessing "function") }} | ||
| {{- end }} | ||
| {{- if hasKey $preprocessing "args" }} | ||
| {{- $_ := set $spec "args" (index $preprocessing "args") }} | ||
| {{- end }} | ||
| {{- if hasKey $preprocessing "inputs" }} | ||
| {{- $_ := set $spec "inputs" (index $preprocessing "inputs") }} | ||
| {{- end }} | ||
| {{- end }} | ||
| {{- end }} | ||
|
|
||
| {{- $_ := set $specs $name $spec }} | ||
| {{- end }} | ||
|
|
||
| {{- toYaml $specs }} | ||
| {{- end -}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| {{- $testconfig := .Values.testconfig | default false }} | ||
| {{- $backendHost := .Values.environment | eq "server" | ternary (printf "https://backend-%s" $.Values.host) ($testconfig | ternary "http://localhost:8079" "http://loculus-backend-service:8079") }} | ||
| {{- $keycloakHost := .Values.environment | eq "server" | ternary (printf "https://authentication-%s" $.Values.host) ($testconfig | ternary "http://localhost:8083" "http://loculus-keycloak-service:8083") }} | ||
| {{- range $key, $values := (.Values.organisms | default .Values.defaultOrganisms) }} | ||
| {{- if $values.ingest }} | ||
| {{- $metadata := (include "loculus.patchMetadataSchema" $values.schema | fromYaml).metadata }} | ||
| --- | ||
| apiVersion: v1 | ||
| kind: ConfigMap | ||
| metadata: | ||
| name: loculus-ingest-config-{{ $key }} | ||
| data: | ||
| config.yaml: | | ||
| {{- $values.ingest.configFile | toYaml | nindent 4 }} | ||
| organism: {{ $key }} | ||
| backend_url: {{ $backendHost }} | ||
| keycloak_token_url: {{ $keycloakHost -}}/realms/loculus/protocol/openid-connect/token | ||
| {{- include "loculus.ingestRename" $metadata | nindent 4 }} | ||
| {{- end }} | ||
| {{- end }} |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.