Merge branch 'main' into search-refactor

.github/workflows/website-tests.yml

@@ -35,7 +35,7 @@ jobs:
       - run: npm ci
       - run: npm run check-format
       - run: npm run check-types
-
+        if: always()
   unitTests:
     name: Unit Tests
     runs-on: codebuild-loculus-ci-${{ github.run_id }}-${{ github.run_attempt }}

backend/build.gradle

@@ -40,7 +40,7 @@ dependencies {
     implementation "org.jetbrains.exposed:exposed-jdbc:0.50.1"
     implementation "org.jetbrains.exposed:exposed-json:0.50.1"
     implementation "org.jetbrains.exposed:exposed-kotlin-datetime:0.50.1"
-    implementation "org.jetbrains.kotlinx:kotlinx-datetime:0.5.0"
+    implementation "org.jetbrains.kotlinx:kotlinx-datetime:0.6.0"
     implementation "org.hibernate.validator:hibernate-validator:8.0.1.Final"
     implementation "org.keycloak:keycloak-admin-client:23.0.7"
 

deploy.py

@@ -188,6 +188,10 @@ def get_codespace_name():
 def generate_configs(from_live=False):
     TEMP_DIR.mkdir(parents=True, exist_ok=True)
 
+    # Delete all files in the temp directory
+    for file in TEMP_DIR.iterdir():
+        file.unlink()
+
     helm_chart = str(HELM_CHART_DIR)
     codespace_name = get_codespace_name()
 
@@ -202,11 +206,24 @@ def generate_configs(from_live=False):
     runtime_config_path =  TEMP_DIR / 'runtime_config.json'
     generate_config(helm_chart, 'templates/loculus-website-config.yaml', runtime_config_path, codespace_name, from_live)
 
+    ingest_configmap_path = TEMP_DIR / 'config.yaml'
+    ingest_template_path = 'templates/loculus-ingest-config.yaml'
+    ingest_configout_path = TEMP_DIR / 'ingest-config.yaml'
+    generate_config(helm_chart, ingest_template_path, ingest_configmap_path, codespace_name, from_live, ingest_configout_path)
+
+    prepro_configmap_path = TEMP_DIR / 'preprocessing-config.yaml'
+    prepro_template_path = 'templates/loculus-preprocessing-config.yaml'
+    prepro_configout_path = TEMP_DIR / 'preprocessing-config.yaml'
+    generate_config(helm_chart, prepro_template_path, prepro_configmap_path, codespace_name, from_live, prepro_configout_path)
+
     run_command(['python3', 'kubernetes/config-processor/config-processor.py', TEMP_DIR, output_dir])
 
-def generate_config(helm_chart, template, output_path, codespace_name=None, from_live=False):
+def generate_config(helm_chart, template, configmap_path, codespace_name=None, from_live=False, output_path=None):
     helm_template_cmd = ['helm', 'template', 'name-does-not-matter', helm_chart, '--show-only', template]
 
+    if not output_path:
+        output_path = configmap_path
+
     if codespace_name:
         helm_template_cmd.extend(['--set', 'codespaceName='+codespace_name])
 
@@ -219,17 +236,25 @@ def generate_config(helm_chart, template, output_path, codespace_name=None, from
         helm_template_cmd.extend(['--set', 'usePublicRuntimeConfigAsServerSide=true'])
     else:
         helm_template_cmd.extend(['--set', 'environment=local'])
+        helm_template_cmd.extend(['--set', 'testconfig=true'])
     helm_output = run_command(helm_template_cmd, capture_output=True, text=True).stdout
     if args.dry_run:
         return
 
-    parsed_yaml = yaml.safe_load(helm_output)
-    config_data = parsed_yaml['data'][output_path.name]
+    parsed_yaml = list(yaml.full_load_all(helm_output))
+    if len(parsed_yaml) == 1:
+        config_data = parsed_yaml[0]['data'][configmap_path.name]
 
-    with open(output_path, 'w') as f:
-        f.write(config_data)
+        with open(output_path, 'w') as f:
+            f.write(config_data)
 
-    print(f"Wrote config to {output_path}")
+        print(f"Wrote config to {output_path}")
+    elif any(substring in template for substring in ["ingest", "preprocessing"]):
+        for doc in parsed_yaml:
+            config_data = yaml.safe_load(doc['data'][configmap_path.name])
+            with open(output_path.with_suffix(f'.{config_data["organism"]}.yaml'), 'w') as f:
+                yaml.dump(config_data, f)
+                print(f"Wrote config to {f.name}")
 
 def install_secret_generator():
     add_helm_repo_command = [