feat(ci): Make deploy script cleaner; use temp file, add logs (#2808)

deploy.py

@@ -4,6 +4,7 @@
 import os
 import subprocess
 import sys
+import tempfile
 import time
 from pathlib import Path
 
@@ -14,6 +15,7 @@
 
 try:
     import yaml
+
 except ImportError as e:
     msg = (
         f"pyyaml is not installed but required by ./deploy.py\n"
@@ -24,7 +26,6 @@
 
 script_path = Path(__file__).resolve()
 ROOT_DIR = script_path.parent
-TEMP_DIR = ROOT_DIR / "temp"
 
 
 CLUSTER_NAME = "testCluster"
@@ -247,18 +248,16 @@ def get_codespace_name():
 
 
 def generate_configs(from_live=False):
-    TEMP_DIR.mkdir(parents=True, exist_ok=True)
+    temp_dir_path = Path(tempfile.mkdtemp())
 
-    # Delete all files in the temp directory
-    for file in TEMP_DIR.iterdir():
-        file.unlink()
+    print(f"Unprocessed config available in temp dir: {temp_dir_path}")
 
     helm_chart = str(HELM_CHART_DIR)
     codespace_name = get_codespace_name()
 
     output_dir = ROOT_DIR / "website" / "tests" / "config"
 
-    backend_config_path = TEMP_DIR / "backend_config.json"
+    backend_config_path = temp_dir_path / "backend_config.json"
     generate_config(
         helm_chart,
         "templates/loculus-backend-config.yaml",
@@ -267,7 +266,7 @@ def generate_configs(from_live=False):
         from_live,
     )
 
-    website_config_path = TEMP_DIR / "website_config.json"
+    website_config_path = temp_dir_path / "website_config.json"
     generate_config(
         helm_chart,
         "templates/loculus-website-config.yaml",
@@ -276,7 +275,7 @@ def generate_configs(from_live=False):
         from_live,
     )
 
-    runtime_config_path = TEMP_DIR / "runtime_config.json"
+    runtime_config_path = temp_dir_path / "runtime_config.json"
     generate_config(
         helm_chart,
         "templates/loculus-website-config.yaml",
@@ -285,9 +284,9 @@ def generate_configs(from_live=False):
         from_live,
     )
 
-    ingest_configmap_path = TEMP_DIR / "config.yaml"
+    ingest_configmap_path = temp_dir_path / "config.yaml"
     ingest_template_path = "templates/ingest-config.yaml"
-    ingest_configout_path = TEMP_DIR / "ingest-config.yaml"
+    ingest_configout_path = temp_dir_path / "ingest-config.yaml"
     generate_config(
         helm_chart,
         ingest_template_path,
@@ -297,9 +296,9 @@ def generate_configs(from_live=False):
         ingest_configout_path,
     )
 
-    prepro_configmap_path = TEMP_DIR / "preprocessing-config.yaml"
+    prepro_configmap_path = temp_dir_path / "preprocessing-config.yaml"
     prepro_template_path = "templates/loculus-preprocessing-config.yaml"
-    prepro_configout_path = TEMP_DIR / "preprocessing-config.yaml"
+    prepro_configout_path = temp_dir_path / "preprocessing-config.yaml"
     generate_config(
         helm_chart,
         prepro_template_path,
@@ -313,10 +312,11 @@ def generate_configs(from_live=False):
         [
             "python3",
             "kubernetes/config-processor/config-processor.py",
-            TEMP_DIR,
+            temp_dir_path,
             output_dir,
         ]
     )
+    print(f"Config generation succeeded, processed config files available in {output_dir}")
 
 
 def generate_config(