implement collect step for depicition pipeline

Author: leoschwarz

src/depiction_targeted_preproc/app_interface/app.yml

@@ -1,3 +1,4 @@
 commands:
   dispatch: python -m depiction_targeted_preproc.app_interface.dispatch_app
-  process: python -m depiction_targeted_preproc.app_interface.run_chunk
+  process: python -m depiction_targeted_preproc.app_interface.process_chunk
+  collect: python -m depiction_targeted_preproc.app_interface.collect_chunk

src/depiction_targeted_preproc/app_interface/collect_chunk.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+import cyclopts
+import yaml
+
+app = cyclopts.App()
+
+
+@app.default
+def collect_chunk(workunit_ref: int | Path, chunk_dir: Path) -> None:
+    chunk_dir = chunk_dir.absolute()
+    zip_file_path = chunk_dir / f"{chunk_dir.name}.zip"
+
+    outputs = [
+        {
+            "type": "bfabric_copy_resource",
+            "local_path": str(zip_file_path.absolute()),
+            "store_entry_path": zip_file_path.name,
+        }
+    ]
+    result = {"outputs": outputs}
+    with (chunk_dir / "outputs.yml").open("w") as f:
+        yaml.safe_dump(result, f)
+
+
+if __name__ == "__main__":
+    app()

src/depiction_targeted_preproc/app_interface/run_chunk.py renamed to src/depiction_targeted_preproc/app_interface/process_chunk.py

@@ -3,7 +3,6 @@
 
 import cyclopts
 import yaml
-
 from depiction_targeted_preproc.pipeline.prepare_params import Params
 from depiction_targeted_preproc.pipeline_config.artifacts_mapping import get_result_files_new
 from depiction_targeted_preproc.workflow.snakemake_invoke import SnakemakeInvoke
@@ -12,37 +11,25 @@
 
 
 @app.default()
-def run_chunk(chunk_dir: Path):
-    zip_file_path = run_workflow(sample_dir=chunk_dir)
-    outputs = [
-        {
-            "type": "bfabric_copy_resource",
-            "local_path": str(zip_file_path.absolute()),
-            "store_entry_path": zip_file_path.name,
-        }
-    ]
-    result = {"outputs": outputs}
-    with (chunk_dir / "outputs.yml").open("w") as f:
-        yaml.safe_dump(result, f)
-
-
-def run_workflow(sample_dir: Path) -> Path:
+def run_workflow(chunk_dir: Path) -> Path:
+    chunk_dir = chunk_dir.absolute()
+
     # TODO to be refactored
-    params = Params.model_validate(yaml.safe_load((sample_dir / "params.yml").read_text()))
-    result_files = get_result_files_new(requested_artifacts=params.requested_artifacts, sample_dir=sample_dir)
+    params = Params.model_validate(yaml.safe_load((chunk_dir / "params.yml").read_text()))
+    result_files = get_result_files_new(requested_artifacts=params.requested_artifacts, sample_dir=chunk_dir)
 
     # invoke snakemake
     # TODO note report file is deactivated because it's currently broken due to dependencies (jinja2)
-    SnakemakeInvoke(report_file=None).invoke(work_dir=sample_dir.parent, result_files=result_files)
+    SnakemakeInvoke(report_file=None).invoke(work_dir=chunk_dir.parent, result_files=result_files)
 
     # zip the results
-    sample_name = sample_dir.name
-    output_dir = sample_dir / "outputs"
+    sample_name = chunk_dir.name
+    output_dir = chunk_dir / "outputs"
     output_dir.mkdir(exist_ok=True)
     zip_file_path = output_dir / f"{sample_name}.zip"
     with zipfile.ZipFile(zip_file_path, "w") as zip_file:
         for result_file in result_files:
-            zip_entry_path = result_file.relative_to(sample_dir.parent)
+            zip_entry_path = result_file.relative_to(chunk_dir.parent)
             zip_file.write(result_file, arcname=zip_entry_path)
     return zip_file_path