improve docker layer caching by creating partial yamls

openproblems-bio · Sep 20, 2024 · 9974e3f · 9974e3f
1 parent 5a0be2c
commit 9974e3f
Show file tree

Hide file tree

Showing 11 changed files with 28 additions and 31 deletions.
diff --git a/src/base/setup_spatialdata_partial.yaml b/src/base/setup_spatialdata_partial.yaml
@@ -0,0 +1,3 @@
+setup:
+  - type: python
+    pypi: [spatialdata]
diff --git a/src/base/setup_txsim_partial.yaml b/src/base/setup_txsim_partial.yaml
@@ -0,0 +1,4 @@
+setup:
+  - type: python
+    pypi: [squidpy, rasterio]
+    github: [theislab/txsim@dev]
diff --git a/src/data_processors/process_dataset/config.vsh.yaml b/src/data_processors/process_dataset/config.vsh.yaml
@@ -10,9 +10,8 @@ resources:
 engines:
   - type: docker
     image: openproblems/base_python:1.0.0
-    setup:
-      - type: python
-        pypi: [spatialdata]
+    __merge__: 
+      - /src/base/setup_spatialdata_partial.yaml
   - type: native
 
 runners:

diff --git a/src/datasets/processors/crop_region/config.vsh.yaml b/src/datasets/processors/crop_region/config.vsh.yaml
@@ -35,10 +35,8 @@ resources:
 engines:
   - type: docker
     image: openproblems/base_python:1.0.0
-    setup:
-      - type: python
-        pypi:
-          - spatialdata
+    __merge__: 
+      - /src/base/setup_spatialdata_partial.yaml
 
 runners:
   - type: executable

diff --git a/src/methods_calculate_cell_volume/alpha_shapes/config.vsh.yaml b/src/methods_calculate_cell_volume/alpha_shapes/config.vsh.yaml
@@ -16,10 +16,12 @@ resources:
 engines:
   - type: docker
     image: openproblems/base_python:1.0.0
+    __merge__: 
+      - /src/base/setup_txsim_partial.yaml
+      - /src/base/setup_spatialdata_partial.yaml
     setup:
       - type: python
-        pypi: [spatialdata, alphashape, descartes, squidpy, rasterio]
-        github: [theislab/txsim@dev]
+        pypi: [alphashape, descartes]
   - type: native
 
 runners:

diff --git a/src/methods_cell_type_annotation/ssam/config.vsh.yaml b/src/methods_cell_type_annotation/ssam/config.vsh.yaml
@@ -16,11 +16,9 @@ resources:
 engines:
   - type: docker
     image: openproblems/base_python:1.0.0
+    __merge__: 
+      - /src/base/setup_txsim_partial.yaml
     setup:
-      - type: python
-        github: [theislab/txsim@dev]
-      - type: python
-        pypi: [squidpy, rasterio]
       - type: python
         pypi: [planktonspace]
   - type: native

diff --git a/src/methods_count_aggregation/basic/config.vsh.yaml b/src/methods_count_aggregation/basic/config.vsh.yaml
@@ -25,10 +25,9 @@ resources:
 engines:
   - type: docker
     image: openproblems/base_python:1.0.0
-    setup:
-      - type: python
-        pypi: [spatialdata, squidpy, rasterio]
-        github: [theislab/txsim@dev]
+    __merge__: 
+      - /src/base/setup_txsim_partial.yaml
+      - /src/base/setup_spatialdata_partial.yaml
   - type: native
 
 runners:

diff --git a/src/methods_expression_correction/gene_efficiency_correction/config.vsh.yaml b/src/methods_expression_correction/gene_efficiency_correction/config.vsh.yaml
@@ -16,9 +16,8 @@ resources:
 engines:
   - type: docker
     image: openproblems/base_python:1.0.0
-    setup:
-      - type: python
-        github: [theislab/txsim@dev]
+    __merge__: 
+      - /src/base/setup_txsim_partial.yaml
   - type: native
 
 runners:

diff --git a/src/methods_normalization/normalize_by_volume/config.vsh.yaml b/src/methods_normalization/normalize_by_volume/config.vsh.yaml
@@ -9,11 +9,8 @@ resources:
 engines:
   - type: docker
     image: openproblems/base_python:1.0.0
-    setup:
-      - type: python
-        github: [theislab/txsim@dev]
-      - type: python
-        pypi: [squidpy, rasterio]
+    __merge__: 
+      - /src/base/setup_txsim_partial.yaml
   - type: native
 
 runners:

diff --git a/src/methods_segmentation/custom/config.vsh.yaml b/src/methods_segmentation/custom/config.vsh.yaml
@@ -15,9 +15,8 @@ resources:
 engines:
   - type: docker
     image: openproblems/base_python:1.0.0
-    setup:
-      - type: python
-        pypi: spatialdata
+    __merge__: 
+      - /src/base/setup_spatialdata_partial.yaml
   - type: native
 
 runners:

diff --git a/src/methods_transcript_assignment/basic/config.vsh.yaml b/src/methods_transcript_assignment/basic/config.vsh.yaml
@@ -19,9 +19,8 @@ resources:
 engines:
   - type: docker
     image: openproblems/base_python:1.0.0
-    setup:
-      - type: python
-        pypi: spatialdata
+    __merge__: 
+      - /src/base/setup_spatialdata_partial.yaml
   - type: native
 
 runners: