feat: install package from git

Szymon Szyszkowski · Szymon Szyszkowski · commit 14864588fac8 · 2025-01-23T14:53:59.000Z
diff --git a/src/ot_orchestration/dags/config/credible_set_qc.yaml b/src/ot_orchestration/dags/config/credible_set_qc.yaml
@@ -1,9 +1,9 @@
 ---
 dataproc:
-  python_main_module: gs://genetics_etl_python_playground/initialisation/gentropy/dev/cli.py
+  python_main_module: gs://genetics_etl_python_playground/initialisation/cli.py
   cluster_metadata:
-    PACKAGE: gs://genetics_etl_python_playground/initialisation/gentropy/dev/gentropy-0.0.0-py3-none-any.whl
-  cluster_init_script: gs://genetics_etl_python_playground/initialisation/gentropy/dev/install_dependencies_on_cluster.sh
+    GENTROPY_REF: v2.0.1
+  cluster_init_script: gs://genetics_etl_python_playground/initialisation/install_dependencies_on_cluster.sh
   cluster_name: otg-credible-set-qc
   autoscaling_policy: otg-etl
 
diff --git a/src/ot_orchestration/dags/config/eqtl_catalogue_ingestion.yaml b/src/ot_orchestration/dags/config/eqtl_catalogue_ingestion.yaml
@@ -4,10 +4,10 @@ eqtl_catalogue_decompressed_susie_path: gs://eqtl_catalogue_data/ebi_ftp/susie_d
 decompression_logs: gs://eqtl_catalogue_data/ebi_ftp/susie_decompressed_tmp/logs.log
 
 dataproc:
-  python_main_module: gs://genetics_etl_python_playground/initialisation/gentropy/dev/cli.py
+  python_main_module: gs://genetics_etl_python_playground/initialisation/cli.py
   cluster_metadata:
-    PACKAGE: gs://genetics_etl_python_playground/initialisation/gentropy/dev/gentropy-0.0.0-py3-none-any.whl
-  cluster_init_script: gs://genetics_etl_python_playground/initialisation/gentropy/dev/install_dependencies_on_cluster.sh
+    GENTROPY_REF: v2.0.1
+  cluster_init_script: gs://genetics_etl_python_playground/initialisation/install_dependencies_on_cluster.sh
   cluster_name: otg-preprocess-eqtl
   autoscaling_policy: eqtl-preprocess
   num_workers: 4
diff --git a/src/ot_orchestration/dags/config/finngen_ingestion.yaml b/src/ot_orchestration/dags/config/finngen_ingestion.yaml
@@ -1,9 +1,9 @@
 ---
 dataproc:
-  python_main_module: gs://genetics_etl_python_playground/initialisation/gentropy/dev/cli.py
+  python_main_module: gs://genetics_etl_python_playground/initialisation/cli.py
   cluster_metadata:
-    PACKAGE: gs://genetics_etl_python_playground/initialisation/gentropy/dev/gentropy-0.0.0-py3-none-any.whl
-  cluster_init_script: gs://genetics_etl_python_playground/initialisation/gentropy/dev/install_dependencies_on_cluster.sh
+    GENTROPY_REF: v2.0.1
+  cluster_init_script: gs://genetics_etl_python_playground/initialisation/install_dependencies_on_cluster.sh
   cluster_name: otg-ingestion-finngen
   autoscaling_policy: finngen-preprocess
   master_disk_size: 2000
diff --git a/src/ot_orchestration/dags/config/finngen_ukb_meta_harmonisation.yaml b/src/ot_orchestration/dags/config/finngen_ukb_meta_harmonisation.yaml
@@ -1,9 +1,9 @@
 ---
 dataproc:
-  python_main_module: gs://genetics_etl_python_playground/initialisation/gentropy/szsz-update-package-for-dataproc-run/cli.py
+  python_main_module: gs://genetics_etl_python_playground/initialisation//cli.py
   cluster_metadata:
-    PACKAGE: gs://genetics_etl_python_playground/initialisation/gentropy/szsz-update-package-for-dataproc-run/gentropy-0.0.0-py3-none-any.whl
-  cluster_init_script: gs://genetics_etl_python_playground/initialisation/0.0.0/install_dependencies_on_cluster.sh
+    GENTROPY_REF: v2.0.1
+  cluster_init_script: gs://genetics_etl_python_playground/install_dependencies_on_cluster.sh
   cluster_name: otg-finngen-ukb-meta
 
 nodes:
diff --git a/src/ot_orchestration/dags/config/foldx_ingestion.yaml b/src/ot_orchestration/dags/config/foldx_ingestion.yaml
@@ -1,9 +1,10 @@
 ---
 dataproc:
-  python_main_module: gs://genetics_etl_python_playground/initialisation/gentropy/dev/cli.py
+  python_main_module: gs://genetics_etl_python_playground/initialisation/cli.py
   cluster_metadata:
-    PACKAGE: gs://genetics_etl_python_playground/initialisation/gentropy/dev/gentropy-0.0.0-py3-none-any.whl
-  cluster_init_script: gs://genetics_etl_python_playground/initialisation/gentropy/dev/install_dependencies_on_cluster.sh
+    GENTROPY_REF: v2.0.1
+  cluster_init_script: gs://genetics_etl_python_playground/initialisation/install_dependencies_on_cluster.sh
+
   cluster_name: foldx-ingestion
   autoscaling_policy: otg-etl
   allow_efm: false
diff --git a/src/ot_orchestration/dags/config/genetics_etl.yaml b/src/ot_orchestration/dags/config/genetics_etl.yaml
@@ -2,11 +2,11 @@
 l2g_gold_standard_path: gs://genetics_etl_python_playground/input/l2g/gold_standard/curation.json
 release_dir: gs://ot_orchestration/releases/24.11_freeze10
 dataproc:
-  python_main_module: gs://genetics_etl_python_playground/initialisation/gentropy/dev/cli.py
+  python_main_module: gs://genetics_etl_python_playground/initialisation/cli.py
   cluster_metadata:
-    PACKAGE: gs://genetics_etl_python_playground/initialisation/gentropy/dev/gentropy-0.0.0-py3-none-any.whl
-  cluster_init_script: gs://genetics_etl_python_playground/initialisation/gentropy/dev/install_dependencies_on_cluster.sh
-  cluster_name: otg-etl
+    GENTROPY_REF: v2.0.1
+  cluster_init_script: gs://genetics_etl_python_playground/initialisation/install_dependencies_on_cluster.sh
+  cluster_name: test-dataproc
   autoscaling_policy: otg-etl
   allow_efm: false
   num_workers: 2
diff --git a/src/ot_orchestration/dags/config/gentropy.yaml b/src/ot_orchestration/dags/config/gentropy.yaml
@@ -1,10 +1,10 @@
 ---
-python_main_module: gs://genetics_etl_python_playground/initialisation/gentropy/{gentropy_version}/cli.py
+python_main_module: gs://genetics_etl_python_playground/initialisation/cli.py
 
 dataproc_cluster_settings:
   cluster_metadata:
-    PACKAGE: gs://genetics_etl_python_playground/initialisation/gentropy/{gentropy_version}/gentropy-0.0.0-py3-none-any.whl
-  cluster_init_script: gs://genetics_etl_python_playground/initialisation/gentropy/{gentropy_version}/install_dependencies_on_cluster.sh
+    GENTROPY_REF: {gentropy_version: ''}
+  cluster_init_script: gs://genetics_etl_python_playground/initialisation/install_dependencies_on_cluster.sh
   autoscaling_policy: otg-etl
   allow_efm: false
   num_workers: 2
diff --git a/src/ot_orchestration/dags/config/gnomad_ingestion.yaml b/src/ot_orchestration/dags/config/gnomad_ingestion.yaml
@@ -1,9 +1,9 @@
 ---
 dataproc:
-  python_main_module: gs://genetics_etl_python_playground/initialisation/gentropy/dev/cli.py
+  python_main_module: gs://genetics_etl_python_playground/initialisation/cli.py
   cluster_metadata:
-    PACKAGE: gs://genetics_etl_python_playground/initialisation/gentropy/dev/gentropy-0.0.0-py3-none-any.whl
-  cluster_init_script: gs://genetics_etl_python_playground/initialisation/gentropy/dev/install_dependencies_on_cluster.sh
+    GENTROPY_REF: v2.0.1
+  cluster_init_script: gs://genetics_etl_python_playground/initialisation/install_dependencies_on_cluster.sh
   cluster_name: gnomad-preprocess
   autoscaling_policy: otg-etl
   allow_efm: false
diff --git a/src/ot_orchestration/dags/config/gwas_catalog_sumstats_pics.yaml b/src/ot_orchestration/dags/config/gwas_catalog_sumstats_pics.yaml
@@ -2,10 +2,10 @@
 dataproc:
   cluster_name: otg-gwascatalog-sumstats-pics
   autoscaling_policy: otg-preprocess-gwascatalog
-  python_main_module: gs://genetics_etl_python_playground/initialisation/gentropy/dev/cli.py
+  python_main_module: gs://genetics_etl_python_playground/initialisation/cli.py
   cluster_metadata:
-    PACKAGE: gs://genetics_etl_python_playground/initialisation/gentropy/dev/gentropy-0.0.0-py3-none-any.whl
-  cluster_init_script: gs://genetics_etl_python_playground/initialisation/gentropy/dev/install_dependencies_on_cluster.sh
+    GENTROPY_REF: v2.0.1
+  cluster_init_script: gs://genetics_etl_python_playground/initialisation/install_dependencies_on_cluster.sh
   num_workers: 5
 
 nodes:
diff --git a/src/ot_orchestration/dags/config/gwas_catalog_sumstats_susie_clumping.yaml b/src/ot_orchestration/dags/config/gwas_catalog_sumstats_susie_clumping.yaml
@@ -1,9 +1,9 @@
 ---
 dataproc:
-  python_main_module: gs://genetics_etl_python_playground/initialisation/gentropy/dev/cli.py
+  python_main_module: gs://genetics_etl_python_playground/initialisation/cli.py
   cluster_metadata:
-    PACKAGE: gs://genetics_etl_python_playground/initialisation/gentropy/dev/gentropy-0.0.0-py3-none-any.whl
-  cluster_init_script: gs://genetics_etl_python_playground/initialisation/gentropy/dev/install_dependencies_on_cluster.sh
+    GENTROPY_REF: v2.0.1
+  cluster_init_script: gs://genetics_etl_python_playground/initialisation/install_dependencies_on_cluster.sh
   cluster_name: otg-gwas-catalog-sumstat-susie-clumping
   autoscaling_policy: otg-etl
 
diff --git a/src/ot_orchestration/dags/config/gwas_catalog_top_hits.yaml b/src/ot_orchestration/dags/config/gwas_catalog_top_hits.yaml
@@ -2,10 +2,10 @@
 dataproc:
   cluster_name: otg-tophit-gwascatalog
   autoscaling_policy: otg-gwascatalog-tophit
-  python_main_module: gs://genetics_etl_python_playground/initialisation/gentropy/dev/cli.py
+  python_main_module: gs://genetics_etl_python_playground/initialisation/cli.py
   cluster_metadata:
-    PACKAGE: gs://genetics_etl_python_playground/initialisation/gentropy/dev/gentropy-0.0.0-py3-none-any.whl
-  cluster_init_script: gs://genetics_etl_python_playground/initialisation/gentropy/dev/install_dependencies_on_cluster.sh
+    GENTROPY_REF: v2.0.1
+  cluster_init_script: gs://genetics_etl_python_playground/initialisation/install_dependencies_on_cluster.sh
   num_workers: 2
 
 nodes:
diff --git a/src/ot_orchestration/dags/config/ukb_ppp_eur_harmonisation.yaml b/src/ot_orchestration/dags/config/ukb_ppp_eur_harmonisation.yaml
@@ -1,9 +1,9 @@
 ---
 dataproc:
-  python_main_module: gs://genetics_etl_python_playground/initialisation/gentropy/dev/cli.py
+  python_main_module: gs://genetics_etl_python_playground/initialisation/cli.py
   cluster_metadata:
-    PACKAGE: gs://genetics_etl_python_playground/initialisation/gentropy/dev/gentropy-0.0.0-py3-none-any.whl
-  cluster_init_script: gs://genetics_etl_python_playground/initialisation/gentropy/dev/install_dependencies_on_cluster.sh
+    GENTROPY_REF: v2.0.1
+  cluster_init_script: gs://genetics_etl_python_playground/initialisation/install_dependencies_on_cluster.sh
   cluster_name: otg-ukb-ppp-eur
   autoscaling_policy: otg-etl
 
diff --git a/src/ot_orchestration/utils/dataproc.py b/src/ot_orchestration/utils/dataproc.py
@@ -105,6 +105,7 @@ def create_cluster(
         num_preemptible_workers=num_preemptible_workers,
         num_workers=num_workers,
         image_version=GCP_DATAPROC_IMAGE,
+        internal_ip_only=False,
         enable_component_gateway=True,
         metadata=cluster_metadata,
         idle_delete_ttl=idle_delete_ttl,
@@ -229,8 +230,8 @@ def submit_pyspark_job(
             "main_python_file_uri": python_main_module,
             "args": args,
             "properties": {
-                "spark.jars": "/opt/conda/miniconda3/lib/python3.10/site-packages/hail/backend/hail-all-spark.jar",
-                "spark.driver.extraClassPath": "/opt/conda/miniconda3/lib/python3.10/site-packages/hail/backend/hail-all-spark.jar",
+                "spark.jars": "/opt/conda/miniconda3/lib/python3.11/site-packages/hail/backend/hail-all-spark.jar",
+                "spark.driver.extraClassPath": "/opt/conda/miniconda3/lib/python3.11/site-packages/hail/backend/hail-all-spark.jar",
                 "spark.executor.extraClassPath": "./hail-all-spark.jar",
                 "spark.serializer": "org.apache.spark.serializer.KryoSerializer",
                 "spark.kryo.registrator": "is.hail.kryo.HailKryoRegistrator",
