From 79cd4201ab2cb2fd20b2bfc7c6cd26ab64c227bf Mon Sep 17 00:00:00 2001
From: Kirill Tsukanov <tsukanoffkirill@gmail.com>
Date: Fri, 1 Dec 2023 16:18:29 +0000
Subject: [PATCH] refactor: universal running instructions

---
 src/batch/README.md         | 20 ++++++++++++--------
 src/batch/eqtl_catalogue.sh | 19 -------------------
 src/batch/runner.sh         | 28 ++++++++++++++++++++++++++++
 3 files changed, 40 insertions(+), 27 deletions(-)
 delete mode 100755 src/batch/eqtl_catalogue.sh
 create mode 100755 src/batch/runner.sh

diff --git a/src/batch/README.md b/src/batch/README.md
index d66c9973a..df9fd4562 100644
--- a/src/batch/README.md
+++ b/src/batch/README.md
@@ -1,13 +1,17 @@
 ## How to run a batch job
+
+Set up:
 ```bash
-# Prepare and upload config.
-python3 eqtl_catalogue.py > config.json
-# Upload the script.
-gsutil cp eqtl_catalogue.py gs://genetics_etl_python_playground/batch/eqtl_catalogue/eqtl_catalogue.py
-gsutil cp eqtl_catalogue.sh gs://genetics_etl_python_playground/batch/eqtl_catalogue/eqtl_catalogue.sh
-# Submit the batch job.
+export MODULE=eqtl_catalogue
+```
+
+Run:
+```bash
+# Upload code.
+gsutil -m cp -r . gs://genetics_etl_python_playground/batch/code
+# Submit batch job.
 gcloud batch jobs submit \
-  eqtl-catalogue24 \
-  --config config.json \
+  "${MODULE}-$(date --utc +"%Y%m%d-%H%M%S")" \
+  --config <(python3 ${MODULE}.py) \
   --location europe-west1
 ```
diff --git a/src/batch/eqtl_catalogue.sh b/src/batch/eqtl_catalogue.sh
deleted file mode 100755
index d7b569285..000000000
--- a/src/batch/eqtl_catalogue.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-exec &> /mnt/share/log.$BATCH_TASK_INDEX.log
-
-echo ">>> Update APT"
-sudo apt -y update
-
-echo ">>> Install packages"
-sudo apt -y install python3 python3-pip python3-setuptools
-
-echo ">>> Update PIP and setuptools"
-sudo python3 -m pip install --upgrade pip setuptools
-echo $?
-
-echo ">>> Install packages"
-sudo python3 -m pip install pandas fsspec ftputil gcsfs pyarrow
-echo $?
-
-echo ">>> Run script"
-sudo python3 /mnt/share/eqtl_catalogue.py ${BATCH_TASK_INDEX}
-echo $?
diff --git a/src/batch/runner.sh b/src/batch/runner.sh
new file mode 100755
index 000000000..266307a8f
--- /dev/null
+++ b/src/batch/runner.sh
@@ -0,0 +1,28 @@
+export MODULE=$1
+
+# Create directory for logs, if necessary.
+mkdir -p /mnt/share/logs/$MODULE
+
+# Redirect all subsequent logs.
+# The $BATCH_TASK_INDEX variable is set by Google Batch.
+exec &> /mnt/share/logs/$MODULE/log.$BATCH_TASK_INDEX.log
+
+echo ">>> Update APT"
+sudo apt -y update
+
+echo ">>> Install packages"
+sudo apt -y install python3 python3-pip python3-setuptools
+
+echo ">>> Update PIP and setuptools"
+sudo python3 -m pip install --upgrade pip setuptools
+echo $?
+
+echo ">>> Install packages"
+sudo python3 -m pip install -r /mnt/share/code/requirements.tt
+echo $?
+
+echo ">>> Run script"
+sudo python3 /mnt/share/code/${MODULE}.py ${BATCH_TASK_INDEX}
+
+echo ">>> Completed"
+echo $?