Merge pull request #559 from moka-guys/archer_automation

Archer automation (#559)

Co-Authored-By: rebeccahaines1 <84131466+rebeccahaines1@users.noreply.github.com>
Co-Authored-By: RachelDuffin <rachel.g.duffin@gmail.com>

config/ad_config.py

@@ -75,6 +75,7 @@
     "email_user": os.path.join(DOCUMENT_ROOT, ".amazon_email_username"),
     "email_pw": os.path.join(DOCUMENT_ROOT, ".amazon_email_pw"),
     "dnanexus_authtoken": os.path.join(DOCUMENT_ROOT, ".dnanexus_auth_token"),
+    "adx_authtoken": ".archer_authentication_mokaguys.txt"
 }
 NOVASEQ_ID = "A01229"  # Novaseq sequencer ID
 RUNFOLDER_PATTERN = "^[0-9]{6}.*$"  # Runfolders start with 6 digits
@@ -90,6 +91,9 @@
 GATK_DOCKER = (
     "broadinstitute/gatk:4.1.8.1"  # TODO this image should have a hash added in future
 )
+ARCHER_DOCKER = (
+    "seglh/archer_api_upload:v1.0.0"
+)
 
 LANE_METRICS_SUFFIX = ".illumina_lane_metrics"
 DEMUX_NOT_REQUIRED_MSG = "%s run. Does not need demultiplexing locally"
@@ -383,6 +387,14 @@ class DemultiplexConfig(PanelConfig):
         f"docker run --rm --user %s:%s -v %s:/input_run {GATK_DOCKER} ./gatk CollectIlluminaLaneMetrics "
         "--RUN_DIRECTORY /input_run --OUTPUT_DIRECTORY /input_run --OUTPUT_PREFIX %s"
     )
+    ADX_CMD = (
+        f"docker run \
+            -v {RUNFOLDERS}/${{run_folder_name}}/Data/Intensities/BaseCalls:/data \
+            -v {DOCUMENT_ROOT}:/auth_file \
+            {ARCHER_DOCKER}  /data \
+            auth_file/{CREDENTIALS["adx_authtoken"]} \
+            ${{job_name}} 2 | tee -a {AD_LOGDIR}/archer_api_upload_logfiles/${{run_folder_name}}_archer_api_logfile.txt"
+    )
     DEMULTIPLEX_TEST_RUNFOLDERS = [
         "999999_NB552085_0496_DEMUXINTEG",
         "999999_M02353_0496_000000000-DEMUX",

config/log_msgs_config.py

@@ -141,8 +141,11 @@
         ),
         "writing_cmds": "Writing dx run commands to %s",
         "running_cmds": "Running dx run commands using dx run bash script",
+        "running_decision_cmds": "Running decision support commands using bash script",
         "dx_run_err": "Error when setting off dx run command. Command: %s. Stdout: %s. Stderr: %s",
+        "decision_run_err": "Error when setting off decision support command. Command: %s. Stdout: %s. Stderr: %s",
         "dx_run_success": "Dx run commands issued successfully for run %s",
+        "decision_run_success": "Decision support cmd is run successfully for run %s",    
         "uploading_rf": (
             "Uploading rest of run folder to DNAnexus using upload_runfolder, ignoring: %s. Stdout stored in logfile: %s"
         ),

setoff_workflows/setoff_workflows.py

@@ -39,6 +39,7 @@
 from config.ad_config import SWConfig
 import logging
 from upload_runfolder.upload_runfolder import UploadRunfolder
+from config.ad_config import DemultiplexConfig
 from toolbox.toolbox import (
     return_scriptlog_config,
     test_upload_software,
@@ -310,6 +311,8 @@ def __init__(self, rf_obj: RunfolderObject, loggers: dict):
         self.write_dx_run_cmds()
         self.pre_pipeline_upload()
         self.run_dx_run_commands()
+        if self.rf_samples_obj.pipeline == "archerdx":
+            self.run_decision_support_commands()
         self.pipeline_emails = PipelineEmails(
             self.rf_obj,
             self.rf_samples_obj,
@@ -776,6 +779,31 @@ def upload_rest_of_runfolder(self) -> None:
             )
             sys.exit(1)
 
+    def run_decision_support_commands(self) -> None:
+        """
+        Execute the decision_support bash script
+            :return None:
+        """
+        decision_support_run_cmd = f"bash {self.rf_obj.decision_support_upload_script}"
+
+        self.loggers["sw"].info(
+            self.loggers["sw"].log_msgs["running_decision_cmds"],
+        )
+        out, err, returncode = execute_subprocess_command(
+            decision_support_run_cmd, self.loggers["sw"], "exit_on_fail"
+        )
+        if returncode != 0:
+            self.loggers["sw"].error(
+                self.loggers["sw"].log_msgs["decision_run_err"],
+                decision_support_run_cmd,
+                out,
+                err,
+            )
+        else:
+            self.loggers["sw"].info(
+                self.loggers["sw"].log_msgs["decision_run_success"],
+                self.rf_obj.runfolder_name,
+            )    
     def run_dx_run_commands(self) -> None:
         """
         Execute the dx run bash script
@@ -883,6 +911,13 @@ def __init__(self, rf_obj: object, rf_samples: object, logger: logging.Logger):
             self.sql_queries.append(
                 sample_cmds_obj.return_oncology_query()
             )  # Get SQL queries
+        # Return the decision support command 
+        ADX_runfolder = f"run_folder_name={self.rf_obj.runfolder_name}"
+        ADX_jobID = f"job_name=$(cat {self.rf_obj.runfolderpath}/RunParameters.xml | grep -oP '(?<=ExperimentName>).*?(?=</ExperimentName)')"
+        docker = DemultiplexConfig.ADX_CMD
+        docker_api_cmd = [ADX_runfolder, ADX_jobID, docker]
+        for api_cmd in docker_api_cmd:
+            self.decision_support_upload_cmds.append(api_cmd)
 
         # Return downstream app commands
         self.workflow_cmds.extend(