From 548403218a60092363a853b5f1350938ce56aa44 Mon Sep 17 00:00:00 2001 From: aawdeh Date: Thu, 1 Feb 2024 17:42:34 -0500 Subject: [PATCH 01/16] Build docker distributed STAR + samtools --- 3rd-party-tools/samtools-dist-star/Dockerfile | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 3rd-party-tools/samtools-dist-star/Dockerfile diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile new file mode 100644 index 00000000..6569bb16 --- /dev/null +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -0,0 +1,41 @@ +# Adding a platform tag to ensure that images built on ARM-based machines doesn't break pipelines +FROM --platform="linux/amd64" ubuntu:22.04 + +ENV TERM=xterm-256color + +LABEL MAINTAINER="Broad Institute DSDE " + +WORKDIR /usr + +# Install dependencies +RUN set -eux; \ + apt update; \ + apt-get install -y autoconf automake bash bzip2 gcc wget make libncurses5-dev zlib1g-dev libbz2-dev liblzma-dev numactl; \ + apt-get install -y build-essential git; \ + + # Install open omics acceleration framework + mkdir temp; \ + ls; \ + pwd; \ + cd temp; \ + # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output + git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ + + # install packages + cd Open-Omics-Acceleration-Framework; \ + git checkout aa-optimus; \ + + + cd pipelines/optimus_starsolo; \ + ./install.sh; \ + + # Tini + cd /usr/temp; \ + wget https://github.com/krallin/tini/releases/download/v0.19.0/tini -O /usr/bin/tini; \ + chmod +x /usr/bin/tini; \ + + # Clean up cached files + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Set tini as default entrypoint +ENTRYPOINT [ "/usr/bin/tini", "--" ] From c885d7912d446986fdc4eb34901b08115d96619a Mon Sep 17 00:00:00 2001 From: aawdeh Date: Thu, 1 Feb 2024 17:48:52 -0500 Subject: [PATCH 02/16] Added dist_star.yml --- .../workflows/build-samtools-dist-star.yml | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/build-samtools-dist-star.yml diff --git a/.github/workflows/build-samtools-dist-star.yml b/.github/workflows/build-samtools-dist-star.yml new file mode 100644 index 00000000..4f5c537f --- /dev/null +++ b/.github/workflows/build-samtools-dist-star.yml @@ -0,0 +1,59 @@ +name: Samtools Dist STAR Mem 2 Ubuntu CI + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the "develop" and "master" branch + pull_request: + branches: [ "develop", "master" ] + paths: + - '3rd-party-tools/samtools-dist-star/**' + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + inputs: + image_tag: + description: 'Docker Image Tag (default: branch_name)' + +env: + PROJECT_NAME: WARP 3rd Party Tools + # Github repo name + REPOSITORY_NAME: ${{ github.event.repository.name }} + # Region-specific Google Docker repository where GOOGLE_PROJECT/REPOSITORY_NAME can be found + DOCKER_REGISTRY: us.gcr.io + GCR_PATH: broad-gotc-prod/samtools-dist-star + TAG: ${{ github.event.inputs.image_tag || github.head_ref || github.ref_name }} + + # A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # The job that builds our container + build: + runs-on: ubuntu-latest + defaults: + run: + working-directory: 3rd-party-tools/samtools-dist-star + # Map a step output to a job output + outputs: + imagePath: ${{ steps.saveImagePath.outputs.url }} + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + - name: Build the Docker image + run: docker build . --file Dockerfile --tag ${DOCKER_REGISTRY}/${GCR_PATH}:${TAG} + - name: Check working directory' + run: | + echo "Current directory: " + pwd + ls -lht + # Save the image path to an output + - id: 'saveImagePath' + run: echo "url=${DOCKER_REGISTRY}/${GCR_PATH}:${TAG}" >> $GITHUB_OUTPUT + # Log into the Google Docker registry + - id: 'Auth' + name: Login to GCR + uses: docker/login-action@v2 + with: + registry: ${{ env.DOCKER_REGISTRY }} + username: _json_key + password: ${{ secrets.GCR_CI_KEY }} + # Push the image to the Google Docker registry + - name: Push image + run: "docker push ${DOCKER_REGISTRY}/${GCR_PATH}:${TAG}" \ No newline at end of file From 4c1c1cba8734407257065e4d37a218398df65818 Mon Sep 17 00:00:00 2001 From: aawdeh Date: Fri, 2 Feb 2024 12:11:04 -0500 Subject: [PATCH 03/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index 6569bb16..9806a60a 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output + # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From 4ba20abc7acd7f6815320ae7b244b51043903dd7 Mon Sep 17 00:00:00 2001 From: aawdeh Date: Wed, 7 Feb 2024 14:43:37 -0500 Subject: [PATCH 04/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index 9806a60a..84509bba 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output for star + # clone fork Open-Omics-Acceleration-Framework with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From 769e613ba999ae0a3da829d8063f4a3baf2635eb Mon Sep 17 00:00:00 2001 From: aawdeh Date: Wed, 7 Feb 2024 17:14:09 -0500 Subject: [PATCH 05/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index 84509bba..e43d3dda 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork Open-Omics-Acceleration-Framework with changes added for input/output for star + # clone fork from Open-Omics-Acceleration-Framework with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From 6ce1faf874badfe035c451684e2c9c2ced557f68 Mon Sep 17 00:00:00 2001 From: aawdeh Date: Wed, 7 Feb 2024 17:23:59 -0500 Subject: [PATCH 06/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index e43d3dda..84509bba 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork from Open-Omics-Acceleration-Framework with changes added for input/output for star + # clone fork Open-Omics-Acceleration-Framework with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From 4a16bdbc1d72840995666ccf9a72ea1d44f803db Mon Sep 17 00:00:00 2001 From: aawdeh Date: Thu, 8 Feb 2024 12:55:49 -0500 Subject: [PATCH 07/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index 84509bba..9806a60a 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork Open-Omics-Acceleration-Framework with changes added for input/output for star + # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From 6ae52caf9a51c5ee08449f978b4ffb93608bfcde Mon Sep 17 00:00:00 2001 From: aawdeh Date: Thu, 8 Feb 2024 13:09:37 -0500 Subject: [PATCH 08/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index 9806a60a..d68e2ba0 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output for star + # clone fork of Open-Omics-Acceleration-Framework repo with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From a314a0e9f2bdf3ae82f58e4ecab019d22084b84f Mon Sep 17 00:00:00 2001 From: aawdeh Date: Thu, 8 Feb 2024 13:51:23 -0500 Subject: [PATCH 09/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index d68e2ba0..9806a60a 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork of Open-Omics-Acceleration-Framework repo with changes added for input/output for star + # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From 66875faf92c5a3487808cdd2ec4775fbe92ac8f9 Mon Sep 17 00:00:00 2001 From: aawdeh Date: Mon, 12 Feb 2024 11:16:47 -0500 Subject: [PATCH 10/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index 9806a60a..d68e2ba0 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output for star + # clone fork of Open-Omics-Acceleration-Framework repo with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From 493023f6bfefc3fe1a4fee5dd213b3109876c9c5 Mon Sep 17 00:00:00 2001 From: aawdeh Date: Mon, 12 Feb 2024 17:00:48 -0500 Subject: [PATCH 11/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index d68e2ba0..9806a60a 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork of Open-Omics-Acceleration-Framework repo with changes added for input/output for star + # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From be856208ded2a786a17495f1ed911a9861c4832b Mon Sep 17 00:00:00 2001 From: aawdeh Date: Tue, 13 Feb 2024 20:42:04 -0500 Subject: [PATCH 12/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index 9806a60a..d68e2ba0 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output for star + # clone fork of Open-Omics-Acceleration-Framework repo with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From c47efe0c06ff604e0d21d043fc33ac88daf370e0 Mon Sep 17 00:00:00 2001 From: aawdeh Date: Tue, 13 Feb 2024 21:29:03 -0500 Subject: [PATCH 13/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index d68e2ba0..9806a60a 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork of Open-Omics-Acceleration-Framework repo with changes added for input/output for star + # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From 9be7eae10bd549f3bf20e4fa5e7c86df988ed906 Mon Sep 17 00:00:00 2001 From: aawdeh Date: Tue, 13 Feb 2024 23:12:07 -0500 Subject: [PATCH 14/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index 9806a60a..d68e2ba0 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output for star + # clone fork of Open-Omics-Acceleration-Framework repo with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From 8c8a9b87b02e452c5597b0d1dd61df3c22c51b59 Mon Sep 17 00:00:00 2001 From: aawdeh Date: Wed, 14 Feb 2024 00:05:23 -0500 Subject: [PATCH 15/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index d68e2ba0..9806a60a 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork of Open-Omics-Acceleration-Framework repo with changes added for input/output for star + # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages From 101988c4f1168f1ecf603c99e287432326003265 Mon Sep 17 00:00:00 2001 From: aawdeh Date: Wed, 14 Feb 2024 00:08:27 -0500 Subject: [PATCH 16/16] Update Dockerfile --- 3rd-party-tools/samtools-dist-star/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party-tools/samtools-dist-star/Dockerfile b/3rd-party-tools/samtools-dist-star/Dockerfile index 9806a60a..d68e2ba0 100644 --- a/3rd-party-tools/samtools-dist-star/Dockerfile +++ b/3rd-party-tools/samtools-dist-star/Dockerfile @@ -18,7 +18,7 @@ RUN set -eux; \ ls; \ pwd; \ cd temp; \ - # clone fork of Open-Omics-Acceleration-Framework with changes added for input/output for star + # clone fork of Open-Omics-Acceleration-Framework repo with changes added for input/output for star git clone --recursive https://github.com/aawdeh/Open-Omics-Acceleration-Framework.git; \ # install packages