polyphonia CI #192
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "polyphonia CI" | |
| on: | |
| push: | |
| pull_request: | |
| branches: | |
| - master | |
| release: | |
| types: | |
| - created | |
| schedule: | |
| - cron: '0 18 * * 1' # weekly at 18:00 on Mondays | |
| env: | |
| HOME: "${{ github.workspace }}" | |
| CACHE_DIR: "${{ github.workspace }}/misc_cache" | |
| MINICONDA_DIR: "${{ github.workspace }}/miniconda" | |
| PYTHONIOENCODING: UTF8 | |
| GITHUB_ACTIONS_COMMIT: ${{ github.sha }} | |
| GITHUB_ACTIONS_BUILD_DIR: ${{ github.workspace }} | |
| GITHUB_ACTIONS_BRANCH: ${{ github.ref }} | |
| GITHUB_ACTIONS_PULL_REQUEST: ${{ github.event.number }} | |
| GITHUB_ACTIONS_PULL_REQUEST_BRANCH: ${{ github.head_ref }} | |
| GITHUB_ACTIONS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }} | |
| GITHUB_ACTIONS_BASE_REF: ${{ github.base_ref }} | |
| DOCKER_REGISTRY: "quay.io" | |
| DOCKER_REPO_PROD: "quay.io/broadinstitute/polyphonia" | |
| DOCKER_REPO_DEV: "quay.io/broadinstitute/polyphonia" | |
| DOCKER_POLYPHONIA_PATH: "/opt/polyphonia" | |
| jobs: | |
| test_build_docker: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: checkout repository | |
| uses: actions/checkout@v2 | |
| - name: fetch tags | |
| run: git fetch --prune --unshallow --tags | |
| - name: Programmatic environment setup | |
| run: | | |
| set -e -x | |
| # $GITHUB_ENV is available for subsequent steps | |
| GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '') | |
| echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV | |
| # | |
| # Set GITHUB_ACTIONS_BRANCH | |
| # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ ) | |
| # for push builds, or builds not triggered by a pull request, this is the name of the branch. | |
| # for builds triggered by a pull request this is the name of the branch targeted by the pull request. | |
| # for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG). | |
| GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD) | |
| if [ -n "$GITHUB_ACTIONS_PULL_REQUEST_BRANCH" ]; then | |
| GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/} | |
| elif [[ "${{ github.event_name }}" == "release" ]]; then | |
| GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG | |
| fi | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV | |
| echo "GITHUB_ACTIONS_SHORT_SHA=$(echo $GITHUB_SHA | head -c8)" >> $GITHUB_ENV | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v1 | |
| - name: Cache Docker layers | |
| uses: actions/cache@v2 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-buildx-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx- | |
| - name: Build and export | |
| id: docker_build | |
| uses: docker/build-push-action@v2 | |
| with: | |
| context: . | |
| push: false | |
| tags: "polyphonia:${{ env.GITHUB_ACTIONS_BRANCH }}-${{ env.GITHUB_ACTIONS_SHORT_SHA }}" | |
| cache-from: type=local,src=/tmp/.buildx-cache | |
| cache-to: type=local,dest=/tmp/.buildx-cache-new | |
| - name: Image digest | |
| run: echo ${{ steps.docker_build.outputs.digest }} | |
| # Temp fix | |
| # https://github.com/docker/build-push-action/issues/252 | |
| # https://github.com/moby/buildkit/issues/1896 | |
| - name: Store build in cache (overwrite) | |
| run: | | |
| rm -rf /tmp/.buildx-cache | |
| mv /tmp/.buildx-cache-new /tmp/.buildx-cache | |
| test_execution: | |
| runs-on: ubuntu-20.04 | |
| needs: test_build_docker | |
| steps: | |
| - name: checkout repository | |
| uses: actions/checkout@v2 | |
| - name: fetch tags | |
| run: git fetch --prune --unshallow --tags | |
| - name: Programmatic environment setup | |
| run: | | |
| set -e -x | |
| # $GITHUB_ENV is available for subsequent steps | |
| GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '') | |
| echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV | |
| # | |
| # Set GITHUB_ACTIONS_BRANCH | |
| # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ ) | |
| # for push builds, or builds not triggered by a pull request, this is the name of the branch. | |
| # for builds triggered by a pull request this is the name of the branch targeted by the pull request. | |
| # for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG). | |
| GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD) | |
| if [ -n "$GITHUB_ACTIONS_PULL_REQUEST_BRANCH" ]; then | |
| GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/} | |
| elif [[ "${{ github.event_name }}" == "release" ]]; then | |
| GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG | |
| fi | |
| echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV | |
| echo "GITHUB_ACTIONS_SHORT_SHA=$(echo $GITHUB_SHA | head -c8)" >> $GITHUB_ENV | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v1 | |
| - name: Cache Docker layers | |
| uses: actions/cache@v2 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-buildx-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx- | |
| # re-build image (from cache; this seems faster than exporting the image and passing as an Actions artifact) | |
| - name: Build and export | |
| id: docker_build | |
| uses: docker/build-push-action@v2 | |
| with: | |
| context: . | |
| push: false | |
| load: true | |
| tags: "polyphonia:${{ env.GITHUB_ACTIONS_BRANCH }}-${{ env.GITHUB_ACTIONS_SHORT_SHA }}" | |
| cache-from: type=local,src=/tmp/.buildx-cache | |
| - name: Image digest | |
| run: echo ${{ steps.docker_build.outputs.digest }} | |
| - name: Create local dir for test output | |
| run: | | |
| mkdir -p output | |
| - name: Run polyphonia (basic execution test) | |
| run: | | |
| docker run -v $(pwd)/bin:${{ env.DOCKER_POLYPHONIA_PATH }} -v $(pwd)/test:${{ env.DOCKER_POLYPHONIA_PATH }}/test -v $(pwd)/output:/tmp/output --rm polyphonia:${{ env.GITHUB_ACTIONS_BRANCH }}-${{ env.GITHUB_ACTIONS_SHORT_SHA }} \ | |
| polyphonia cross_contamination | |
| - name: Run polyphonia (cross_contamination test) | |
| run: | | |
| set -e -x | |
| docker run -v $(pwd)/bin:${{ env.DOCKER_POLYPHONIA_PATH }} -v $(pwd)/test:${{ env.DOCKER_POLYPHONIA_PATH }}/test -v $(pwd)/output:/tmp/output --rm polyphonia:${{ env.GITHUB_ACTIONS_BRANCH }}-${{ env.GITHUB_ACTIONS_SHORT_SHA }} \ | |
| polyphonia cross_contamination \ | |
| --ref ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/NC_045512.2.fasta \ | |
| --vcf ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01315-2021.bam_LoFreq.vcf \ | |
| --vcf ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01323-2021.bam_LoFreq.vcf \ | |
| --consensus ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01315-2021.fasta \ | |
| --consensus ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01323-2021.fasta \ | |
| --plate-map ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01315_23-2021_plate_map.txt \ | |
| --min-depth 0 \ | |
| --output /tmp/output/potential_cross-contamination.txt \ | |
| --overwrite TRUE | |
| DIFF=$(diff $(pwd)/output/potential_cross-contamination.txt $(pwd)/test/output/potential_cross-contamination.txt) | |
| if [ "$DIFF" != "" ]; then | |
| echo "Failure: cross_contamination test output does not match expected output" | |
| echo " expected output:" | |
| cat $(pwd)/output/potential_cross-contamination.txt | |
| echo " test output:" | |
| cat ${{ env.DOCKER_POLYPHONIA_PATH }}/test/output/potential_cross-contamination.txt | |
| exit 1 | |
| else | |
| echo "Success: cross_contamination test" | |
| fi | |
| - name: Run polyphonia (cross_contamination test with aligned consensus sequences) | |
| run: | | |
| set -e -x | |
| docker run -v $(pwd)/bin:${{ env.DOCKER_POLYPHONIA_PATH }} -v $(pwd)/test:${{ env.DOCKER_POLYPHONIA_PATH }}/test -v $(pwd)/output:/tmp/output --rm polyphonia:${{ env.GITHUB_ACTIONS_BRANCH }}-${{ env.GITHUB_ACTIONS_SHORT_SHA }} \ | |
| polyphonia cross_contamination \ | |
| --ref ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/NC_045512.2.fasta \ | |
| --vcf ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01315-2021.bam_LoFreq.vcf \ | |
| --vcf ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01323-2021.bam_LoFreq.vcf \ | |
| --consensus-aligned ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01315_23-2021_MAFFT_aligned.fasta \ | |
| --plate-map ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01315_23-2021_plate_map.txt \ | |
| --min-depth 0 \ | |
| --output /tmp/output/potential_cross-contamination.txt \ | |
| --overwrite TRUE | |
| DIFF=$(diff $(pwd)/output/potential_cross-contamination.txt $(pwd)/test/output/potential_cross-contamination.txt) | |
| if [ "$DIFF" != "" ]; then | |
| echo "Failure: cross_contamination test output does not match expected output" | |
| echo " expected output:" | |
| cat $(pwd)/output/potential_cross-contamination.txt | |
| echo " test output:" | |
| cat ${{ env.DOCKER_POLYPHONIA_PATH }}/test/output/potential_cross-contamination.txt | |
| exit 1 | |
| else | |
| echo "Success: cross_contamination test" | |
| fi | |
| - name: Run polyphonia (cross_contamination test with aligned consensus sequences with gap in reference) | |
| run: | | |
| set -e -x | |
| docker run -v $(pwd)/bin:${{ env.DOCKER_POLYPHONIA_PATH }} -v $(pwd)/test:${{ env.DOCKER_POLYPHONIA_PATH }}/test -v $(pwd)/output:/tmp/output --rm polyphonia:${{ env.GITHUB_ACTIONS_BRANCH }}-${{ env.GITHUB_ACTIONS_SHORT_SHA }} \ | |
| polyphonia cross_contamination \ | |
| --ref ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/NC_045512.2.fasta \ | |
| --vcf ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01315-2021.bam_LoFreq.vcf \ | |
| --vcf ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01323-2021.bam_LoFreq.vcf \ | |
| --consensus-aligned ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01315_23-2021_MAFFT_aligned_gap_added.fasta \ | |
| --plate-map ${{ env.DOCKER_POLYPHONIA_PATH }}/test/input/USA-MA-Broad_CRSP-01315_23-2021_plate_map.txt \ | |
| --min-depth 0 \ | |
| --output /tmp/output/potential_cross-contamination.txt \ | |
| --overwrite TRUE | |
| DIFF=$(diff $(pwd)/output/potential_cross-contamination.txt $(pwd)/test/output/potential_cross-contamination.txt) | |
| if [ "$DIFF" != "" ]; then | |
| echo "Failure: cross_contamination test output does not match expected output" | |
| echo " expected output:" | |
| cat $(pwd)/output/potential_cross-contamination.txt | |
| echo " test output:" | |
| cat ${{ env.DOCKER_POLYPHONIA_PATH }}/test/output/potential_cross-contamination.txt | |
| exit 1 | |
| else | |
| echo "Success: cross_contamination test" | |
| fi | |