Merge pull request #28 from alliance-genome/KANBAN-508_API

API setup and implementation - KANBAN-508

.github/workflows/PR-validation.yml

@@ -8,8 +8,23 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 jobs:
+  container-build-context:
+    name: define container image build context variables
+    runs-on: ubuntu-22.04
+    outputs:
+      tagname: ${{ steps.store_tagname.outputs.tagname }}
+    steps:
+    - name: Store image tag in env
+      id: store_tagname
+      shell: bash
+      run: |
+        echo "tagname=PR-${{ github.event.pull_request.number }}-validation" >> $GITHUB_OUTPUT
   pipeline-seq-retrieval-container-image-build:
     name: pipeline/seq_retrieval container-image build
+    needs:
+      - container-build-context
+    permissions:
+      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
     runs-on: ubuntu-22.04
     steps:
       - name: Check out repository code
@@ -20,6 +35,17 @@ jobs:
             pipeline/seq_retrieval/
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+      # This step will configure environment variables to be used by all steps
+      # involving AWS interaction further down
+      - name: AWS credentials configuration
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
+          role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-seq-retrieval-image-upload
+          aws-region: us-east-1
+      - name: Amazon ECR login
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
       - name: Build container image
         uses: docker/build-push-action@v5
         with:
@@ -32,6 +58,11 @@ jobs:
         with:
           name: seq_retrieval_image
           path: /tmp/pavi_seq_retrieval_docker_image.tar
+      - name: Load, tag and push image to registry
+        run: |
+          docker load --input /tmp/pavi_seq_retrieval_docker_image.tar
+          docker tag agr_pavi/pipeline_seq_retrieval:latest ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_seq_retrieval:${{needs.container-build-context.outputs.tagname}}
+          docker push ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_seq_retrieval:${{needs.container-build-context.outputs.tagname}}
   pipeline-seq-retrieval-code-checks:
     name: pipeline/seq_retrieval code checks
     runs-on: ubuntu-22.04
@@ -60,6 +91,10 @@ jobs:
           make run-unit-tests
   pipeline-alignment-container-image-build:
     name: pipeline/alignment container-image build
+    needs:
+      - container-build-context
+    permissions:
+      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
     runs-on: ubuntu-22.04
     steps:
       - name: Check out repository code
@@ -70,6 +105,17 @@ jobs:
             pipeline/alignment/
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+      # This step will configure environment variables to be used by all steps
+      # involving AWS interaction further down
+      - name: AWS credentials configuration
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
+          role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-alignment-image-upload
+          aws-region: us-east-1
+      - name: Amazon ECR login
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
       - name: Build container image
         uses: docker/build-push-action@v5
         with:
@@ -82,6 +128,11 @@ jobs:
         with:
           name: alignment_image
           path: /tmp/pavi_alignment_docker_image.tar
+      - name: Load, tag and push image to registry
+        run: |
+          docker load --input /tmp/pavi_alignment_docker_image.tar
+          docker tag agr_pavi/pipeline_alignment:latest ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_alignment:${{needs.container-build-context.outputs.tagname}}
+          docker push ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_alignment:${{needs.container-build-context.outputs.tagname}}
   pipeline-alignment-unit-testing:
     name: pipeline/alignment unit testing
     needs:
@@ -114,8 +165,11 @@ jobs:
   pipeline-workflow-integration-testing:
     name: pipeline/workflow integration testing
     needs:
+      - container-build-context
       - pipeline-seq-retrieval-container-image-build
       - pipeline-alignment-container-image-build
+    permissions:
+      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
     runs-on: ubuntu-22.04
     defaults:
       run:
@@ -146,17 +200,28 @@ jobs:
       - name: Load alignment Docker image
         run: |
           docker load --input /tmp/pavi_alignment_docker_image.tar
-      - name: Run integration test
+      - name: Run local integration test
         run: |
-          make run-integration-test
+          make run-integration-test-local
+      # This step will configure environment variables to be used by all steps
+      # involving AWS interaction further down
+      - name: AWS credentials configuration
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
+          role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-workflow-aws-test
+          aws-region: us-east-1
+      - name: Run AWS integration test
+        run: |
+          make run-integration-test-aws TAG_NAME=${{needs.container-build-context.outputs.tagname}}
   pipeline-aws-infra-code-checks:
     name: pipeline/aws_infra code checks
     permissions:
       id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
     runs-on: ubuntu-22.04
     defaults:
       run:
-        working-directory: pipeline/aws_infra
+        working-directory: ./pipeline/aws_infra
     steps:
     - name: Check out repository code
       uses: actions/checkout@v4
@@ -193,3 +258,128 @@ jobs:
         aws-region: us-east-1
     - name: Validate production CDK stack code
       run: cdk diff PaviPipelineCdkStack
+  api-code-checks:
+    name: API code checks
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./api/
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          sparse-checkout: |
+            api/
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Python typing test
+        run: |
+          make run-python-type-check
+      - name: Python style test
+        run: |
+          make run-python-style-check
+  api-container-image-build:
+    name: API container-image build
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build container image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./
+          file: api/Dockerfile
+          push: false
+          tags: agr_pavi/api:latest
+          outputs: type=docker,dest=/tmp/pavi_api_docker_image.tar
+      - name: Upload image as artifact (share between jobs)
+        uses: actions/upload-artifact@v4
+        with:
+          name: api_image
+          path: /tmp/pavi_api_docker_image.tar
+  api-unit-integration-testing:
+    name: API unit and integration testing
+    needs:
+      - pipeline-seq-retrieval-container-image-build
+      - pipeline-alignment-container-image-build
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./api/
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Download seq_retrieval image artifact (from previous job)
+        uses: actions/download-artifact@v4
+        with:
+          name: seq_retrieval_image
+          path: /tmp
+      - name: Download alignment image artifact (from previous job)
+        uses: actions/download-artifact@v4
+        with:
+          name: alignment_image
+          path: /tmp
+      - name: Load seq_retrieval Docker image
+        run: |
+          docker load --input /tmp/pavi_seq_retrieval_docker_image.tar
+      - name: Load alignment Docker image
+        run: |
+          docker load --input /tmp/pavi_alignment_docker_image.tar
+      - name: Run unit and integration tests
+        run: |
+          make run-tests
+  api-container-integration-testing:
+    name: API container integration testing (AWS batch execution)
+    needs:
+      - container-build-context
+      - pipeline-seq-retrieval-container-image-build
+      - pipeline-alignment-container-image-build
+      - api-container-image-build
+    permissions:
+      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./api/
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          sparse-checkout: |
+            api/
+            pipeline/workflow/tests/
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Download API image artifact (from previous job)
+        uses: actions/download-artifact@v4
+        with:
+          name: api_image
+          path: /tmp
+      - name: Load API Docker image
+        run: |
+          docker load --input /tmp/pavi_api_docker_image.tar
+      # This step will configure environment variables to be used by all steps
+      # involving AWS interaction further down
+      - name: AWS credentials configuration
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
+          role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-api-aws-test
+          aws-region: us-east-1
+      - name: Run container integration test (AWS execution)
+        run: |
+          make run-integration-test-container TAG_NAME=${{needs.container-build-context.outputs.tagname}}

api/.coveragerc

@@ -0,0 +1,16 @@
+[run]
+source = src
+omit =
+    *__init__*
+
+[report]
+exclude_lines =
+    # Exclude defensive assertion code
+    raise AssertionError
+    raise NotImplementedError
+
+    # Exclude main method calling (not testable through unit testing)
+    if __name__ == .__main__.:
+
+    # Exclude all functions labeled with "pragma: no cover"
+    pragma: no cover

api/.flake8

@@ -0,0 +1,6 @@
+[flake8]
+ignore = E266, E501, W503
+max-complexity = 18
+select = B,C,E,F,W,T4
+exclude = venv,node_modules
+per-file-ignores = __init__.py:F401

api/.gitignore

@@ -0,0 +1,9 @@
+# Local dev env executables and symlinks
+nextflow.sh
+nextflow.config
+*.nf
+# Nextflow output files
+seq_regions*.json
+.nextflow*
+pipeline-results*/
+work/

api/Dockerfile

@@ -0,0 +1,31 @@
+# Note: Building this container requires the repository root directory as build context directory (../)
+# as files from other components (workflow) must be included in the API container image
+FROM python:3.12-slim-bullseye
+
+WORKDIR /usr/src/app
+
+RUN apt update && apt upgrade -y
+RUN apt install -y make curl
+
+COPY api/requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY api/src/ ./
+
+# Nextflow requirements, executable and configs
+RUN apt install -y default-jre
+COPY pipeline/workflow/Makefile workflow_makefile
+RUN make -f workflow_makefile nextflow.sh
+COPY pipeline/workflow/nextflow.config nextflow.config
+COPY pipeline/workflow/protein-msa.nf protein-msa.nf
+
+# Expose necessary ports
+EXPOSE 8080
+
+# Default environment variables
+ENV API_RESULTS_PATH_PREFIX="s3://agr-pavi-pipeline-nextflow/main/results/"
+ENV API_EXECUTION_ENV="aws"
+ENV API_PIPELINE_IMAGE_TAG="main"
+
+# Start the API server application
+CMD [ "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"]

api/Dockerfile.dockerignore

@@ -0,0 +1,9 @@
+# Local dev env symlinks
+api/src/nextflow.sh
+api/src/nextflow.config
+api/src/*.nf
+# Nextflow output files
+api/src/seq_regions*.json
+api/src/.nextflow*
+api/src/pipeline-results*/
+api/src/work/