Run Workflows #458
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Workflows | |
| on: | |
| workflow_dispatch: | |
| # Have to declare parameters here for those that will be sent through 'workflow-dispatch' | |
| # event in branch_dispatch.yml. Otherwise, there'll be github API errors: | |
| # '"message": "Unexpected inputs provided: ...",' | |
| inputs: | |
| event_type: | |
| description: An arbitrary string used to dispatch steps | |
| required: true | |
| type: string | |
| commit_message: | |
| description: The commit message | |
| required: true | |
| type: string | |
| sender_repo: | |
| description: The repository which initiated the workflow dispatch | |
| required: true | |
| type: string | |
| sender_repo_owner: | |
| description: The account name of the repository initiated the workflow dispatch | |
| required: true | |
| type: string | |
| wic_owner: | |
| description: The account name of the wic repository | |
| required: true | |
| type: string | |
| wic_ref: | |
| description: The branch name within the wic repository | |
| required: true | |
| type: string | |
| mm-workflows_owner: | |
| description: The account name of the mm-workflows repository | |
| required: true | |
| type: string | |
| mm-workflows_ref: | |
| description: The branch name within the mm-workflows repository | |
| required: true | |
| type: string | |
| image-workflows_owner: | |
| description: The account name of the image-workflows repository | |
| required: true | |
| type: string | |
| image-workflows_ref: | |
| description: The branch name within the image-workflows repository | |
| required: true | |
| type: string | |
| defaults: | |
| run: | |
| shell: bash -l {0} # Invoke bash in login mode, NOT interactive mode. | |
| # This will cause bash to look for the startup file ~/.bash_profile, NOT ~/.bashrc | |
| # This is important since conda init writes to ~/.bashrc | |
| permissions: | |
| actions: read | |
| contents: read | |
| pull-requests: read | |
| jobs: | |
| run_workflows: | |
| # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency | |
| # This will prevent DOS attacks from people blasting the CI with rapid fire commits. | |
| concurrency: | |
| group: ${{ github.workflow }}-cwltool-${{ github.ref }}-${{ inputs.sender_repo }}-${{ inputs.mm-workflows_ref}} | |
| cancel-in-progress: true | |
| runs-on: [self-hosted, linux] | |
| steps: | |
| - name: Checkout workflow-inference-compiler | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ inputs.wic_owner }}/workflow-inference-compiler | |
| ref: ${{ inputs.wic_ref }} | |
| path: workflow-inference-compiler | |
| - name: Checkout biobb_adapters | |
| if: always() | |
| uses: actions/checkout@v3 | |
| with: | |
| # NOTE: temporarily hardcode jfennick & master because we can only | |
| # have up to 10 input parameters for workflow_dispatch... | |
| repository: jfennick/biobb_adapters | |
| ref: master | |
| path: biobb_adapters | |
| - name: Checkout mm-workflows | |
| if: always() | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ inputs.mm-workflows_owner }}/mm-workflows | |
| ref: ${{ inputs.mm-workflows_ref }} | |
| path: mm-workflows | |
| - name: Checkout image-workflows | |
| if: always() | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ inputs.image-workflows_owner }}/image-workflows | |
| ref: ${{ inputs.image-workflows_ref }} | |
| path: image-workflows | |
| - name: Remove old global config | |
| if: always() | |
| run: rm -rf "/home/$(whoami)/wic/" && rm -rf "/home/$(whoami)/.toil/" | |
| # For self-hosted runners, make sure we use new global config settings | |
| # Using pypy increases performance / decreases cwltool runtime by about a minute. | |
| - name: Append pypy to conda environment files | |
| if: runner.os != 'Windows' | |
| run: cd workflow-inference-compiler/install/ && echo " - pypy" >> system_deps.yml | |
| - name: Remove old mamba environment | |
| if: always() | |
| run: rm -rf "/home/$(whoami)/miniconda3/envs/wic_github_actions/" | |
| # For self-hosted runners, make sure we install into a new mamba environment | |
| # NOTE: Every time the github self-hosted runner executes, it sets "set -e" in ~/.bash_profile | |
| # So if we rm -rf the old mamba environment without also removing the mamba init code in ~/.bash_profile | |
| # (or removing the file altogether), then unless we immediately re-create the environment, | |
| # (i.e. if we try to run any other commands between removing and re-creating the environment) | |
| # we will get "EnvironmentNameNotFound: Could not find conda environment: wic_github_actions" | |
| # and (again, due to "set -e") the workflow step will fail. | |
| - name: Setup mamba (linux, macos) | |
| if: always() | |
| uses: conda-incubator/setup-miniconda@v3.0.1 | |
| with: | |
| miniforge-variant: Mambaforge-pypy3 | |
| miniforge-version: latest | |
| environment-file: workflow-inference-compiler/install/system_deps.yml | |
| activate-environment: wic_github_actions | |
| channels: conda-forge | |
| python-version: "3.9.*" # pypy is not yet compatible with 3.10 and 3.11 | |
| # - name: Docker pull | |
| # if: always() | |
| # run: cd mm-workflows/ && ./dockerPull.sh | |
| # # For self-hosted runners, make sure the docker cache is up-to-date. | |
| - name: Install Workflow Inference Compiler | |
| if: always() | |
| run: cd workflow-inference-compiler/ && pip install ".[all_except_runner_src]" | |
| - name: Install Molecular Modeling Workflows | |
| if: always() | |
| # Also run mm-workflows command to generate | |
| # mm-workflows/autogenerated/schemas/config_schemas.json | |
| # NOTE: Use ".[test]" instead of ".[all_except_runner_src]" | |
| # We do not want or need to install the workflow_deps extra. | |
| # (Many of the packages conflict with pypy.) | |
| run: cd mm-workflows/ && pip install ".[test]" && mm-workflows --generate_schemas | |
| - name: Generate WIC Python API Workflows (*.py -> *.wic) | |
| if: always() | |
| run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()' | |
| - name: Generate WIC Validation Jsonschema | |
| if: always() | |
| run: cd workflow-inference-compiler/ && wic --generate_schemas | |
| # Please read docs/validation.md#Property-Based-Testing | |
| # This is essentially an integration test for all of the | |
| # WIC Python API workflows as well as the WIC Python API itself. | |
| - name: Validate WIC Python API Workflows (*.py -> *.wic) | |
| if: always() | |
| run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()' | |
| - name: cwl-docker-extract (i.e. recursively docker pull) | |
| if: always() | |
| run: cd workflow-inference-compiler/ && pytest -k test_cwl_docker_extract | |
| # For self-hosted runners, make sure the docker cache is up-to-date. | |
| - name: PyTest Run Workflows | |
| if: always() | |
| # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 | |
| run: cd workflow-inference-compiler/ && pytest -k test_run_workflows_on_push --workers 8 --cwl_runner cwltool # --cov | |
| # NOTE: The steps below are for repository_dispatch only. For all other steps, please insert above | |
| # this comment. | |
| # Need to store success value in environment variable, rather than using 'success()' in-line inside a run tag. | |
| # Otherwise: "The workflow is not valid. ... Unrecognized function: 'success'." | |
| # https://github.com/actions/runner/issues/834#issuecomment-889484016 | |
| - name: The workflow has succeeded | |
| if: ${{ success() }} | |
| run: | | |
| echo 'workflow_success=true' >> "$GITHUB_ENV" | |
| # It is not clear from the documentation, but the 'success()' and 'failure()' functions | |
| # do not consider skipped steps. Specifically, the above 'success()' function will not | |
| # affect the 'failure()' function here. | |
| # https://docs.github.com/en/actions/learn-github-actions/expressions#status-check-function | |
| - name: The workflow has failed | |
| if: ${{ failure() }} | |
| run: | | |
| echo 'workflow_success=false' >> "$GITHUB_ENV" | |
| # See token.md | |
| - name: Generate a token | |
| if: always() | |
| id: generate_token | |
| uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 | |
| with: | |
| app_id: ${{ secrets.APP_ID }} | |
| private_key: ${{ secrets.APP_PRIVATE_KEY }} | |
| - name: Reply CI results to sender | |
| # In case of failure, we still need to return the failure status to the original repository. | |
| # Use 'always()' so this step runs even if there's a failure and use the bash if-statement | |
| # to only run this step only if the repository_dispatch sends the signal. | |
| # https://github.com/actions/runner/issues/834#issuecomment-907096707 | |
| # Use inputs.sender_repo to reply the original sender. | |
| if: always() | |
| uses: ./workflow-inference-compiler/.github/my_actions/reply_sender/ # Must start with ./ | |
| with: | |
| github_repository: ${{ github.repository }} | |
| event_type: ${{ inputs.event_type }} | |
| sender_repo: ${{ inputs.sender_repo }} | |
| operating_system: self-hosted-linux | |
| commit_message: ${{ inputs.commit_message }} | |
| mm_workflows_ref: ${{ inputs.mm-workflows_ref }} | |
| workflow_success: ${{ env.workflow_success }} | |
| access_token: ${{ steps.generate_token.outputs.token }} |