Run Workflows Weekly #48
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Workflows Weekly | |
| # The purpose of this CI file is to stress test our workflows at scale. | |
| # The *_weekly.wic workflows scatter over a much larger number of examples. | |
| # Note that toil-cwl-runner is slightly slower than cwltool, but | |
| # there are deadlocking problems with `cwltool --parallel` | |
| on: | |
| # push: # Do NOT run weekly workflows on every push! | |
| # pull_request: (either on origin or on upstream pull request) | |
| schedule: | |
| # Instead, run it every Tuesday at midnight. | |
| - cron: '0 0 * * 2' | |
| workflow_dispatch: | |
| defaults: | |
| run: | |
| shell: bash -l {0} # Invoke bash in login mode, NOT interactive mode. | |
| # This will cause bash to look for the startup file ~/.bash_profile, NOT ~/.bashrc | |
| # This is important since conda init writes to ~/.bashrc | |
| permissions: | |
| actions: read | |
| contents: read | |
| pull-requests: read | |
| jobs: | |
| run_weekly_toil: | |
| # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency | |
| # This will prevent DOS attacks from people blasting the CI with rapid fire commits. | |
| concurrency: | |
| group: ${{ github.workflow }}-weekly-toil-${{ github.ref }} | |
| cancel-in-progress: true | |
| runs-on: [self-hosted, linux] | |
| steps: | |
| - name: Checkout workflow-inference-compiler | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ github.repository_owner }}/workflow-inference-compiler | |
| ref: master | |
| path: workflow-inference-compiler | |
| - name: Checkout biobb_adapters | |
| if: always() | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: jfennick/biobb_adapters | |
| ref: master | |
| path: biobb_adapters | |
| - name: Checkout mm-workflows | |
| if: always() | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ github.repository_owner }}/mm-workflows | |
| ref: main | |
| path: mm-workflows | |
| - name: Checkout image-workflows | |
| if: always() | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ github.repository_owner }}/image-workflows | |
| ref: main | |
| path: image-workflows | |
| - name: Remove old global config | |
| if: always() | |
| run: rm -rf "/home/$(whoami)/wic/" && rm -rf "/home/$(whoami)/.toil/" | |
| # For self-hosted runners, make sure we use new global config settings | |
| - name: Remove old mamba environment | |
| if: always() | |
| run: rm -rf "/home/$(whoami)/miniconda3/envs/wic_github_actions/" | |
| # For self-hosted runners, make sure we install into a new mamba environment | |
| # Toil is currently incompatible with pypy | |
| # - name: Append pypy to conda environment files | |
| # if: runner.os != 'Windows' | |
| # run: cd workflow-inference-compiler/install/ && echo " - pypy" >> system_deps.yml | |
| - name: Setup mamba (linux, macos) | |
| if: always() | |
| uses: conda-incubator/setup-miniconda@v3.0.1 | |
| with: | |
| miniforge-variant: Mambaforge # NOT Mambaforge-pypy3 | |
| # Toil is (currently) incompatible with pypy. Mambaforge-pypy3 | |
| # installs pypy in the base environment (only). Although we are using | |
| # another environment, better to avoid the problem altogether by | |
| # not using Mambaforge-pypy3 | |
| miniforge-version: latest | |
| environment-file: workflow-inference-compiler/install/system_deps.yml | |
| activate-environment: wic_github_actions | |
| use-mamba: true | |
| channels: conda-forge | |
| python-version: "3.9.*" # pypy is not yet compatible with 3.10 and 3.11 | |
| # - name: Docker pull | |
| # if: always() | |
| # run: cd mm-workflows/ && ./dockerPull.sh | |
| # # For self-hosted runners, make sure the docker cache is up-to-date. | |
| - name: Install Workflow Inference Compiler | |
| if: always() | |
| run: cd workflow-inference-compiler/ && pip install ".[all_except_runner_src]" | |
| - name: Install Molecular Modeling Workflows | |
| if: always() | |
| # Also run mm-workflows command to generate | |
| # mm-workflows/autogenerated/schemas/config_schemas.json | |
| # NOTE: Use ".[test]" instead of ".[all_except_runner_src]" | |
| # We do not want or need to install the workflow_deps extra. | |
| # (Many of the packages conflict with pypy.) | |
| run: cd mm-workflows/ && pip install ".[test]" && mm-workflows --generate_schemas | |
| - name: Generate Sophios Python API Workflows (*.py -> *.wic) | |
| if: always() | |
| run: cd workflow-inference-compiler/ && python -c 'import sophios; import sophios.plugins; sophios.plugins.blindly_execute_python_workflows()' | |
| - name: Generate Sophios Validation Jsonschema | |
| if: always() | |
| run: cd workflow-inference-compiler/ && sophios --generate_schemas | |
| # Please read docs/validation.md#Property-Based-Testing | |
| # This is essentially an integration test for all of the | |
| # Sophios Python API workflows as well as the WIC Python API itself. | |
| - name: Validate Sophios Python API Workflows (*.py -> *.wic) | |
| if: always() | |
| run: cd workflow-inference-compiler/ && python -c 'import sophios; import sophios.plugins; sophios.plugins.blindly_execute_python_workflows()' | |
| - name: cwl-docker-extract (i.e. recursively docker pull) | |
| if: always() | |
| run: cd workflow-inference-compiler/ && pytest -k test_cwl_docker_extract | |
| # For self-hosted runners, make sure the docker cache is up-to-date. | |
| - name: PyTest Run Workflows (On Push) | |
| if: always() | |
| # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 | |
| run: cd workflow-inference-compiler/ && pytest -k test_run_workflows_on_push --workers 8 --cwl_runner toil-cwl-runner # --cov | |
| - name: PyTest Run Inlined Workflows (On Push) | |
| if: always() | |
| # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 | |
| run: cd workflow-inference-compiler/ && pytest -k test_run_inlined_workflows_on_push --workers 8 --cwl_runner toil-cwl-runner # --cov | |
| - name: PyTest Run Workflows (Weekly) | |
| if: always() | |
| # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 | |
| run: cd workflow-inference-compiler/ && pytest -k test_run_workflows_weekly --workers 8 --cwl_runner toil-cwl-runner # --cov | |
| - name: PyTest Run Inlined Workflows (Weekly) | |
| if: always() | |
| # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 | |
| run: cd workflow-inference-compiler/ && pytest -k test_run_inlined_workflows_weekly --workers 8 --cwl_runner toil-cwl-runner # --cov |