forked from PolusAI/workflow-inference-compiler
-
Notifications
You must be signed in to change notification settings - Fork 1
224 lines (199 loc) · 9.18 KB
/
run_workflows.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
name: Run Workflows
on:
workflow_dispatch:
# Have to declare parameters here for those that will be sent through 'workflow-dispatch'
# event in branch_dispatch.yml. Otherwise, there'll be github API errors:
# '"message": "Unexpected inputs provided: ...",'
inputs:
event_type:
description: An arbitrary string used to dispatch steps
required: true
type: string
commit_message:
description: The commit message
required: true
type: string
sender_repo:
description: The repository which initiated the workflow dispatch
required: true
type: string
sender_repo_owner:
description: The account name of the repository initiated the workflow dispatch
required: true
type: string
wic_owner:
description: The account name of the wic repository
required: true
type: string
wic_ref:
description: The branch name within the wic repository
required: true
type: string
mm-workflows_owner:
description: The account name of the mm-workflows repository
required: true
type: string
mm-workflows_ref:
description: The branch name within the mm-workflows repository
required: true
type: string
image-workflows_owner:
description: The account name of the image-workflows repository
required: true
type: string
image-workflows_ref:
description: The branch name within the image-workflows repository
required: true
type: string
defaults:
run:
shell: bash -l {0} # Invoke bash in login mode, NOT interactive mode.
# This will cause bash to look for the startup file ~/.bash_profile, NOT ~/.bashrc
# This is important since conda init writes to ~/.bashrc
permissions:
actions: read
contents: read
pull-requests: read
jobs:
run_workflows:
# See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency
# This will prevent DOS attacks from people blasting the CI with rapid fire commits.
concurrency:
group: ${{ github.workflow }}-cwltool-${{ github.ref }}-${{ inputs.sender_repo }}-${{ inputs.mm-workflows_ref}}
cancel-in-progress: true
runs-on: [self-hosted, linux]
steps:
- name: Checkout workflow-inference-compiler
uses: actions/checkout@v3
with:
repository: ${{ inputs.wic_owner }}/workflow-inference-compiler
ref: ${{ inputs.wic_ref }}
path: workflow-inference-compiler
- name: Checkout biobb_adapters
if: always()
uses: actions/checkout@v3
with:
# NOTE: temporarily hardcode jfennick & master because we can only
# have up to 10 input parameters for workflow_dispatch...
repository: jfennick/biobb_adapters
ref: master
path: biobb_adapters
- name: Checkout mm-workflows
if: always()
uses: actions/checkout@v3
with:
repository: ${{ inputs.mm-workflows_owner }}/mm-workflows
ref: ${{ inputs.mm-workflows_ref }}
path: mm-workflows
- name: Checkout image-workflows
if: always()
uses: actions/checkout@v3
with:
repository: ${{ inputs.image-workflows_owner }}/image-workflows
ref: ${{ inputs.image-workflows_ref }}
path: image-workflows
- name: Remove old global config
if: always()
run: rm -rf "/home/$(whoami)/wic/" && rm -rf "/home/$(whoami)/.toil/"
# For self-hosted runners, make sure we use new global config settings
# Using pypy increases performance / decreases cwltool runtime by about a minute.
- name: Append pypy to conda environment files
if: runner.os != 'Windows'
run: cd workflow-inference-compiler/install/ && echo " - pypy" >> system_deps.yml
- name: Remove old mamba environment
if: always()
run: rm -rf "/home/$(whoami)/miniconda3/envs/wic_github_actions/"
# For self-hosted runners, make sure we install into a new mamba environment
# NOTE: Every time the github self-hosted runner executes, it sets "set -e" in ~/.bash_profile
# So if we rm -rf the old mamba environment without also removing the mamba init code in ~/.bash_profile
# (or removing the file altogether), then unless we immediately re-create the environment,
# (i.e. if we try to run any other commands between removing and re-creating the environment)
# we will get "EnvironmentNameNotFound: Could not find conda environment: wic_github_actions"
# and (again, due to "set -e") the workflow step will fail.
- name: Setup mamba (linux, macos)
if: always()
uses: conda-incubator/setup-miniconda@v3.0.1
with:
miniforge-variant: Mambaforge-pypy3
miniforge-version: latest
environment-file: workflow-inference-compiler/install/system_deps.yml
activate-environment: wic_github_actions
channels: conda-forge
python-version: "3.9.*" # pypy is not yet compatible with 3.10 and 3.11
# - name: Docker pull
# if: always()
# run: cd mm-workflows/ && ./dockerPull.sh
# # For self-hosted runners, make sure the docker cache is up-to-date.
- name: Install Workflow Inference Compiler
if: always()
run: cd workflow-inference-compiler/ && pip install ".[all]"
- name: Install Molecular Modeling Workflows
if: always()
# Also run mm-workflows command to generate
# mm-workflows/autogenerated/schemas/config_schemas.json
# NOTE: Use ".[test]" instead of ".[all]"
# We do not want or need to install the workflow_deps extra.
# (Many of the packages conflict with pypy.)
run: cd mm-workflows/ && pip install ".[test]" && mm-workflows --generate_schemas
- name: Generate WIC Python API Workflows (*.py -> *.wic)
if: always()
run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()'
- name: Generate WIC Validation Jsonschema
if: always()
run: cd workflow-inference-compiler/ && wic --generate_schemas
# Please read docs/validation.md#Property-Based-Testing
# This is essentially an integration test for all of the
# WIC Python API workflows as well as the WIC Python API itself.
- name: Validate WIC Python API Workflows (*.py -> *.wic)
if: always()
run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()'
- name: cwl-docker-extract (i.e. recursively docker pull)
if: always()
run: cd workflow-inference-compiler/ && pytest -k test_cwl_docker_extract
# For self-hosted runners, make sure the docker cache is up-to-date.
- name: PyTest Run Workflows
if: always()
# NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252
run: cd workflow-inference-compiler/ && pytest -k test_run_workflows_on_push --workers 8 --cwl_runner cwltool # --cov
# NOTE: The steps below are for repository_dispatch only. For all other steps, please insert above
# this comment.
# Need to store success value in environment variable, rather than using 'success()' in-line inside a run tag.
# Otherwise: "The workflow is not valid. ... Unrecognized function: 'success'."
# https://github.com/actions/runner/issues/834#issuecomment-889484016
- name: The workflow has succeeded
if: ${{ success() }}
run: |
echo 'workflow_success=true' >> "$GITHUB_ENV"
# It is not clear from the documentation, but the 'success()' and 'failure()' functions
# do not consider skipped steps. Specifically, the above 'success()' function will not
# affect the 'failure()' function here.
# https://docs.github.com/en/actions/learn-github-actions/expressions#status-check-function
- name: The workflow has failed
if: ${{ failure() }}
run: |
echo 'workflow_success=false' >> "$GITHUB_ENV"
# See token.md
- name: Generate a token
if: always()
id: generate_token
uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92
with:
app_id: ${{ secrets.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}
- name: Reply CI results to sender
# In case of failure, we still need to return the failure status to the original repository.
# Use 'always()' so this step runs even if there's a failure and use the bash if-statement
# to only run this step only if the repository_dispatch sends the signal.
# https://github.com/actions/runner/issues/834#issuecomment-907096707
# Use inputs.sender_repo to reply the original sender.
if: always()
uses: ./workflow-inference-compiler/.github/my_actions/reply_sender/ # Must start with ./
with:
github_repository: ${{ github.repository }}
event_type: ${{ inputs.event_type }}
sender_repo: ${{ inputs.sender_repo }}
operating_system: self-hosted-linux
commit_message: ${{ inputs.commit_message }}
mm_workflows_ref: ${{ inputs.mm-workflows_ref }}
workflow_success: ${{ env.workflow_success }}
access_token: ${{ steps.generate_token.outputs.token }}