-
Notifications
You must be signed in to change notification settings - Fork 0
272 lines (241 loc) · 8.43 KB
/
wipac-cicd.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
name: wipac ci/cd
on: [ push ]
env:
THIS_IMAGE_WITH_TAG: 'ghcr.io/wipacrepo/skydriver:vX.Y.Z'
EWMS_PILOT_TASK_TIMEOUT: 999
SCAN_BACKLOG_RUNNER_SHORT_DELAY: 1
SCAN_BACKLOG_RUNNER_DELAY: 1
SCAN_BACKLOG_PENDING_ENTRY_TTL_REVIVE: 200
LOG_LEVEL: debug
# mandatory env vars...
EWMS_ADDRESS: http://localhost:8081
EWMS_TOKEN_URL: 65f3b929
EWMS_CLIENT_ID: b75a974d
EWMS_CLIENT_SECRET: 411b16fe
S3_URL: a4f92304
S3_ACCESS_KEY_ID: 36c5c849
S3_ACCESS_KEY_ID__K8S_SECRET_KEY: 230ec9dc
S3_SECRET_KEY: 8dea68a1
S3_SECRET_KEY__K8S_SECRET_KEY: cdf7c60b
S3_BUCKET: 72017610
K8S_SECRET_NAME: super-secrets
MIN_SKYMAP_SCANNER_TAG: "v3.21.2" # TODO: remove once skyscan v4 is out (that's the real min)
jobs:
py-versions:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.versions.outputs.matrix }}
steps:
- uses: actions/checkout@v4
- id: versions
uses: WIPACrepo/wipac-dev-py-versions-action@v2.5
#############################################################################
# LINTERS
#############################################################################
flake8:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
- uses: WIPACrepo/wipac-dev-flake8-action@v1.2
with:
max-complexity: 10
mypy:
needs: [ py-versions ]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.py3 }}
- uses: WIPACrepo/wipac-dev-mypy-action@v2.0
#############################################################################
# PACKAGING
#############################################################################
writable-branch-detect:
runs-on: ubuntu-latest
outputs:
OKAY: ${{ steps.detect.outputs.OKAY }}
steps:
- name: is this a bot-writable branch?
id: detect
# dependabot can't access normal secrets
# & don't run non-branch triggers (like tags)
# & we don't want to trigger an update on PR's merge to main/master/default (which is a branch)
run: |
set -euo pipefail
if [[ \
${{github.actor}} != 'dependabot[bot]' && \
${{github.ref_type}} == 'branch' && \
${{format('refs/heads/{0}', github.event.repository.default_branch)}} != ${{github.ref}} \
]]; then
echo "OKAY=true" >> "$GITHUB_OUTPUT"
echo "yes, this branch is compatible"
else
echo "OKAY=false" >> "$GITHUB_OUTPUT"
echo "no, this branch is incompatible"
fi
py-setup:
needs: [ writable-branch-detect ]
runs-on: ubuntu-latest
steps:
- if: needs.writable-branch-detect.outputs.OKAY == 'true'
uses: actions/checkout@v4
with:
token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
- if: needs.writable-branch-detect.outputs.OKAY == 'true'
uses: WIPACrepo/wipac-dev-py-setup-action@v3.1
with:
base-keywords: WIPAC IceCube
py-dependencies:
needs: [ writable-branch-detect ]
runs-on: ubuntu-latest
steps:
- if: needs.writable-branch-detect.outputs.OKAY == 'true'
uses: actions/checkout@v4
with:
token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
- if: needs.writable-branch-detect.outputs.OKAY == 'true'
uses: docker/setup-buildx-action@v2
- if: needs.writable-branch-detect.outputs.OKAY == 'true'
uses: docker/build-push-action@v3
with:
context: .
file: Dockerfile
tags: skydriver:py-dep-this
load: true
- if: needs.writable-branch-detect.outputs.OKAY == 'true'
uses: WIPACrepo/wipac-dev-py-dependencies-action@v2.1
#############################################################################
# TESTS
#############################################################################
unit-tests:
needs: [ py-versions ]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.py3 }}
- name: pip install
run: |
set -euo pipefail
pip install --upgrade pip wheel setuptools
pip install .[tests]
- name: test
run: |
set -euo pipefail
pytest -vvv tests/unit --exitfirst
- name: Dump logs
if: always()
run: |
set -euo pipefail
cat pytest.logs || true
integration-tests:
needs: [ py-versions ]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
services:
mongo:
image: bitnami/mongodb:4
ports:
- 27017:27017
steps:
- uses: actions/checkout@v4
- uses: docker/setup-buildx-action@v2
- uses: docker/build-push-action@v3
with:
context: .
cache-from: type=gha
cache-to: type=gha,mode=min
file: Dockerfile
tags: wipac/skydriver:local
load: true
- name: test
run: |
set -euo pipefail
pip install .[tests]
python tests/integration/dummy_ewms.py &> ./dummy_ewms.out &
export LATEST_TAG=$( \
curl -I https://github.com/icecube/skymap_scanner/releases/latest \
| awk -F '/' '/^location/ {print substr($NF, 1, length($NF)-1)}' \
| sed 's/v//' \
)
echo $LATEST_TAG # this tag may be off if there's a delay between GH release & docker hub
# make test script
DIR="test-script-dir"
mkdir $DIR
echo "#!/bin/bash" >> $DIR/test-script.sh
echo "set -xe" >> $DIR/test-script.sh
echo "pip install .[tests]" >> $DIR/test-script.sh
echo "python -m pytest -vvv tests/integration --exitfirst" >> $DIR/test-script.sh
chmod +x $DIR/test-script.sh
cat $DIR/test-script.sh
docker run --network="host" --rm -i --name test \
--env LATEST_TAG=$LATEST_TAG \
--env THIS_IMAGE_WITH_TAG=$THIS_IMAGE_WITH_TAG \
--env K8S_SECRET_NAME=$K8S_SECRET_NAME \
$(env | grep '^SKYSCAN_' | awk '$0="--env "$0') \
$(env | grep '^EWMS_' | awk '$0="--env "$0') \
$(env | grep '^S3_' | awk '$0="--env "$0') \
$(env | grep '^CI' | awk '$0="--env "$0') \
$(env | grep '^CI_' | awk '$0="--env "$0') \
$(env | grep '^SCAN_' | awk '$0="--env "$0') \
$(env | grep '^MIN_SKYMAP_SCANNER_TAG' | awk '$0="--env "$0') \
--mount type=bind,source=$(realpath $DIR),target=/local/$DIR \
wipac/skydriver:local \
/local/$DIR/test-script.sh
#
- name: dump test logs
if: always()
run: |
set -euo pipefail
docker logs test || true
- name: dump dummy-ewms logs
if: always()
run: |
set -euo pipefail
cat ./dummy_ewms.out
- name: dump mongo logs
if: always()
run: |
set -euo pipefail
docker logs "${{ job.services.mongo.id }}" || true
test-build-docker:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: docker/setup-buildx-action@v2
- uses: docker/build-push-action@v3
with:
context: .
file: Dockerfile
tags: wipac/skydriver:local
release:
# only run on main/master/default
if: format('refs/heads/{0}', github.event.repository.default_branch) == github.ref
needs: [ flake8, mypy, py-setup, py-dependencies, unit-tests, integration-tests, test-build-docker ]
runs-on: ubuntu-latest
concurrency: release
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
- name: Python Semantic Release
uses: python-semantic-release/python-semantic-release@v7.34.6
with:
github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
# repository_username: __token__
# repository_password: ${{ secrets.PYPI_TOKEN }}