Skip to content

Commit 9453aca

Browse files
ric-evansric-evans
committed
tests: query the SkyScanK8sJobs coll (fr) - 1
1 parent 206b6dd commit 9453aca

File tree

2 files changed

+183
-9
lines changed

2 files changed

+183
-9
lines changed

‎.github/workflows/wipac-cicd.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: wipac ci/cd
33
on: [ push ]
44

55
env:
6-
THIS_IMAGE_WITH_TAG: 'ghcr.io/wipacrepo/skydriver:latest'
6+
THIS_IMAGE_WITH_TAG: 'ghcr.io/wipacrepo/skydriver:vX.Y.Z'
77
EWMS_PILOT_TASK_TIMEOUT: 999
88
SCAN_BACKLOG_RUNNER_SHORT_DELAY: 1
99
SCAN_BACKLOG_RUNNER_DELAY: 1

‎tests/integration/test_rest_routes.py

Lines changed: 182 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -126,11 +126,11 @@ async def _launch_scan(
126126
)
127127

128128
# query the ScanRequests coll
129-
doc = await mongo_client["SkyDriver_DB"]["ScanRequests"].find_one(
129+
doc_sr = await mongo_client["SkyDriver_DB"]["ScanRequests"].find_one(
130130
{"scan_id": post_resp["scan_id"]}, {"_id": 0}
131131
)
132-
pprint.pprint(doc)
133-
assert doc == dict(
132+
pprint.pprint(doc_sr)
133+
assert doc_sr == dict(
134134
scan_id=post_resp["scan_id"],
135135
rescan_ids=[],
136136
#
@@ -157,17 +157,191 @@ async def _launch_scan(
157157
# misc
158158
skyscan_mq_client_timeout_wait_for_first_message=None,
159159
i3_event_id=post_resp["i3_event_id"],
160-
rest_address=doc["rest_address"], # see below
160+
rest_address=doc_sr["rest_address"], # see below
161161
scanner_server_env_from_user=post_scan_body["scanner_server_env"],
162162
)
163-
assert re.fullmatch(rf"{re.escape('http://localhost:')}\d+", doc["rest_address"])
163+
assert re.fullmatch(rf"{re.escape('http://localhost:')}\d+", doc_sr["rest_address"])
164164

165165
# query the SkyScanK8sJobs coll
166166
# -> since the scanner-server metadata is no longer stored in the manifest
167-
doc = await mongo_client["SkyDriver_DB"]["SkyScanK8sJobs"].find_one(
168-
{"scan_id": post_resp["scan_id"]}
167+
doc_k8s = await mongo_client["SkyDriver_DB"]["SkyScanK8sJobs"].find_one(
168+
{"scan_id": post_resp["scan_id"]}, {"_id": 0}
169169
)
170-
pprint.pprint(doc)
170+
pprint.pprint(doc_k8s)
171+
assert doc_k8s == {
172+
"scan_id": post_resp["scan_id"],
173+
"skyscan_k8s_job_dict": {
174+
"apiVersion": "batch/v1",
175+
"kind": "Job",
176+
"metadata": {
177+
"annotations": {"argocd.argoproj.io/sync-options": "Prune=false"},
178+
"labels": {"app.kubernetes.io/instance": None},
179+
"name": f"skyscan-{post_resp['scan_id']}",
180+
"namespace": None,
181+
},
182+
"spec": {
183+
"activeDeadlineSeconds": 86400,
184+
"backoffLimit": 0,
185+
"template": {
186+
"metadata": {"labels": {"app": "scanner-instance"}},
187+
"spec": {
188+
"containers": [
189+
{
190+
"args": scanner_server_args.split(),
191+
"command": [],
192+
"env": [
193+
{
194+
"name": "SKYSCAN_EWMS_JSON",
195+
"value": "/common-space/ewms.json",
196+
},
197+
{
198+
"name": "SKYSCAN_SKYDRIVER_ADDRESS",
199+
"value": doc_sr["rest_address"],
200+
},
201+
{
202+
"name": "SKYSCAN_SKYDRIVER_SCAN_ID",
203+
"value": post_resp["scan_id"],
204+
},
205+
{
206+
"name": "SKYSCAN_EWMS_PILOT_LOG",
207+
"value": "WARNING",
208+
},
209+
{
210+
"name": "SKYSCAN_MQ_CLIENT_LOG",
211+
"value": "WARNING",
212+
},
213+
{"name": "SKYSCAN_BROKER_AUTH", "value": ""},
214+
{"name": "SKYSCAN_SKYDRIVER_AUTH", "value": ""},
215+
]
216+
+ [ # add those from 'post_scan_body'
217+
{"name": k, "value": v}
218+
for k, v in post_scan_body[
219+
"scanner_server_env"
220+
].items()
221+
],
222+
"image": f"icecube/skymap_scanner:{os.environ['LATEST_TAG']}",
223+
"name": f'skyscan-server-{post_resp["scan_id"]}',
224+
"resources": {
225+
"limits": {"cpu": "1", "memory": "1024000000"},
226+
"requests": {
227+
"cpu": "1",
228+
"ephemeral-storage": "1M",
229+
"memory": "1024000000",
230+
},
231+
},
232+
"volumeMounts": [
233+
{
234+
"mountPath": "/common-space",
235+
"name": "common-space-volume",
236+
}
237+
],
238+
},
239+
{
240+
"args": [
241+
"/common-space/startup.json",
242+
"--wait-indefinitely",
243+
],
244+
"command": ["python", "-m", "s3_sidecar.post"],
245+
"env": [
246+
{"name": "S3_URL", "value": os.environ["S3_URL"]},
247+
{
248+
"name": "S3_ACCESS_KEY_ID",
249+
"valueFrom": {
250+
"secretKeyRef": {
251+
"key": os.environ["S3_ACCESS_KEY_ID"],
252+
"name": None,
253+
}
254+
},
255+
},
256+
{
257+
"name": "S3_SECRET_KEY",
258+
"valueFrom": {
259+
"secretKeyRef": {
260+
"key": os.environ["S3_SECRET_KEY"],
261+
"name": None,
262+
}
263+
},
264+
},
265+
{
266+
"name": "S3_BUCKET",
267+
"value": os.environ["S3_BUCKET"],
268+
},
269+
{
270+
"name": "S3_OBJECT_KEY",
271+
"value": f"{post_resp['scan_id']}-s3-object",
272+
},
273+
],
274+
"image": os.environ["THIS_IMAGE_WITH_TAG"],
275+
"name": f"sidecar-s3-{post_resp['scan_id']}",
276+
"resources": {
277+
"limits": {"cpu": "0.25", "memory": "256Mi"},
278+
"requests": {
279+
"cpu": "0.25",
280+
"ephemeral-storage": "1M",
281+
"memory": "256Mi",
282+
},
283+
},
284+
"restartPolicy": "OnFailure",
285+
"volumeMounts": [
286+
{
287+
"mountPath": "/common-space",
288+
"name": "common-space-volume",
289+
}
290+
],
291+
},
292+
],
293+
"initContainers": [
294+
{
295+
"args": [
296+
post_resp["scan_id"],
297+
"--json-out",
298+
"/common-space/ewms.json",
299+
],
300+
"command": ["python", "-m", "ewms_init_container"],
301+
"env": [
302+
{
303+
"name": "SKYSCAN_SKYDRIVER_ADDRESS",
304+
"value": doc_sr["rest_address"],
305+
},
306+
{"name": "SKYSCAN_SKYDRIVER_AUTH", "value": ""},
307+
{
308+
"name": "EWMS_ADDRESS",
309+
"value": os.environ["EWMS_ADDRESS"],
310+
},
311+
{
312+
"name": "EWMS_TOKEN_URL",
313+
"value": os.environ["EWMS_TOKEN_URL"],
314+
},
315+
{
316+
"name": "EWMS_CLIENT_ID",
317+
"value": os.environ["EWMS_CLIENT_ID"],
318+
},
319+
{
320+
"name": "EWMS_CLIENT_SECRET",
321+
"value": os.environ["EWMS_CLIENT_SECRET"],
322+
},
323+
{
324+
"name": "QUEUE_ALIAS_TOCLIENT",
325+
"value": "to-client-queue",
326+
},
327+
{
328+
"name": "QUEUE_ALIAS_FROMCLIENT",
329+
"value": "from-client-queue",
330+
},
331+
],
332+
"image": os.environ["THIS_IMAGE_WITH_TAG"],
333+
"name": f"init-ewms-{post_resp['scan_id']}",
334+
}
335+
],
336+
"restartPolicy": "Never",
337+
"serviceAccountName": None,
338+
"volumes": [{"emptyDir": {}, "name": "common-space-volume"}],
339+
},
340+
},
341+
"ttlSecondsAfterFinished": 600,
342+
},
343+
},
344+
}
171345
# TODO
172346

173347
assert 0

0 commit comments

Comments
 (0)