-
Notifications
You must be signed in to change notification settings - Fork 0
/
.gitlab-ci.yml
166 lines (154 loc) · 4.81 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
stages:
- run_tests
# - offline_eval
- build_main_recommender_image
- build_nightly_image
- build_parser_image
- deploy_parser
- deploy_recommender
- start_backup
- restore_from_backup
run_tests:
only:
- /^release-tests.*$/
image: "python:3.11-bookworm"
stage: run_tests
before_script:
- python --version
- rm /bin/sh && ln -s /bin/bash /bin/sh
- apt-get update
&& apt-get -y install gcc gnupg2 && apt-get -y install curl
&& curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
&& curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list
- apt-get update
&& ACCEPT_EULA=Y apt-get -y install msodbcsql18
&& ACCEPT_EULA=Y apt-get -y install mssql-tools
- apt-get update
&& apt-get -y install unixodbc-dev
&& pip install --no-cache-dir --upgrade pip
&& pip install --no-cache-dir -r ./tests/test-requirements.txt
&& pip install --no-cache-dir -r ./inference/requirements.txt
- apt-get update
- pip install -e ./db_utils
script:
- python -m pip install coverage # Runs unit tests & creates a coverage report
- python -m coverage run -m unittest discover -s ./tests
- python -m coverage report
- OFFLINE_EVAL_PASS = $(python3 ./inference/offline_eval.py)
- echo $OFFLINE_EVAL_PASS
build_main_recommender_image:
only:
- /^release-recommender-main-.*$/
stage: build_main_recommender_image
image: docker:20.10.16
services:
- name: docker:20.10.16-dind
alias: thedockerhost
variables:
DOCKER_HOST: tcp://thedockerhost:2375/
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
before_script:
- docker login -u $REGISTRY_USER -p $REGISTRY_PASS
script:
- cd ./inference
- docker build -f dockerfile -t blo7/comp585_team-5_recommender:latest .
- docker push blo7/comp585_team-5_recommender:latest
tags:
- docker-build
build_nightly_image:
only:
- /^release-recommender-nightly-.*$/
stage: build_nightly_image
image: docker:20.10.16
services:
- name: docker:20.10.16-dind
alias: thedockerhost
variables:
DOCKER_HOST: tcp://thedockerhost:2375/
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
before_script:
- docker login -u $REGISTRY_USER -p $REGISTRY_PASS
script:
- cd ./inference
- docker build -f dockerfile-nightly -t blo7/comp585_team-5_recommender_nightly:$CI_COMMIT_TAG .
- docker push blo7/comp585_team-5_recommender_nightly:$CI_COMMIT_TAG
tags:
- docker-build
build_parser_image:
only:
- /^release-parser.*$/
stage: build_parser_image
image: docker:20.10.16
services:
- name: docker:20.10.16-dind
alias: thedockerhost
variables:
DOCKER_HOST: tcp://thedockerhost:2375/
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
before_script:
- docker login -u $REGISTRY_USER -p $REGISTRY_PASS
script:
- cd ./data_parsing
- docker build -t blo7/comp585_team-5_parser:latest .
- docker push blo7/comp585_team-5_parser:latest
tags:
- docker-build
deploy_parser:
only:
- /^release-parser.*$/
stage: deploy_parser
image: docker:20.10.16
before_script:
- apk update
- apk add sshpass
script:
- sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=no blo7@fall2023-comp585-5.cs.mcgill.ca "
docker login -u $REGISTRY_USER -p $REGISTRY_PASS &&
docker image pull blo7/comp585_team-5_parser:latest &&
docker run -d --name python_kafka_reader --restart always --network monitoring-network blo7/comp585_team-5_parser:latest"
tags:
- docker-run
deploy_recommender:
only:
- /^release-recommender.*$/
stage: deploy_recommender
image: docker:20.10.16
before_script:
- apk update
- apk add sshpass
script:
- sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=no blo7@fall2023-comp585-5.cs.mcgill.ca "
cd /home/blo7/localcode/team-5/load_balancer/;
python3 deploy_inference_container.py --nightly $CI_COMMIT_TAG"
tags:
- docker-run
start_backup:
rules:
- if: $CI_PIPELINE_SOURCE == "scheduled_event"
stage: start_backup
image: docker:20.10.16
before_script:
- apk update
- apk add sshpass
script:
- sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=no blo7@fall2023-comp585-5.cs.mcgill.ca "
cd ./db_backups &&
docker exec -t database pg_dumpall -c -U postgres > backup_`date +%d-%m-%Y"_"%H_%M_%S`.sql"
tags:
- docker-run
restore_from_backup:
rules:
- if: $CI_RESTORE_FROM_BACKUP != null
stage: restore_from_backup
image: docker:20.10.16
before_script:
- apk update
- apk add sshpass
script:
- sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=no blo7@fall2023-comp585-5.cs.mcgill.ca "
cd ./db_backups &&
cat $CI_RESTORE_FROM_BACKUP | docker exec -i database psql -U postgres "
timeout: 3 minutes