-
Notifications
You must be signed in to change notification settings - Fork 16
212 lines (198 loc) · 6.91 KB
/
test-tutor-aspects.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
name: Test Tutor Aspects
on:
pull_request:
branches: [ main ]
paths-ignore:
- .github/**
- .ci/**
- .gitignore
- CHANGELOG.md
- setup.cfg
- tutoraspects/__about__.py
env:
TUTOR_ROOT: ./.ci/
jobs:
local:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: setup python
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install python reqs
run: pip install -r requirements/dev.txt
- name: Install aspects
run: pip install -e .
- name: Tutor config save
run: tutor config save
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Tutor build openedx
run: tutor images build openedx aspects aspects-superset
- name: Tutor start
run: tutor local start -d
- name: Tutor init
run: tutor local do init
- name: Test alembic
run: |
tutor local do alembic -c "downgrade base"
tutor local do alembic -c "upgrade head"
- name: Test DBT
run: tutor local do dbt -c "test"
- name: Load test
run: tutor local do load-xapi-test-data --num_batches=5 --batch_size=5
- name: Import demo course
run: tutor local do importdemocourse
- name: Test commands
run: tutor local do dump-courses-to-clickhouse --options "--force"
- name: Tutor stop
run: tutor local stop
dev:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: setup python
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install python reqs
run: pip install -r requirements/dev.txt
- name: Install aspects
run: pip install -e .
- name: Tutor config save
run: tutor config save
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Tutor build openedx
run: tutor images build openedx-dev aspects aspects-superset
- name: Tutor start
run: tutor dev start -d
- name: Tutor init
run: tutor dev do init
- name: Test alembic
run: |
tutor dev do alembic -c "downgrade base"
tutor dev do alembic -c "upgrade head"
- name: Test DBT
run: tutor dev do dbt -c "test"
- name: Load test
run: tutor dev do load-xapi-test-data --num_batches=5 --batch_size=5
- name: Import demo course
run: tutor dev do importdemocourse
- name: Test commands
run: tutor dev do dump-courses-to-clickhouse --options "--force"
- name: Tutor stop
run: tutor dev stop
k8s:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: setup python
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Generate env
run: |
pip install -r requirements/dev.txt
pip install -e .
tutor config save
- name: Run Kubernetes tools
uses: stefanprodan/kube-tools@v1
with:
kubectl: 1.23.0
kustomize: 4.4.1
helmv3: 3.7.2
kubeconform: 0.4.13
command: |
kustomize build $TUTOR_ROOT/env | kubeconform -strict -ignore-missing-schemas -kubernetes-version 1.22.0
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@d5af243ce7bacb67384aa6c5b1fc5f169e30903e
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: false
# all of these default to true, but feel free to set to
# "false" if necessary for your workflow
android: true
dotnet: true
haskell: true
large-packages: false
swap-storage: true
- name: Tutor build openedx
run: tutor images build openedx aspects aspects-superset
- name: Create k8s Kind Cluster
uses: helm/kind-action@v1.8.0
- name: Mount docker image
run: |
kind get clusters
kind load docker-image $(tutor images printtag openedx) --name chart-testing
kind load docker-image $(tutor images printtag aspects) --name chart-testing
kind load docker-image $(tutor images printtag aspects-superset) --name chart-testing
- name: Setup namespace
run: |
kubectl config set-context --current --namespace=openedx
kubectl get pods
- name: Init k8s environment
run: |
tutor k8s start
tutor k8s do init
- name: Test alembic
run: |
tutor k8s do alembic -c "downgrade base"
tutor k8s do alembic -c "upgrade head"
- name: Test DBT
run: tutor k8s do dbt -c "test"
- name: Load test
run: tutor k8s do load-xapi-test-data --num_batches=5 --batch_size=5
- name: Import demo course
run: tutor k8s do importdemocourse
- name: Test commands
run: tutor k8s do dump-courses-to-clickhouse --options "--force"
- name: Check failure logs
if: failure()
run: |
# Use an array to store pod names
pod_list=($(kubectl get pods -o jsonpath='{.items[*].metadata.name}'))
failed_jobs=""
# Loop through each pod and check for failure
for pod_name in "${pod_list[@]}"; do
# Get the pod phase/status
pod_phase=$(kubectl get pod "$pod_name" -o jsonpath='{.status.phase}')
if [[ "$pod_phase" != "Running" && "$pod_phase" != "Succeeded" ]]; then
# Job is not in Running or Succeeded state, consider it as failed
failed_jobs="$failed_jobs $pod_name"
# Print the logs for the failing pod
echo "hi $pod_name"
fi
done
# Check if any jobs failed
if [ -n "$failed_jobs" ]; then
echo "The following jobs failed: $failed_jobs"
else
echo "All jobs succeeded."
fi
- name: Check all the logs
if: '!cancelled()'
run: |
job_list=($(kubectl get jobs | grep job | awk '{print $1}'))
# Function to print logs for a pod
print_pod_logs() {
local pod_name="$1"
echo "Logs for pod: $pod_name"
kubectl logs "$pod_name"
echo "------------------------"
}
# Loop through each job-related pod and print logs
for job_name in "${job_list[@]}"; do
# Get the pods related to the current job
job_pod_list=($(kubectl get pods | grep -E "${job_name}-.*" | awk '{print $1}'))
for pod_name in "${job_pod_list[@]}"; do
print_pod_logs "$pod_name"
done
done