-
Notifications
You must be signed in to change notification settings - Fork 0
283 lines (239 loc) · 10.3 KB
/
dbt_ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
name: CI test new and modified models
concurrency: ci-${{ github.ref }}
on:
pull_request:
env:
DEBUG: true
DBT_PROFILE_NAME: ${{ vars.DBT_PROFILE_NAME }}
DBT_SNOWFLAKE_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_ACCOUNT }}
DBT_SNOWFLAKE_USER: ${{ vars.DBT_SNOWFLAKE_USER }}
DBT_SNOWFLAKE_ROLE: ${{ vars.DBT_SNOWFLAKE_ROLE }}
DBT_SNOWFLAKE_PASSWORD: ${{ secrets.DBT_SNOWFLAKE_PASSWORD }}
DBT_SNOWFLAKE_DATABASE: ${{ vars.DBT_SNOWFLAKE_DATABASE }}
DBT_SNOWFLAKE_WAREHOUSE: ${{ vars.DBT_SNOWFLAKE_WAREHOUSE }}
DBT_SNOWFLAKE_THREADS: ${{ vars.DBT_SNOWFLAKE_THREADS }}
PR_NUMBER: ${{ github.event.pull_request.number }}
BRANCH_NAME: ${{ github.head_ref }}
DBT_PROJECT_FOLDER: ${{ vars.DBT_PROJECT_FOLDER }}
jobs:
initial_compile:
# dependabot does not have access to Github Action secrets
if: github.actor != 'dependabot[bot]'
# For creating the initial manifest.json based on the branch before any commits are made
name: Create initial manifest.json
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Switch to base commit
run: |
git checkout ${{ github.event.pull_request.base.sha }}
- name: Create profiles.yml
run: |
mkdir -p /home/$(whoami)/.dbt/
echo "${DBT_PROFILE_NAME}:" > /home/$(whoami)/.dbt/profiles.yml
echo " target: development" >> /home/$(whoami)/.dbt/profiles.yml
echo " outputs:" >> /home/$(whoami)/.dbt/profiles.yml
echo " development:" >> /home/$(whoami)/.dbt/profiles.yml
echo " type: snowflake" >> /home/$(whoami)/.dbt/profiles.yml
echo " account: ${DBT_SNOWFLAKE_ACCOUNT}" >> /home/$(whoami)/.dbt/profiles.yml
echo " user: ${DBT_SNOWFLAKE_USER}" >> /home/$(whoami)/.dbt/profiles.yml
echo " role: ${DBT_SNOWFLAKE_ROLE}" >> /home/$(whoami)/.dbt/profiles.yml
echo " password: ${DBT_SNOWFLAKE_PASSWORD}" >> /home/$(whoami)/.dbt/profiles.yml
echo " database: ${DBT_SNOWFLAKE_DATABASE}" >> /home/$(whoami)/.dbt/profiles.yml
echo " warehouse: ${DBT_SNOWFLAKE_WAREHOUSE}" >> /home/$(whoami)/.dbt/profiles.yml
echo " schema: pr" >> /home/$(whoami)/.dbt/profiles.yml
echo " threads: ${DBT_SNOWFLAKE_THREADS}" >> /home/$(whoami)/.dbt/profiles.yml
echo " client_session_keep_alive: false" >> /home/$(whoami)/.dbt/profiles.yml
- name: Print profiles.yml
if: ${{ env.DEBUG == 'true' }}
run: |
echo "----------------------------------------"
echo "Debug mode: Show content of profiles.yml"
echo "Note that secrets will be masked"
echo "----------------------------------------"
cat /home/$(whoami)/.dbt/profiles.yml
- name: Read Python version from .python-version
run: echo "python_version=$(cat .python-version)" >> "$GITHUB_ENV"
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.python_version }}
- name: Cache pip dependencies
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dbt
run: |
python -m pip install --upgrade pip
grep dbt-snowflake requirements.txt | xargs -n 1 pip install
- name: Run dbt debug
if: ${{ env.DEBUG == 'true' }}
run: |
cd $DBT_PROJECT_FOLDER
echo "----------------------------------------"
echo "Debug mode: Execute dbt debug"
echo "----------------------------------------"
dbt debug
- name: Run dbt deps
run: |
cd $DBT_PROJECT_FOLDER
dbt deps
- name: Run dbt compile
run: |
cd $DBT_PROJECT_FOLDER
dbt compile
- name: Print manifest.json
if: ${{ env.DEBUG == 'true' }}
run: |
cd $DBT_PROJECT_FOLDER
cat target/manifest.json
- name: Upload manifest.json
uses: actions/upload-artifact@v3
with:
name: manifest
path: ${{ vars.DBT_PROJECT_FOLDER }}/target/manifest.json
dbt_build:
# dependabot does not have access to Github Action secrets
if: github.actor != 'dependabot[bot]'
name: Test new and updated dbt models
needs: initial_compile
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Create profiles.yml
run: |
BRANCH_NAME=$(echo "${BRANCH_NAME}" | tr '-' '_')
mkdir -p /home/$(whoami)/.dbt/
echo "${DBT_PROFILE_NAME}:" > /home/$(whoami)/.dbt/profiles.yml
echo " target: development" >> /home/$(whoami)/.dbt/profiles.yml
echo " outputs:" >> /home/$(whoami)/.dbt/profiles.yml
echo " development:" >> /home/$(whoami)/.dbt/profiles.yml
echo " type: snowflake" >> /home/$(whoami)/.dbt/profiles.yml
echo " account: ${DBT_SNOWFLAKE_ACCOUNT}" >> /home/$(whoami)/.dbt/profiles.yml
echo " user: ${DBT_SNOWFLAKE_USER}" >> /home/$(whoami)/.dbt/profiles.yml
echo " role: ${DBT_SNOWFLAKE_ROLE}" >> /home/$(whoami)/.dbt/profiles.yml
echo " password: ${DBT_SNOWFLAKE_PASSWORD}" >> /home/$(whoami)/.dbt/profiles.yml
echo " database: ${DBT_SNOWFLAKE_DATABASE}" >> /home/$(whoami)/.dbt/profiles.yml
echo " warehouse: ${DBT_SNOWFLAKE_WAREHOUSE}" >> /home/$(whoami)/.dbt/profiles.yml
echo " schema: pr_${BRANCH_NAME##*/}_${PR_NUMBER}" >> /home/$(whoami)/.dbt/profiles.yml
echo " threads: ${DBT_SNOWFLAKE_THREADS}" >> /home/$(whoami)/.dbt/profiles.yml
echo " client_session_keep_alive: false" >> /home/$(whoami)/.dbt/profiles.yml
- name: Print profiles.yml
if: ${{ env.DEBUG == 'true' }}
run: |
echo "----------------------------------------"
echo "Debug mode: Show content of profiles.yml"
echo "Note that secrets will be masked"
echo "----------------------------------------"
cat /home/$(whoami)/.dbt/profiles.yml
- name: Read Python version from .python-version
run: echo "python_version=$(cat .python-version)" >> "$GITHUB_ENV"
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.python_version }}
- name: Cache pip dependencies
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dbt
run: |
python -m pip install --upgrade pip
grep dbt-snowflake requirements.txt | xargs -n 1 pip install
- name: Run dbt debug
if: ${{ env.DEBUG == 'true' }}
run: |
cd $DBT_PROJECT_FOLDER
echo "----------------------------------------"
echo "Debug mode: Execute dbt debug"
echo "----------------------------------------"
dbt debug
- name: Run dbt deps
run: |
cd $DBT_PROJECT_FOLDER
dbt deps
- name: Download manifest.json
uses: actions/download-artifact@v3
with:
name: manifest
path: ${{ vars.DBT_PROJECT_FOLDER }}/
- name: Print initial manifest.json
if: ${{ env.DEBUG == 'true' }}
run: |
cd $DBT_PROJECT_FOLDER
cat manifest.json
- name: Run dbt compile
if: ${{ env.DEBUG == 'true' }}
run: |
cd $DBT_PROJECT_FOLDER
echo "----------------------------------------"
echo "Debug mode: Execute dbt compile"
echo "----------------------------------------"
dbt compile
- name: Run dbt seed, run, and test
run: |
cd $DBT_PROJECT_FOLDER
echo ""
echo ""
echo "---------- dbt seed ----------"
dbt seed
echo ""
echo ""
echo "---------- dbt run ----------"
dbt run --select +state:modified --state ./
echo ""
echo ""
echo "---------- dbt test ----------"
dbt test --select state:modified --state ./ | tee dbt_output.log
exit ${PIPESTATUS[0]}
- name: Print sql from failing tests
if: ${{ failure() }}
run: |
cd $DBT_PROJECT_FOLDER
# Define the log file
log_file="dbt_output.log"
# Get the number of failed tests
num_failed_tests=$(grep -o "ERROR=[0-9]*" "$log_file" | awk -F= '{print $2}')
# If there are no failed tests, exit the script
if [ "$num_failed_tests" -eq 0 ]; then
echo "No failed tests found."
exit 0
fi
# Loop through each failed test and extract the required information
for i in $(seq 1 "$num_failed_tests"); do
header=$(grep -m "$i" "Failure in test" "$log_file" | tail -n 1 | cut -c 15-)
description=$(grep -A 1 -m "$i" "Failure in test" "$log_file" | tail -n 1 | cut -c 15-)
code_path=$(grep -m "$i" "compiled Code at" "$log_file" | tail -n 1 | awk '{print $5}')
# Print the extracted information
echo "$header"
echo "$description"
# Print the content of the code file
if [ -f "$code_path" ]; then
grep -vE '^$' "$code_path"
else
echo "Code file not found: $code_path"
fi
# Add a separator between failed tests, except for the last one
if [ "$i" -ne "$num_failed_tests" ]; then
echo ""
echo "----------------------------------------"
echo ""
fi
done
if [ "$DEBUG" = "true" ]; then
echo ""
echo "----------------------------------------"
echo "Debug mode: Printing the entire dbt_output.log content:"
echo "----------------------------------------"
cat "$log_file"
fi