-
Notifications
You must be signed in to change notification settings - Fork 1.2k
288 lines (280 loc) · 11.3 KB
/
backend.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: Backend
on:
push:
workflow_dispatch:
pull_request:
branches:
- dev
- 1.2
paths-ignore:
- 'docs/**'
- '**/*.md'
- '**/*.sql'
concurrency:
group: backend-${{ github.event.pull_request.number || github.ref }}
jobs:
check:
name: Check Code
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Check Style
run: |
./mvnw -T 4C -B --no-snapshot-updates clean spotless:check -P flink-all
build_npm:
name: Build_NPM
runs-on: ubuntu-latest
timeout-minutes: 30
needs: check
env:
MAVEN_OPTS: -Xmx2G -Xms2G
steps:
- uses: actions/checkout@v3
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
frontend:
- 'dinky-web/**'
- uses: actions/setup-node@v3
if: steps.filter.outputs.frontend == 'true'
with:
node-version: 16
- name: Get npm cache directory
id: npm-cache-dir
if: steps.filter.outputs.frontend == 'true'
run: |
echo "::set-output name=dir::$(npm config get cache)"
- uses: actions/cache@v3
id: npm-cache # use this to check for `cache-hit` ==> if: steps.npm-cache.outputs.cache-hit != 'true'
if: steps.filter.outputs.frontend == 'true'
with:
path: |
${{ steps.npm-cache-dir.outputs.dir }}
dinky-web/dist
key: ${{ runner.os }}-node-${{ hashFiles('dinky-web/**/package.json') }}
restore-keys: |
${{ runner.os }}-node-
- name: Install Dependencies
if: steps.filter.outputs.frontend == 'true'
run: cd dinky-web && npm install --no-audit --progress=false --legacy-peer-deps
- name: Npm Web Build
if: steps.filter.outputs.frontend == 'true'
run: cd dinky-web && npm run build
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: dinky-web
path: ./dinky-web/dist
build_release:
name: Build Release
runs-on: ubuntu-latest
needs: check
strategy:
fail-fast: true
matrix:
jdk: [ 8, 11 ]
flink: [ '1.14', '1.15', '1.16', '1.17', '1.18', '1.19', '1.20' ]
timeout-minutes: 30
env:
MAVEN_OPTS: -Xmx2G -Xms2G
steps:
- uses: actions/checkout@v3
# maven编译
- name: Set up JDK ${{ matrix.jdk }}
uses: actions/setup-java@v2
with:
java-version: ${{ matrix.jdk }}
distribution: 'adopt'
- name: Cache local Maven repository
uses: actions/cache@v3
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/flink
key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Cache local Flink repository
uses: actions/cache@v3
with:
path: ~/.m2/repository/org/apache/flink
key: ${{ runner.os }}-${{ matrix.flink }}-maven-${{ hashFiles('pom.xml') }}
restore-keys: |
${{ runner.os }}-${{ matrix.flink }}-maven-
- name: Build and Package
run: |
./mvnw -B clean install \
-Dmaven.test.skip=false \
-Dspotless.check.skip=true \
-Denforcer.skip=false \
-Dmaven.javadoc.skip=true \
-P prod,flink-single-version,flink-${{ matrix.flink }},maven-central \
--no-snapshot-updates
# 检查打包的大小
- name: Check package size
run: |
./check_package_size.sh
- name: Upload artifact
uses: actions/upload-artifact@v4
if: ${{ matrix.jdk == 8 }}
with:
name: dinky-realease-${{ matrix.flink }}
path: ./build/dinky-release*.tar.gz
# - name: Run Unit tests
# run: |
# ./mvnw -T 2C -B clean verify -Dmaven.test.skip=false -Dgpg.skip=true --no-snapshot-updates
run_e2e:
name: Run E2E
needs: build_release
strategy:
fail-fast: true
matrix:
flink: [ '1.15', '1.16', '1.17', '1.18', '1.19', '1.20' ]
runs-on: ubuntu-latest
services:
registry:
image: registry:2
ports:
- 5000:5000
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Init Docker Network
run: |
docker network create -d bridge --subnet 172.28.0.0/16 --gateway 172.28.0.1 dinky_net
- name: Download artifact
uses: actions/download-artifact@v4
with:
name: dinky-realease-${{ matrix.flink }}
path: ./build
# 设置 QEMU, 后面 docker buildx 依赖此.
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
# 设置 Docker buildx, 方便构建 Multi platform 镜像
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver-opts: network=host
- name: Build Dinky Image
uses: docker/build-push-action@v5
with:
context: .
file: ./e2e_test/docker-compose-env/Dockerfile
# 是否 docker push
push: true
build-args: |
FLINK_VERSION=${{ matrix.flink }}
tags: |
localhost:5000/dinky/dinky-test:flink
- name: Build Flink Image
uses: docker/build-push-action@v5
with:
context: .
file: ./e2e_test/docker-compose-env/FlinkDockerfile
# 是否 docker push
push: true
build-args: |
FLINK_VERSION=${{ matrix.flink }}
tags: |
localhost:5000/dinky/flink:flink
- name: Init Env Jar
run: |
mkdir O e2e_test/docker-compose-env/dinky/jars
wget -O e2e_test/docker-compose-env/dinky/mysql-connector-java-8.0.30.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.30/mysql-connector-java-8.0.30.jar &&
wget -O e2e_test/docker-compose-env/flink/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar https://repo1.maven.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar &&
wget -O e2e_test/docker-compose-env/dinky/javax.ws.rs-api-2.1.1.jar https://repo1.maven.org/maven2/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar
wget -O e2e_test/docker-compose-env/dinky/jars/flink-doris-connector.jar https://repo1.maven.org/maven2/org/apache/doris/flink-doris-connector-${{ matrix.flink }}/24.0.1/flink-doris-connector-${{ matrix.flink }}-24.0.1.jar
wget -O e2e_test/docker-compose-env/dinky/jars/flink-sql-connector-mysql-cdc.jar https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-mysql-cdc/3.2.0/flink-sql-connector-mysql-cdc-3.2.0.jar
cp e2e_test/docker-compose-env/dinky/mysql-connector-java-8.0.30.jar e2e_test/docker-compose-env/dinky/jars/mysql-connector-java.jar
- name: Init Run Docker MySQL
uses: hoverkraft-tech/compose-action@v2.0.2
with:
compose-file: ./e2e_test/docker-compose-env/mysql/docker-compose.yml
# - name: Init System Env And Clear Docker Build Cache
# run: |
# echo y | docker builder prune
# df -h
# ulimit -a
# sudo swapoff -a
# sudo sysctl -w vm.max_map_count=2000000
# - name: Init Run Docker Doris
# uses: hoverkraft-tech/compose-action@v2.0.2
# with:
# compose-file: ./e2e_test/docker-compose-env/doris/docker-compose.yml
- name: Init Run Docker Dinky
uses: hoverkraft-tech/compose-action@v2.0.2
with:
compose-file: ./e2e_test/docker-compose-env/dinky/docker-compose.yml
- name: Init Run Docker Hadoop
uses: hoverkraft-tech/compose-action@v2.0.2
with:
compose-file: ./e2e_test/docker-compose-env/hadoop/docker-compose.yml
- name: Init Run Docker Flink
uses: hoverkraft-tech/compose-action@v2.0.2
with:
compose-file: ./e2e_test/docker-compose-env/flink/docker-compose.yml
# k8s env
- name: Init k3s
uses: nolar/setup-k3d-k3s@v1
with:
version: latest
k3d-args: -s 1 --network dinky_net --api-port 172.28.0.1:6550
- name: Get k3s kube config
run: k3d kubeconfig get --all && mkdir ./kube && k3d kubeconfig get --all > ./kube/k3s.yaml && sed -i 's/0.0.0.0/172.28.0.1/g' ./kube/k3s.yaml
- name: Init k8s RBAC and namespace
run: |
kubectl create namespace dinky
kubectl create serviceaccount dinky -n dinky
kubectl create clusterrolebinding flink-role-binding-dinky --clusterrole=edit --serviceaccount=dinky:dinky
- name: Init k3s main images
run: |
docker exec k3d-k3s-default-server-0 crictl pull library/busybox:latest
docker exec k3d-k3s-default-server-0 crictl pull flink:${{ matrix.flink }}-scala_2.12-java8
docker pull localhost:5000/dinky/flink:flink
docker tag localhost:5000/dinky/flink:flink dinky/flink:flink
docker save -o flink.tar dinky/flink:flink
k3d images import ./flink.tar
rm -rf ./flink.tar
- name: Test k3s host
run: |
curl -k https://172.28.0.1:6550
- name: Cp Flink Jar Deps
run: |
docker cp dinky:/opt/dinky/ ./dinky-release
mv ./dinky-release/jar/dinky-app*.jar e2e_test/docker-compose-env/dinky/dinky-app.jar
- name: Run python http server
run: |
mkdir -p logs
ls e2e_test/docker-compose-env/dinky/
nohup python -m http.server -d e2e_test/docker-compose-env/dinky/ 9001 > ./logs/python_http.log &
- name: Run Docker Python Script
run: |
docker run -v ./e2e_test/tools:/app -w /app -v ./kube:/kube -v ./e2e_test/docker-compose-env/dinky:/dinky/jar -v ./dinky-release/extends/flink${{ matrix.flink }}:/opt/flink/lib -v ./e2e_test/docker-compose-env/dinky/mysql-connector-java-8.0.30.jar:/opt/flink/lib/mysql-connector-java-8.0.30.jar --net dinky_net --rm --entrypoint /bin/bash python:3.9 -c 'pip install -r requirements.txt && python main.py dinky:8888 ${{ matrix.flink }}'
- name: Get k8s pods info and logs
if: ${{ always() }}
run: |
chmod -R 755 ./e2e_test/view_k8s_all_pod_logs.sh
./e2e_test/view_k8s_all_pod_logs.sh dinky
- name: Get Python HttpServer log
if: ${{ always() }}
run: |
cat ./logs/python_http.log