-
Notifications
You must be signed in to change notification settings - Fork 0
252 lines (226 loc) · 8.21 KB
/
kind-e2e.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
#
# Copyright 2021 The Dapr Authors
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: E2E tests on KinD
on:
# Manual trigger
workflow_dispatch:
# On pull requests
pull_request:
branches:
- master
- 'release-*'
- 'feature/*'
defaults:
run:
shell: bash
jobs:
# This workflow runs our e2e tests in a local KinD cluster. Since it
# does not required a paid cluster and special credentials, it does
# not require /ok-to-test. That allows contributors to run E2E tests
# in their Draft PRs without explicit approval from Dapr
# maintainers.
#
# One other benefit of a fully-localized workflow is that tests are
# running on a fresh cluster everytime, thus being decoupled from
# shared resource issues.
#
# However, KinD currently does not support Windows nodes so this is
# not intended to be a complete replacement for our AKS-based E2E
# workflows.
e2e:
name: e2e
runs-on: ubuntu-latest
env:
REGISTRY_PORT: 5000
REGISTRY_NAME: kind-registry
DAPR_REGISTRY: localhost:5000/dapr
DAPR_TAG: dev
DAPR_NAMESPACE: dapr-tests
# Useful for upgrade/downgrade/compatibility tests
# TODO: Make this auto-populated based on GitHub's releases.
DAPR_TEST_N_MINUS_1_IMAGE: "ghcr.io/dapr/daprd:1.12.0"
DAPR_TEST_N_MINUS_2_IMAGE: "ghcr.io/dapr/daprd:1.11.4"
# Container registry where to cache e2e test images
DAPR_CACHE_REGISTRY: "dapre2eacr.azurecr.io"
PULL_POLICY: IfNotPresent
strategy:
fail-fast: false # Keep running if one leg fails.
matrix:
k8s-version:
- v1.23.13
- v1.24.7
- v1.25.3
mode:
- ha
- non-ha
# Map between K8s and KinD versions.
# This is attempting to make it a bit clearer what's being tested.
# See: https://github.com/kubernetes-sigs/kind/releases/tag/v0.11.1
include:
- k8s-version: v1.23.13
kind-version: v0.17.0
kind-image-sha: sha256:ef453bb7c79f0e3caba88d2067d4196f427794086a7d0df8df4f019d5e336b61
dapr-test-config-store: "postgres"
- k8s-version: v1.24.7
kind-version: v0.17.0
kind-image-sha: sha256:577c630ce8e509131eab1aea12c022190978dd2f745aac5eb1fe65c0807eb315
dapr-test-config-store: "redis"
- k8s-version: v1.25.3
kind-version: v0.17.0
kind-image-sha: sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1
dapr-test-config-store: "redis"
exclude:
- k8s-version: v1.23.13
mode: non-ha
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Set up Go
id: setup-go
uses: actions/setup-go@v3
with:
go-version-file: 'go.mod'
- name: Configure KinD
# Generate a KinD configuration file that uses:
# (a) a couple of worker nodes: this is needed to run both
# ZooKeeper + Kakfa, and
# (b) a local registry: Due to `kind load` performance
# https://github.com/kubernetes-sigs/kind/issues/1165, using
# a local repository speeds up the image pushes into KinD
# significantly.
run: |
cat > kind.yaml <<EOF
apiVersion: kind.x-k8s.io/v1alpha4
kind: Cluster
nodes:
- role: control-plane
image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
- role: worker
image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
- role: worker
image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
- role: worker
image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:$REGISTRY_PORT"]
endpoint = ["http://$REGISTRY_NAME:$REGISTRY_PORT"]
EOF
# Log the generated kind.yaml for easy reference.
cat kind.yaml
# Set log target directories
echo "DAPR_CONTAINER_LOG_PATH=$GITHUB_WORKSPACE/container_logs/${{ matrix.k8s-version }}_${{ matrix.mode }}" >> $GITHUB_ENV
echo "DAPR_TEST_LOG_PATH=$GITHUB_WORKSPACE/test_logs/${{ matrix.k8s-version }}_${{ matrix.mode }}" >> $GITHUB_ENV
- name: Create KinD Cluster
uses: helm/kind-action@v1.5.0
with:
config: kind.yaml
cluster_name: kind
version: ${{ matrix.kind-version }}
- name: Get KinD info
run: |
kubectl cluster-info --context kind-kind
NODE_IP=$(kubectl get nodes \
-lkubernetes.io/hostname!=kind-control-plane \
-ojsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')
echo "MINIKUBE_NODE_IP=$NODE_IP" >> $GITHUB_ENV
- name: Setup test output
shell: bash
run: |
export TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report
echo "TEST_OUTPUT_FILE_PREFIX=$TEST_OUTPUT_FILE_PREFIX" >> $GITHUB_ENV
- name: Setup local registry
run: |
# Run a registry.
docker run -d --restart=always \
-p $REGISTRY_PORT:$REGISTRY_PORT --name $REGISTRY_NAME registry:2
# Connect the registry to the KinD network.
docker network connect "kind" $REGISTRY_NAME
- name: Setup Helm
uses: azure/setup-helm@v1
with:
version: v3.3.4
- name: Build and push Dapr
run: |
make build-linux
make docker-build
make docker-push
- name: Build and push test apps
run: |
make build-push-e2e-app-all
- name: Setup Dapr
run: |
make setup-helm-init
make create-test-namespace
export ADDITIONAL_HELM_SET="dapr_operator.logLevel=debug,dapr_operator.watchInterval=20s"
if [[ "${{ matrix.mode }}" == "ha" ]]; then
export HA_MODE=true
else
export HA_MODE=false
fi
make docker-deploy-k8s
- name: Setup Redis
run: |
make setup-test-env-redis
- name: Setup Kafka
run: |
make setup-test-env-kafka
- name: Setup Zipkin
run: |
make setup-test-env-zipkin
- name: Setup postgres
run: |
make setup-test-env-postgres
- name: Setup test components
run: |
make setup-test-components
env:
DAPR_TEST_CONFIG_STORE: ${{ matrix.dapr-test-config-store }}
- name: Free up some diskspace
run: |
docker image prune -a -f
- name: Run tests
run: |
make test-e2e-all
env:
DAPR_TEST_CONFIG_STORE: ${{ matrix.dapr-test-config-store }}
- name: Save control plane logs
if: always()
run: |
make save-dapr-control-plane-k8s-logs
# Container log files can be bigger than the maximum file size allowed by GitHub
- name: Compress logs
if: always()
run: |
gzip --fast -r ${{ env.DAPR_CONTAINER_LOG_PATH }}
gzip --fast -r ${{ env.DAPR_TEST_LOG_PATH }}
shell: bash
- name: Upload container logs
if: always()
uses: actions/upload-artifact@master
with:
name: ${{ matrix.k8s-version }}_${{ matrix.mode}}_container_logs
path: ${{ env.DAPR_CONTAINER_LOG_PATH }}
- name: Upload test logs
if: always()
uses: actions/upload-artifact@master
with:
name: ${{ matrix.k8s-version }}_${{ matrix.mode}}_test_logs
path: ${{ env.DAPR_TEST_LOG_PATH }}
- name: Upload test results
if: always()
uses: actions/upload-artifact@master
with:
#TODO: .json suffix can be removed from artifact name after test analytics scripts are updated
name: ${{ matrix.k8s-version }}_${{ matrix.mode }}_test_e2e.json
path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_e2e.*