|
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +# Copyright 2022 The Machine Controller Authors. |
| 4 | +# |
| 5 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | +# you may not use this file except in compliance with the License. |
| 7 | +# You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, software |
| 12 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | +# See the License for the specific language governing permissions and |
| 15 | +# limitations under the License. |
| 16 | + |
| 17 | +### Contains commonly used functions for the other scripts. |
| 18 | + |
| 19 | +# Required for signal propagation to work so |
| 20 | +# the cleanup trap gets executed when a script |
| 21 | +# receives a SIGINT |
| 22 | +set -o monitor |
| 23 | + |
| 24 | +# appendTrap appends to existing traps, if any. It is needed because Bash replaces existing handlers |
| 25 | +# rather than appending: https://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal |
| 26 | +# Needing this func is a strong indicator that Bash is not the right language anymore. Also, this |
| 27 | +# basically needs unit tests. |
| 28 | +appendTrap() { |
| 29 | + command="$1" |
| 30 | + signal="$2" |
| 31 | + |
| 32 | + # Have existing traps, must append |
| 33 | + if [[ "$(trap -p | grep $signal)" ]]; then |
| 34 | + existingHandlerName="$(trap -p | grep $signal | awk '{print $3}' | tr -d "'")" |
| 35 | + |
| 36 | + newHandlerName="${command}_$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13)" |
| 37 | + # Need eval to get a random func name |
| 38 | + eval "$newHandlerName() { $command; $existingHandlerName; }" |
| 39 | + echodate "Appending $command as trap for $signal, existing command $existingHandlerName" |
| 40 | + trap $newHandlerName $signal |
| 41 | + # First trap |
| 42 | + else |
| 43 | + echodate "Using $command as trap for $signal" |
| 44 | + trap $command $signal |
| 45 | + fi |
| 46 | +} |
| 47 | + |
| 48 | +containerize() { |
| 49 | + local cmd="$1" |
| 50 | + local image="${CONTAINERIZE_IMAGE:-quay.io/kubermatic/util:2.0.0}" |
| 51 | + local gocache="${CONTAINERIZE_GOCACHE:-/tmp/.gocache}" |
| 52 | + local gomodcache="${CONTAINERIZE_GOMODCACHE:-/tmp/.gomodcache}" |
| 53 | + local skip="${NO_CONTAINERIZE:-}" |
| 54 | + |
| 55 | + # short-circuit containerize when in some cases it needs to be avoided |
| 56 | + [ -n "$skip" ] && return |
| 57 | + |
| 58 | + if ! [ -f /.dockerenv ]; then |
| 59 | + echodate "Running $cmd in a Docker container using $image..." |
| 60 | + mkdir -p "$gocache" |
| 61 | + mkdir -p "$gomodcache" |
| 62 | + |
| 63 | + exec docker run \ |
| 64 | + -v "$PWD":/go/src/k8c.io/kubermatic \ |
| 65 | + -v "$gocache":"$gocache" \ |
| 66 | + -v "$gomodcache":"$gomodcache" \ |
| 67 | + -w /go/src/k8c.io/kubermatic \ |
| 68 | + -e "GOCACHE=$gocache" \ |
| 69 | + -e "GOMODCACHE=$gomodcache" \ |
| 70 | + -u "$(id -u):$(id -g)" \ |
| 71 | + --entrypoint="$cmd" \ |
| 72 | + --rm \ |
| 73 | + -it \ |
| 74 | + $image $@ |
| 75 | + |
| 76 | + exit $? |
| 77 | + fi |
| 78 | +} |
| 79 | + |
| 80 | +echodate() { |
| 81 | + # do not use -Is to keep this compatible with macOS |
| 82 | + echo "[$(date +%Y-%m-%dT%H:%M:%S%:z)]" "$@" |
| 83 | +} |
| 84 | + |
| 85 | +# returns the current time as a number of milliseconds |
| 86 | +nowms() { |
| 87 | + echo $(($(date +%s%N) / 1000000)) |
| 88 | +} |
| 89 | + |
| 90 | +# returns the number of milliseconds elapsed since the given time |
| 91 | +elapsed() { |
| 92 | + echo $(($(nowms) - $1)) |
| 93 | +} |
| 94 | + |
| 95 | +# pushes a Prometheus metric to a pushgateway |
| 96 | +pushMetric() { |
| 97 | + local metric="$1" |
| 98 | + local value="$2" |
| 99 | + local labels="${3:-}" |
| 100 | + local kind="${4:-gauge}" |
| 101 | + local help="${5:-}" |
| 102 | + local pushgateway="${PUSHGATEWAY_URL:-}" |
| 103 | + local job="ci" |
| 104 | + local instance="${PROW_JOB_ID:-}" |
| 105 | + local prowjob="${JOB_NAME:-}" |
| 106 | + |
| 107 | + if [ -z "$pushgateway" ]; then |
| 108 | + return |
| 109 | + fi |
| 110 | + |
| 111 | + local payload="# TYPE $metric $kind" |
| 112 | + |
| 113 | + if [ -n "$help" ]; then |
| 114 | + payload="$payload\n# HELP $metric $help" |
| 115 | + fi |
| 116 | + |
| 117 | + if [ -n "$labels" ]; then |
| 118 | + labels=",$labels" |
| 119 | + fi |
| 120 | + |
| 121 | + payload="$payload\n$metric{prowjob=\"$prowjob\"$labels} $value\n" |
| 122 | + |
| 123 | + echo -e "$payload" | curl --data-binary @- -s "$pushgateway/metrics/job/$job/instance/$instance" |
| 124 | +} |
| 125 | + |
| 126 | +pushElapsed() { |
| 127 | + pushMetric "$1" $(elapsed $2) "${3:-}" "${4:-}" "${5:-}" |
| 128 | +} |
| 129 | + |
| 130 | +retry() { |
| 131 | + # Works only with bash but doesn't fail on other shells |
| 132 | + start_time=$(date +%s) |
| 133 | + set +e |
| 134 | + actual_retry $@ |
| 135 | + rc=$? |
| 136 | + set -e |
| 137 | + elapsed_time=$(($(date +%s) - $start_time)) |
| 138 | + write_junit "$rc" "$elapsed_time" |
| 139 | + return $rc |
| 140 | +} |
| 141 | + |
| 142 | +write_junit() { |
| 143 | + # Doesn't make any sense if we don't know a testname |
| 144 | + if [ -z "${TEST_NAME:-}" ]; then return; fi |
| 145 | + # Only run in CI |
| 146 | + if [ -z "${ARTIFACTS:-}" ]; then return; fi |
| 147 | + |
| 148 | + rc=$1 |
| 149 | + duration=${2:-0} |
| 150 | + errors=0 |
| 151 | + failure="" |
| 152 | + if [ "$rc" -ne 0 ]; then |
| 153 | + errors=1 |
| 154 | + failure='<failure type="Failure">Step failed</failure>' |
| 155 | + fi |
| 156 | + TEST_CLASS="${TEST_CLASS:-Kubermatic}" |
| 157 | + cat << EOF > ${ARTIFACTS}/junit.$(echo $TEST_NAME | sed 's/ /_/g' | tr '[:upper:]' '[:lower:]').xml |
| 158 | +<?xml version="1.0" ?> |
| 159 | +<testsuites> |
| 160 | + <testsuite errors="$errors" failures="$errors" name="$TEST_CLASS" tests="1"> |
| 161 | + <testcase classname="$TEST_CLASS" name="$TEST_NAME" time="$duration"> |
| 162 | + $failure |
| 163 | + </testcase> |
| 164 | + </testsuite> |
| 165 | +</testsuites> |
| 166 | +EOF |
| 167 | +} |
| 168 | + |
| 169 | +# We use an extra wrapping to write junit and have a timer |
| 170 | +actual_retry() { |
| 171 | + retries=$1 |
| 172 | + shift |
| 173 | + |
| 174 | + count=0 |
| 175 | + delay=1 |
| 176 | + until "$@"; do |
| 177 | + rc=$? |
| 178 | + count=$((count + 1)) |
| 179 | + if [ $count -lt "$retries" ]; then |
| 180 | + echo "Retry $count/$retries exited $rc, retrying in $delay seconds..." > /dev/stderr |
| 181 | + sleep $delay |
| 182 | + else |
| 183 | + echo "Retry $count/$retries exited $rc, no more retries left." > /dev/stderr |
| 184 | + return $rc |
| 185 | + fi |
| 186 | + delay=$((delay * 2)) |
| 187 | + done |
| 188 | + return 0 |
| 189 | +} |
| 190 | + |
| 191 | +start_docker_daemon_ci() { |
| 192 | + # DOCKER_REGISTRY_MIRROR_ADDR is injected via Prow preset; |
| 193 | + # start-docker.sh is part of the build image. |
| 194 | + DOCKER_REGISTRY_MIRROR="${DOCKER_REGISTRY_MIRROR_ADDR:-}" DOCKER_MTU=1400 start-docker.sh |
| 195 | +} |
| 196 | + |
| 197 | +start_docker_daemon() { |
| 198 | + if docker stats --no-stream > /dev/null 2>&1; then |
| 199 | + echodate "Not starting Docker again, it's already running." |
| 200 | + return |
| 201 | + fi |
| 202 | + |
| 203 | + # Start Docker daemon |
| 204 | + echodate "Starting Docker" |
| 205 | + dockerd > /tmp/docker.log 2>&1 & |
| 206 | + |
| 207 | + echodate "Started Docker successfully" |
| 208 | + appendTrap docker_logs EXIT |
| 209 | + |
| 210 | + # Wait for Docker to start |
| 211 | + echodate "Waiting for Docker" |
| 212 | + retry 5 docker stats --no-stream |
| 213 | + echodate "Docker became ready" |
| 214 | +} |
| 215 | + |
| 216 | +check_all_deployments_ready() { |
| 217 | + local namespace="$1" |
| 218 | + |
| 219 | + # check that Deployments have been created |
| 220 | + local deployments |
| 221 | + deployments=$(kubectl -n $namespace get deployments -o json) |
| 222 | + |
| 223 | + if [ $(echo "$deployments" | jq '.items | length') -eq 0 ]; then |
| 224 | + echodate "No Deployments created yet." |
| 225 | + return 1 |
| 226 | + fi |
| 227 | + |
| 228 | + # check that all Deployments are ready |
| 229 | + local unready |
| 230 | + unready=$(echo "$deployments" | jq -r '[.items[] | select(.spec.replicas > 0) | select (.status.availableReplicas < .spec.replicas) | .metadata.name] | @tsv') |
| 231 | + if [ -n "$unready" ]; then |
| 232 | + echodate "Not all Deployments have finished rolling out, namely: $unready" |
| 233 | + return 1 |
| 234 | + fi |
| 235 | + |
| 236 | + return 0 |
| 237 | +} |
0 commit comments