From 917fb180129940651e1560a389e7311dad5dc347 Mon Sep 17 00:00:00 2001 From: jwaldrip Date: Fri, 7 Apr 2017 15:45:55 -0600 Subject: [PATCH 1/8] set labels on boot, and instrument retries --- sidecar.sh | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/sidecar.sh b/sidecar.sh index 82eb214..378288a 100644 --- a/sidecar.sh +++ b/sidecar.sh @@ -5,7 +5,7 @@ set -e # Set vars ip=${POD_IP-`hostname -i`} # ip address of pod redis_port=${NODE_PORT_NUMBER-6379} # redis port -sentinel_port=${SENTINEL_PORT_NUMBER-26379} # sentinel port +sentinel_port=${SENTINEL_PORT_NUMBER-26379} # sentinel port group_name="$POD_NAMESPACE-$(hostname | sed 's/-[0-9]$//')" # master group name quorum="${SENTINEL_QUORUM-2}" # quorum needed @@ -17,12 +17,30 @@ parallel_syncs=${PARALEL_SYNCS-1} # Get all the kubernetes pods labels=`echo $(cat /etc/pod-info/labels) | tr -d '"' | tr " " ","` +try_step_interval=${TRY_STEP_INTERVAL-"1"} +max_tries=${MAX_TRIES-"3"} +retry() { + local tries=0 + until $@ ; do + status=$? + tries=$(($tries + 1)) + if [ $tries -gt $max_tries ] ; then + >&2 echo "Failed to run \`$@\` after $max_tries tries..." + return $status + fi + sleepsec=$(($tries * $try_step_interval)) + >&2 echo "Failed: \`$@\`, retyring in $sleepsec seconds..." + sleep $sleepsec + done + return $? +} + cli(){ - redis-cli -p $redis_port $@ + retry redis-cli -p $redis_port $@ } sentinel-cli(){ - redis-cli -p $sentinel_port $@ + retry redis-cli -p $sentinel_port $@ } ping() { @@ -75,7 +93,7 @@ hosts(){ boot(){ sleep $(($failover_timeout / 1000)) - ping-both || exit 1 + ping-both master=$(active-master) if [[ -n "$master" ]] ; then become-slave-of $master @@ -85,6 +103,12 @@ boot(){ fi echo "Ready!" touch booted + + set-role-label "none" # set roll label to nothing +} + +set-role-label(){ + kubectl label --overwrite pods `hostname` role=$1 } monitor-label(){ @@ -93,7 +117,7 @@ monitor-label(){ ping-both || exit 1 current_role=`role` if [[ "$last_role" != "$current_role" ]] ; then - kubectl label --overwrite pods `hostname` role=$current_role + set-role-label $current_role last_role=$current_role fi sleep 1 From bed31e9c983226e2ca928065c68ab27640a60336 Mon Sep 17 00:00:00 2001 From: jwaldrip Date: Fri, 7 Apr 2017 15:49:46 -0600 Subject: [PATCH 2/8] set the roll at the beginning --- sidecar.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sidecar.sh b/sidecar.sh index 378288a..3ef0699 100644 --- a/sidecar.sh +++ b/sidecar.sh @@ -92,6 +92,7 @@ hosts(){ } boot(){ + set-role-label "none" # set roll label to nothing sleep $(($failover_timeout / 1000)) ping-both master=$(active-master) @@ -103,8 +104,6 @@ boot(){ fi echo "Ready!" touch booted - - set-role-label "none" # set roll label to nothing } set-role-label(){ From 8a1b65262c9a0d4952b1ef1e81c1d4bd99ac49c8 Mon Sep 17 00:00:00 2001 From: jwaldrip Date: Fri, 7 Apr 2017 16:09:18 -0600 Subject: [PATCH 3/8] lock a version --- build-and-push.sh | 2 +- k8s/redis-node-statefulset.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-and-push.sh b/build-and-push.sh index d26bf9a..93d2ebd 100755 --- a/build-and-push.sh +++ b/build-and-push.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -image="commercialtribe/redis-sentinel-sidecar" +image="commercialtribe/redis-sentinel-sidecar:v20170407.1" login(){ docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD diff --git a/k8s/redis-node-statefulset.yaml b/k8s/redis-node-statefulset.yaml index 23d8bb3..62487cf 100644 --- a/k8s/redis-node-statefulset.yaml +++ b/k8s/redis-node-statefulset.yaml @@ -58,7 +58,7 @@ spec: # Sidecar - name: sidecar - image: commercialtribe/redis-sentinel-sidecar:latest + image: commercialtribe/redis-sentinel-sidecar:v20170407.1 imagePullPolicy: Always env: - name: POD_NAMESPACE From 29da76512e937287a84ee5a3ef2221207c8fc91e Mon Sep 17 00:00:00 2001 From: jwaldrip Date: Fri, 7 Apr 2017 16:26:58 -0600 Subject: [PATCH 4/8] if docker pull then fail push --- .travis.yml | 2 +- build-and-push.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6b109a8..84eb537 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ deploy: - provider: script script: ./build-and-push.sh on: - branch: master + all_branches: true - provider: script script: ./deploy.sh gke_commercial-tribe-staging_us-central1-a_staging develop on: diff --git a/build-and-push.sh b/build-and-push.sh index 93d2ebd..3b22ffe 100755 --- a/build-and-push.sh +++ b/build-and-push.sh @@ -6,6 +6,7 @@ login(){ } push(){ + if docker pull $image ; then exit 1 ; fi docker push $image } From b988db50fd5a430ee40b74b07750ae10bd09671c Mon Sep 17 00:00:00 2001 From: jwaldrip Date: Fri, 7 Apr 2017 16:34:47 -0600 Subject: [PATCH 5/8] allow for the same image to be overwritten --- Dockerfile | 2 +- build-and-push.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7ff12b3..8546c81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,6 @@ ADD https://storage.googleapis.com/kubernetes-release/release/v1.5.3/bin/linux/a RUN chmod +x /usr/local/bin/kubectl WORKDIR /app -ADD sidecar.sh /app/sidecar.sh +ADD . /app RUN chmod +x /app/sidecar.sh CMD /app/sidecar.sh diff --git a/build-and-push.sh b/build-and-push.sh index 3b22ffe..93d2ebd 100755 --- a/build-and-push.sh +++ b/build-and-push.sh @@ -6,7 +6,6 @@ login(){ } push(){ - if docker pull $image ; then exit 1 ; fi docker push $image } From 300cc046a7efa8f352ea703e4d0e5e045dde997e Mon Sep 17 00:00:00 2001 From: jwaldrip Date: Fri, 7 Apr 2017 16:34:51 -0600 Subject: [PATCH 6/8] allow for the same image to be overwritten --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4ef9703 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +* +!sidecar.sh From 99a9e881274b16cb7caeb54423daf5d6696c9682 Mon Sep 17 00:00:00 2001 From: jwaldrip Date: Fri, 7 Apr 2017 17:19:13 -0600 Subject: [PATCH 7/8] change container names --- k8s/redis-node-statefulset.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/redis-node-statefulset.yaml b/k8s/redis-node-statefulset.yaml index 62487cf..0d9851e 100644 --- a/k8s/redis-node-statefulset.yaml +++ b/k8s/redis-node-statefulset.yaml @@ -15,7 +15,7 @@ spec: terminationGracePeriodSeconds: 10 containers: # Redis - - name: redis + - name: redis-node image: redis:3.2 command: - redis-server @@ -40,7 +40,7 @@ spec: <<: *healthcheck # Sentinel - - name: sentinel + - name: redis-sentinel image: redis:3.2 command: [ "bash", "-c", "touch sentinel.conf && redis-sentinel sentinel.conf" ] ports: @@ -57,7 +57,7 @@ spec: <<: *healthcheck # Sidecar - - name: sidecar + - name: redis-sidecar image: commercialtribe/redis-sentinel-sidecar:v20170407.1 imagePullPolicy: Always env: From 599c61e620bf2d3276a7c74fcbc15dbd32a82185 Mon Sep 17 00:00:00 2001 From: jwaldrip Date: Fri, 7 Apr 2017 17:24:16 -0600 Subject: [PATCH 8/8] update docker --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8546c81..f271f11 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM redis:3.2 MAINTAINER Jason Waldrip -ADD https://storage.googleapis.com/kubernetes-release/release/v1.5.3/bin/linux/amd64/kubectl /usr/local/bin/kubectl +ADD https://storage.googleapis.com/kubernetes-release/release/v1.6.0/bin/linux/amd64/kubectl /usr/local/bin/kubectl RUN chmod +x /usr/local/bin/kubectl WORKDIR /app