diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4ef9703 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +* +!sidecar.sh diff --git a/.travis.yml b/.travis.yml index 6b109a8..84eb537 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ deploy: - provider: script script: ./build-and-push.sh on: - branch: master + all_branches: true - provider: script script: ./deploy.sh gke_commercial-tribe-staging_us-central1-a_staging develop on: diff --git a/Dockerfile b/Dockerfile index 7ff12b3..f271f11 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ FROM redis:3.2 MAINTAINER Jason Waldrip -ADD https://storage.googleapis.com/kubernetes-release/release/v1.5.3/bin/linux/amd64/kubectl /usr/local/bin/kubectl +ADD https://storage.googleapis.com/kubernetes-release/release/v1.6.0/bin/linux/amd64/kubectl /usr/local/bin/kubectl RUN chmod +x /usr/local/bin/kubectl WORKDIR /app -ADD sidecar.sh /app/sidecar.sh +ADD . /app RUN chmod +x /app/sidecar.sh CMD /app/sidecar.sh diff --git a/build-and-push.sh b/build-and-push.sh index d26bf9a..93d2ebd 100755 --- a/build-and-push.sh +++ b/build-and-push.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -image="commercialtribe/redis-sentinel-sidecar" +image="commercialtribe/redis-sentinel-sidecar:v20170407.1" login(){ docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD diff --git a/k8s/redis-node-statefulset.yaml b/k8s/redis-node-statefulset.yaml index 23d8bb3..0d9851e 100644 --- a/k8s/redis-node-statefulset.yaml +++ b/k8s/redis-node-statefulset.yaml @@ -15,7 +15,7 @@ spec: terminationGracePeriodSeconds: 10 containers: # Redis - - name: redis + - name: redis-node image: redis:3.2 command: - redis-server @@ -40,7 +40,7 @@ spec: <<: *healthcheck # Sentinel - - name: sentinel + - name: redis-sentinel image: redis:3.2 command: [ "bash", "-c", "touch sentinel.conf && redis-sentinel sentinel.conf" ] ports: @@ -57,8 +57,8 @@ spec: <<: *healthcheck # Sidecar - - name: sidecar - image: commercialtribe/redis-sentinel-sidecar:latest + - name: redis-sidecar + image: commercialtribe/redis-sentinel-sidecar:v20170407.1 imagePullPolicy: Always env: - name: POD_NAMESPACE diff --git a/sidecar.sh b/sidecar.sh index 82eb214..3ef0699 100644 --- a/sidecar.sh +++ b/sidecar.sh @@ -5,7 +5,7 @@ set -e # Set vars ip=${POD_IP-`hostname -i`} # ip address of pod redis_port=${NODE_PORT_NUMBER-6379} # redis port -sentinel_port=${SENTINEL_PORT_NUMBER-26379} # sentinel port +sentinel_port=${SENTINEL_PORT_NUMBER-26379} # sentinel port group_name="$POD_NAMESPACE-$(hostname | sed 's/-[0-9]$//')" # master group name quorum="${SENTINEL_QUORUM-2}" # quorum needed @@ -17,12 +17,30 @@ parallel_syncs=${PARALEL_SYNCS-1} # Get all the kubernetes pods labels=`echo $(cat /etc/pod-info/labels) | tr -d '"' | tr " " ","` +try_step_interval=${TRY_STEP_INTERVAL-"1"} +max_tries=${MAX_TRIES-"3"} +retry() { + local tries=0 + until $@ ; do + status=$? + tries=$(($tries + 1)) + if [ $tries -gt $max_tries ] ; then + >&2 echo "Failed to run \`$@\` after $max_tries tries..." + return $status + fi + sleepsec=$(($tries * $try_step_interval)) + >&2 echo "Failed: \`$@\`, retyring in $sleepsec seconds..." + sleep $sleepsec + done + return $? +} + cli(){ - redis-cli -p $redis_port $@ + retry redis-cli -p $redis_port $@ } sentinel-cli(){ - redis-cli -p $sentinel_port $@ + retry redis-cli -p $sentinel_port $@ } ping() { @@ -74,8 +92,9 @@ hosts(){ } boot(){ + set-role-label "none" # set roll label to nothing sleep $(($failover_timeout / 1000)) - ping-both || exit 1 + ping-both master=$(active-master) if [[ -n "$master" ]] ; then become-slave-of $master @@ -87,13 +106,17 @@ boot(){ touch booted } +set-role-label(){ + kubectl label --overwrite pods `hostname` role=$1 +} + monitor-label(){ last_role=none while true ; do ping-both || exit 1 current_role=`role` if [[ "$last_role" != "$current_role" ]] ; then - kubectl label --overwrite pods `hostname` role=$current_role + set-role-label $current_role last_role=$current_role fi sleep 1