sentinel

sentinel/redis.nomad

@@ -0,0 +1,125 @@
+job "redis" {
+  datacenters = [[ .datacenters | toJson ]]
+  type        = "service"
+
+  update {
+    max_parallel = 1
+    stagger      = "10s"
+  }
+
+  // add 0 to force convert int to int64, otherwise loop will error
+  [[ range $id := loop ( add .redis.count 0 ) ]]
+  group "redis-[[$id]]" {
+    count = 1
+
+    network {
+      mode = "bridge"
+
+      port "db" {
+        static = 6379
+      }
+    }
+
+    service {
+      name = "redis"
+      tags = ["[[$id]]"]
+
+      port = "db"
+
+      check {
+        type     = "script"
+        task     = "redis"
+        name     = "Redis Leader or Follower Check"
+        command  = "/bin/bash"
+        args     = ["/local/redis-hc.sh"]
+        interval = "5s"
+        timeout  = "5s"
+      }
+    }
+
+    ephemeral_disk {
+      migrate = true
+      size    = 500
+      sticky  = true
+    }
+
+    task "redis" {
+      driver = "docker"
+
+      template {
+        data = <<EOH
+[[ fileContents ( print "./scripts/redis-hc.sh" ) ]]
+        EOH
+
+        destination = "local/redis-hc.sh"
+        change_mode = "noop"
+      }
+
+      config {
+        image   = "redis:6.2"
+        command = "bash"
+        args = [
+          "-c",
+          <<EOH
+[[ fileContents ( print "./scripts/redis.sh" ) ]]
+EOH
+        ]
+      }
+
+      resources {
+        cpu    = 500
+        memory = 256
+      }
+    }
+  }
+  [[- end ]]
+
+  group "sentinel" {
+    count = 3
+
+    network {
+      mode = "bridge"
+
+      port "sentinel" {
+        static = 26379
+      }
+    }
+
+    service {
+      name = "redis-sentinel"
+      port = "sentinel"
+    }
+
+    ephemeral_disk {
+      migrate = true
+      size    = 500
+      sticky  = true
+    }
+
+    task "sentinel" {
+      driver = "docker"
+
+      env {
+        // n/2 + 1
+        QUORUM = "[[ (add (divide  2 .sentienl.count) 1) ]]"
+      }
+
+      config {
+        image = "redis:6.2"
+
+        command = "bash"
+        args = [
+          "-c",
+          <<EOH
+[[ fileContents ( print "./scripts/sentinel.sh" ) ]]
+EOH
+        ]
+      }
+
+      resources {
+        cpu    = 500
+        memory = 256
+      }
+    }
+  }
+}

sentinel/redis.yml

@@ -0,0 +1,6 @@
+---
+redis:
+  count: 3
+sentienl:
+  count: 3
+datacenters: ["dc1"]

sentinel/scripts/redis-hc.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+REDIS_PASSWORD=password
+
+# check if is master or replica of a master
+
+export REDISCLI_AUTH="$REDIS_PASSWORD"
+
+replication_output="$(redis-cli --raw -h localhost -p 6379 info replication)"
+echo "$replication_output" | grep "role:master" >/dev/null
+is_master=$?
+
+if [[ $is_master -eq 0 ]]; then
+    echo "current node is master"
+    exit 0
+fi
+
+echo "current node is replica"
+master_host="$(echo "$replication_output" | grep "master_host:" | cut -d: -f2 | tr -d '\n\r')"
+echo "ping to server @ $master_host"
+timeout 1 redis-cli -h $master_host --raw ping
+conn_success=$?
+
+if [[ $conn_success -eq 0 ]]; then
+    echo "master @ $master_host reachable"
+    exit 0
+else
+    echo "connection to master @ $master_host failed"
+    exit 1
+fi

sentinel/scripts/redis.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+REDIS_CONFIG_PATH=/local/redis.conf
+
+SENTINEL_URL=redis-sentinel.service.consul
+SENTINEL_PORT=26379
+
+INIT_MASTER=0.redis.service.consul
+
+REDIS_CONFIG_TMPL='
+loglevel verbose
+dir /alloc/data
+
+masterauth password
+requirepass password
+'
+
+echo "$REDIS_CONFIG_TMPL" >$REDIS_CONFIG_PATH
+
+echo "finding master..."
+if [ "$(redis-cli -h $SENTINEL_URL -p $SENTINEL_PORT ping)" != "PONG" ]; then
+    echo "sentinel not found, defaulting to redis-0"
+    if [[ "$NOMAD_GROUP_NAME" == "redis-0" ]]; then
+        echo "this is redis-0, not updating config..."
+    else
+        echo "updating redis.conf..."
+        echo "slaveof $INIT_MASTER $NOMAD_PORT_db" >>$REDIS_CONFIG_PATH
+    fi
+else
+    echo "sentinel found, finding master"
+    MASTER="$(redis-cli --raw -h $SENTINEL_URL -p $SENTINEL_PORT sentinel get-master-addr-by-name mymaster | head -1 | tr -d '\n\r')"
+    echo "master found: $MASTER"
+    if [[ "$MASTER" == "$NOMAD_HOST_IP_db" ]]; then
+        echo "master is self, not updating config..."
+    else
+        echo "updating redis.conf..."
+        echo "slaveof $MASTER $NOMAD_PORT_db" >>$REDIS_CONFIG_PATH
+    fi
+fi
+
+exec redis-server $REDIS_CONFIG_PATH

sentinel/scripts/sentinel.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+SENTINEL_CONFIG_PATH=/local/sentinel.conf
+
+REDIS_PASSWORD=password
+
+nodes=0.redis.service.consul,1.redis.service.consul,2.redis.service.consul
+
+# TODO: wait for discovery of redis first in consul dns sd.
+
+FORCE_INIT=false
+RETRY_WAIT=5
+
+export REDISCLI_AUTH="$REDIS_PASSWORD"
+
+# https://download.redis.io/redis-stable/sentinel.conf
+function init_config {
+    SENTINEL_CONFIG="
+dir /alloc/data
+
+sentinel resolve-hostnames yes
+
+sentinel down-after-milliseconds mymaster 5000
+sentinel failover-timeout mymaster 60000
+sentinel parallel-syncs mymaster 1
+sentinel auth-pass mymaster $REDIS_PASSWORD
+"
+
+    echo "$SENTINEL_CONFIG" >$SENTINEL_CONFIG_PATH
+}
+
+# in case the underlying host for this job changes,
+# the existing configuration would contain wrong announce ip that still pointing to the old node
+# always fixup the announce ip by updating it to the current
+function patch_announce_ip {
+    echo "updating announce ip to $NOMAD_HOST_IP_sentinel"
+    # 1a --> append 1st line, prepend
+    sed -i.bk '/^sentinel announce-ip/d' $SENTINEL_CONFIG_PATH
+    sed -i.bk "1asentinel announce-ip $NOMAD_HOST_IP_sentinel" $SENTINEL_CONFIG_PATH
+}
+
+
+discover_count=0
+function patch_master_replica {
+    for node in $${nodes//,/ }; do
+        echo "finding master at $node"
+        MASTER="$(redis-cli --raw -h $node info replication | grep master_host: | cut -d: -f2 | tr -d '\n\r')"
+        if [[ "$MASTER" == "" ]]; then
+            echo "no master found"
+        else
+            echo "** found $MASTER **"
+            break
+        fi
+    done
+
+    if [[ "$MASTER" == "" ]]; then
+        discover_count=$(( discover_count + 1))
+        echo "discovered $discover_count time"
+        if [[ $discover_count -ge 5 ]]; then
+            echo "too many discover failures, crash"
+            exit 1
+        fi
+        echo "no master found from any redis instance, wait $RETRY_WAIT s before retry"
+        sleep $RETRY_WAIT
+        patch_master_replica
+    fi
+
+    sed -i.bk '/^sentinel monitor mymaster/d' $SENTINEL_CONFIG_PATH
+    sed -i.bk '/^sentinel known-replica mymaster/d' $SENTINEL_CONFIG_PATH
+    sed -i.bk "1asentinel monitor mymaster $MASTER 6379 $QUORUM" $SENTINEL_CONFIG_PATH
+}
+
+if [[ -f $SENTINEL_CONFIG_PATH ]] && [[ $FORCE_INIT != 'true' ]]; then
+    echo "found existing config, skip init config"
+else
+    init_config
+fi
+
+patch_announce_ip
+patch_master_replica
+
+exec redis-sentinel $SENTINEL_CONFIG_PATH