diff --git a/sentinel/redis.nomad b/sentinel/redis.nomad new file mode 100644 index 0000000..3288ebf --- /dev/null +++ b/sentinel/redis.nomad @@ -0,0 +1,125 @@ +job "redis" { + datacenters = [[ .datacenters | toJson ]] + type = "service" + + update { + max_parallel = 1 + stagger = "10s" + } + + // add 0 to force convert int to int64, otherwise loop will error + [[ range $id := loop ( add .redis.count 0 ) ]] + group "redis-[[$id]]" { + count = 1 + + network { + mode = "bridge" + + port "db" { + static = 6379 + } + } + + service { + name = "redis" + tags = ["[[$id]]"] + + port = "db" + + check { + type = "script" + task = "redis" + name = "Redis Leader or Follower Check" + command = "/bin/bash" + args = ["/local/redis-hc.sh"] + interval = "5s" + timeout = "5s" + } + } + + ephemeral_disk { + migrate = true + size = 500 + sticky = true + } + + task "redis" { + driver = "docker" + + template { + data = </dev/null +is_master=$? + +if [[ $is_master -eq 0 ]]; then + echo "current node is master" + exit 0 +fi + +echo "current node is replica" +master_host="$(echo "$replication_output" | grep "master_host:" | cut -d: -f2 | tr -d '\n\r')" +echo "ping to server @ $master_host" +timeout 1 redis-cli -h $master_host --raw ping +conn_success=$? + +if [[ $conn_success -eq 0 ]]; then + echo "master @ $master_host reachable" + exit 0 +else + echo "connection to master @ $master_host failed" + exit 1 +fi diff --git a/sentinel/scripts/redis.sh b/sentinel/scripts/redis.sh new file mode 100644 index 0000000..9547023 --- /dev/null +++ b/sentinel/scripts/redis.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +REDIS_CONFIG_PATH=/local/redis.conf + +SENTINEL_URL=redis-sentinel.service.consul +SENTINEL_PORT=26379 + +INIT_MASTER=0.redis.service.consul + +REDIS_CONFIG_TMPL=' +loglevel verbose +dir /alloc/data + +masterauth password +requirepass password +' + +echo "$REDIS_CONFIG_TMPL" >$REDIS_CONFIG_PATH + +echo "finding master..." +if [ "$(redis-cli -h $SENTINEL_URL -p $SENTINEL_PORT ping)" != "PONG" ]; then + echo "sentinel not found, defaulting to redis-0" + if [[ "$NOMAD_GROUP_NAME" == "redis-0" ]]; then + echo "this is redis-0, not updating config..." + else + echo "updating redis.conf..." + echo "slaveof $INIT_MASTER $NOMAD_PORT_db" >>$REDIS_CONFIG_PATH + fi +else + echo "sentinel found, finding master" + MASTER="$(redis-cli --raw -h $SENTINEL_URL -p $SENTINEL_PORT sentinel get-master-addr-by-name mymaster | head -1 | tr -d '\n\r')" + echo "master found: $MASTER" + if [[ "$MASTER" == "$NOMAD_HOST_IP_db" ]]; then + echo "master is self, not updating config..." + else + echo "updating redis.conf..." + echo "slaveof $MASTER $NOMAD_PORT_db" >>$REDIS_CONFIG_PATH + fi +fi + +exec redis-server $REDIS_CONFIG_PATH diff --git a/sentinel/scripts/sentinel.sh b/sentinel/scripts/sentinel.sh new file mode 100644 index 0000000..7a1e5a1 --- /dev/null +++ b/sentinel/scripts/sentinel.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +SENTINEL_CONFIG_PATH=/local/sentinel.conf + +REDIS_PASSWORD=password + +nodes=0.redis.service.consul,1.redis.service.consul,2.redis.service.consul + +# TODO: wait for discovery of redis first in consul dns sd. + +FORCE_INIT=false +RETRY_WAIT=5 + +export REDISCLI_AUTH="$REDIS_PASSWORD" + +# https://download.redis.io/redis-stable/sentinel.conf +function init_config { + SENTINEL_CONFIG=" +dir /alloc/data + +sentinel resolve-hostnames yes + +sentinel down-after-milliseconds mymaster 5000 +sentinel failover-timeout mymaster 60000 +sentinel parallel-syncs mymaster 1 +sentinel auth-pass mymaster $REDIS_PASSWORD +" + + echo "$SENTINEL_CONFIG" >$SENTINEL_CONFIG_PATH +} + +# in case the underlying host for this job changes, +# the existing configuration would contain wrong announce ip that still pointing to the old node +# always fixup the announce ip by updating it to the current +function patch_announce_ip { + echo "updating announce ip to $NOMAD_HOST_IP_sentinel" + # 1a --> append 1st line, prepend + sed -i.bk '/^sentinel announce-ip/d' $SENTINEL_CONFIG_PATH + sed -i.bk "1asentinel announce-ip $NOMAD_HOST_IP_sentinel" $SENTINEL_CONFIG_PATH +} + + +discover_count=0 +function patch_master_replica { + for node in $${nodes//,/ }; do + echo "finding master at $node" + MASTER="$(redis-cli --raw -h $node info replication | grep master_host: | cut -d: -f2 | tr -d '\n\r')" + if [[ "$MASTER" == "" ]]; then + echo "no master found" + else + echo "** found $MASTER **" + break + fi + done + + if [[ "$MASTER" == "" ]]; then + discover_count=$(( discover_count + 1)) + echo "discovered $discover_count time" + if [[ $discover_count -ge 5 ]]; then + echo "too many discover failures, crash" + exit 1 + fi + echo "no master found from any redis instance, wait $RETRY_WAIT s before retry" + sleep $RETRY_WAIT + patch_master_replica + fi + + sed -i.bk '/^sentinel monitor mymaster/d' $SENTINEL_CONFIG_PATH + sed -i.bk '/^sentinel known-replica mymaster/d' $SENTINEL_CONFIG_PATH + sed -i.bk "1asentinel monitor mymaster $MASTER 6379 $QUORUM" $SENTINEL_CONFIG_PATH +} + +if [[ -f $SENTINEL_CONFIG_PATH ]] && [[ $FORCE_INIT != 'true' ]]; then + echo "found existing config, skip init config" +else + init_config +fi + +patch_announce_ip +patch_master_replica + +exec redis-sentinel $SENTINEL_CONFIG_PATH