Initial commit

.editorconfig

@@ -0,0 +1,9 @@
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+max_line_length = 120
+tab_width = 2
+trim_trailing_whitespace = true

.github/dependabot.yml

@@ -0,0 +1,10 @@
+# See GitHub's docs for more information on this file:
+# https://docs.github.com/en/free-pro-team@latest/github/administering-a-repository/configuration-options-for-dependency-updates
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every weekday
+      interval: "daily"

.github/workflows/ci.yml

@@ -0,0 +1,73 @@
+on:
+  push:
+    branches: [ "*" ]
+    tags: [ "*" ]
+  schedule:
+    - cron: "0 10 * * 1"
+
+env:
+  REGISTRY: "ghcr.io"
+  IMAGE_NAME: "${{ github.repository }}"
+  DOCKER_LAYER_CACHE: "/tmp/.buildx-cache"
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        MCROUTER_UPSTREAM_IMAGE_TAG:
+          - "2023.07.17.00-1-20240929"
+          - "latest"
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Cache Docker layers
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.DOCKER_LAYER_CACHE }}
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,event=push,enable=true,prefix=wiki-${{ matrix.MCROUTER_UPSTREAM_IMAGE_TAG }}-md-,format=short
+            type=raw,event=push,enable={{ is_default_branch }},value=wiki-${{ matrix.MCROUTER_UPSTREAM_IMAGE_TAG }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: "."
+          file: "./Dockerfile"
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64 #,linux/arm64/v8 - until we self-build or wikipedia publishes arm64 builds
+          provenance: false
+          pull: true
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          build-args: |
+            MCROUTER_UPSTREAM_IMAGE_TAG=${{ matrix.MCROUTER_UPSTREAM_IMAGE_TAG }}
+          cache-from: type=local,src=${{ env.DOCKER_LAYER_CACHE }}
+          cache-to: type=local,dest=${{ env.DOCKER_LAYER_CACHE }}

.gitignore

@@ -0,0 +1,3 @@
+.idea
+.venv
+*.iml

Dockerfile

@@ -0,0 +1,17 @@
+ARG MCROUTER_UPSTREAM_IMAGE_TAG="2023.07.17.00-1-20240929"
+FROM docker-registry.wikimedia.org/mcrouter:${MCROUTER_UPSTREAM_IMAGE_TAG} AS upstream
+
+USER root
+RUN apt -y update && \
+    apt -y dist-upgrade && \
+    apt -y install --no-install-recommends \
+      dnsutils \
+      jq \
+      vim && \
+    apt -y autoremove && \
+    rm -rf /var/cache/*
+
+ENV PATH="/scripts:$PATH"
+COPY --chown=root:root ./scripts /scripts
+
+USER mcrouter

README.md

@@ -0,0 +1,41 @@
+# mcrouter for Kubernetes
+
+## Summary
+
+Facebook's [mcrouter](https://github.com/facebook/mcrouter) is a cool project for anyone with the need to horizontally
+scale caching clusters in dynamic environments like Kubernetes, where specific workload instances cannot expect
+unchanging internal IP addresses.
+
+Finally, to achieve a truly dynamic setup is annoyingly verbose due to needing to regenerate JSON configuration files
+and diff them on a timer after doing a bunch of DNS lookups per memcached cluster.
+
+## Usage
+
+Write your mcrouter configuration as you usually do, using `dnssrv:` prefix to get a dynamic list of servers in a pool
+based on a headless Kubernetes service.
+
+For example, for headless service `foo-headless` in namespace `memcache`:
+
+```json
+{
+  "route": "PoolRoute|dynamic",
+  "pools": {
+    "dynamic": {
+      "servers": [
+        "dnssrv:_memcache._tcp.foo-headless.memcache.svc.cluster.local"
+      ]
+    }
+  }
+}
+```
+
+## Limitations
+
+1. You can have pools with static servers, but you cannot mix dynamic and static server entries in the same pool
+2. in-cluster IPv6 addressing is not supported
+3. These images are downstream
+   of [Wikipedia's images of mcrouter](https://docker-registry.wikimedia.org/mcrouter/tags/), as compiling mcrouter is
+   currently exceedingly difficult, slow and resource-intensive (
+   see [PR #449](https://github.com/facebook/mcrouter/pull/449) and linked issues/PRs). Debated instead using a plain
+   image, but the extra size of a full image for 2 shell scripts seemed a little silly, when one probably wants to use a
+   readymade mcrouter image anyway de to how annoying it is to build.

scripts/lib.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# minimal IP address regex
+# not there for security but rather to catch severe DNS misconfigurations if they don't raise error codes
+# from https://stackoverflow.com/a/36760050 with minimal replacements to support whatever stupid limitations afflict crusty grep setups
+REGEX_IP4="^((25[0-5]|(2[0-4]|1[0-9]|[1-9]?)[0-9]\.){3}(25[0-5]|(2[0-4]|1[0-9]|[1-9]?)[0-9]))$"
+# REGEX_IP6="lol good luck"
+
+# in-place jq
+function jq_i() {
+  local command=$1
+  local file=$2
+
+  local tmpfile
+  tmpfile=$(mktemp)
+
+  if jq "$command" "$file" > "$tmpfile"; then
+    mv -f "$tmpfile" "$file"
+  else
+    rm "$tmpfile"
+    return 1
+  fi
+}
+
+function resolve_cluster_pods_to() {
+  local cluster_dnssrv=$1
+  local destination_file=$2
+
+  if [ -z "$cluster_dnssrv" ] || [ -z "$destination_file" ]; then
+    help
+  fi
+
+  local srv_result
+  if ! srv_result=$(dig +short "$cluster_dnssrv" SRV); then
+    echo "Failed resolving SRV records for \"$cluster_dnssrv\" (dig(1) exit code $?)"
+    exit 1
+  fi
+
+  if [ -z "$srv_result" ]; then
+    echo "Empty response while looking up SRV records at \"$cluster_dnssrv\""
+    exit 1
+  fi
+
+  # poor man's brittle-but-still-technically-consistent sorting
+  srv_result=$(echo "$srv_result" | sort -n -k4)
+
+  local pods_json=()
+  while IFS= read -r line; do
+    # extract pod-specific DNS A record
+    if ! pod_a_record=$(echo "$line" | awk '{print $4}'); then
+      echo "Failed extracting pod A record from SRV lookup response"
+      exit 1
+    fi
+    echo "> Resolving pod A record \"$pod_a_record\""
+
+    # grab the pod's name from the A record
+    if ! pod_name=$(echo "$pod_a_record" | cut -d '.' -f1); then
+      echo "Failed extracting pod's name from A record"
+      exit 1
+    fi
+
+    # resolve the A record to get the relevant pod's cluster IP
+    if ! pod_ip_lookup=$(dig +short "$pod_a_record"); then
+      echo "Failed resolving pod cluster IP from its A record \"$pod_a_record\" (dig(1) exit code $?)"
+      exit 1
+    fi
+
+    # extract IP from pod IP lookup
+    if ! pod_ip=$(echo "$pod_ip_lookup" | grep -Eio "$REGEX_IP4"); then
+      echo "Failed extracting pod IP address from A record response. Unexpected response format:"
+      echo "$pod_ip_lookup"
+      exit 1
+    fi
+
+    if ! pod_port=$(echo "$line" | cut -d ' ' -f3 | grep -Eio "^[1-9][0-9]*$"); then
+      echo "Failed extracting pod service port from SRV lookup response. Unexpected response format:"
+      echo "$line"
+      exit 1
+    fi
+
+    echo "<+ name=\"$pod_name\""
+    echo "   ip=\"$pod_ip\""
+    echo "   port=$pod_port"
+
+    pods_json+=("{ \"name\": \"$pod_name\", \"ip\": \"$pod_ip\", \"port\": $pod_port }")
+  done <<< "$srv_result"
+
+  echo ""
+  echo "Found the following cluster pods:"
+
+  i=1
+  ilen=${#pods_json[@]}
+
+  printf "[\n" | tee "$destination_file"
+  for pod_json in "${pods_json[@]}"; do
+    printf "  %s" "$pod_json" | tee -a "$destination_file"
+    if [ "$i" != "$ilen" ]; then
+      printf "," | tee -a "$destination_file"
+    fi
+    printf "\n" | tee -a "$destination_file"
+    i=$(( i+1 ))
+  done
+  echo "]" | tee -a "$destination_file"
+}
+
+function are_different() {
+  local a="$1"
+  local b="$2"
+
+  if ! [ -f "$a" ] && [ -f "$b" ]; then
+    return 0
+  elif [ -f "$a" ] && ! [ -f "$b" ]; then
+    return 0
+  fi
+
+  if [ "$(sha256sum "$a" | cut -d ' ' -f1)" == "$(sha256sum "$b" | cut -d ' ' -f2)" ]; then
+    return 0
+  else
+    return 1
+  fi
+}

scripts/watch-config.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPTDIR="$(dirname "$0")"
+source "$SCRIPTDIR/lib.sh"
+
+CONFIG_TEMPLATE="${1:-${CONFIG_TEMPLATE:-"/config/config.tpl.json"}}"
+CONFIG_OUTPUT="${CONFIG_OUTPUT:-"/config/config.json"}"
+WATCH_INTERVAL_SECONDS="${WATCH_INTERVAL_SECONDS:-5}"
+
+echo "----------------------------------"
+echo "| mcrouter config watcher v0.0.1 |"
+echo "----------------------------------"
+echo "template: ${CONFIG_TEMPLATE}"
+echo "  output: ${CONFIG_OUTPUT}"
+echo "interval: ${WATCH_INTERVAL_SECONDS}"
+echo ""
+
+if ! [ -f "${CONFIG_TEMPLATE}" ]; then
+  echo "Configuration template file ${CONFIG_TEMPLATE} not found"
+  exit 1
+fi
+
+echo "Configuration template"
+echo "----------------------------------"
+cat "$CONFIG_TEMPLATE"
+echo ""
+
+if ! jq . "${CONFIG_TEMPLATE}" >/dev/null; then
+  echo "Configuration template file is not valid Json."
+  exit 1
+fi
+
+while true; do
+  echo "Refreshing mcrouter configuration..."
+  workdir="$(mktemp -d)"
+  conf_epoch_file="$workdir/config.json"
+  cp "$CONFIG_TEMPLATE" "$conf_epoch_file"
+
+  if ! template_clusters_raw=$(jq -r ".pools[].servers[]" "${CONFIG_TEMPLATE}" | sort -u | grep -Ei '^dnssrv\:') >/dev/null; then
+    echo "Unable to resolve the list of clusters used by the configuration template... Retrying in ${WATCH_INTERVAL_SECONDS}s..."
+    sleep "${WATCH_INTERVAL_SECONDS}"
+    continue
+  fi
+
+  template_clusters=()
+  for template_cluster_rline in $template_clusters_raw; do
+    template_cluster="${template_cluster_rline/dnssrv:/}"
+    echo "+ queued $template_cluster"
+    template_clusters+=("$template_cluster")
+  done
+
+  # if one cluster cannot be resolved, do not continue, but also don't just crash the pod
+  # instead, we just try again in the next round
+  successful_lookups="true"
+
+  i=1
+  ilen=${#template_clusters[@]}
+  for cluster_dnssrv in "${template_clusters[@]}"; do
+    cluster_file="$(mktemp -p "$workdir")"
+    echo "===== [$i/$ilen] - Processing $cluster_dnssrv... ====="
+    if ! cluster_info_lookup=$(resolve_cluster_pods_to "$cluster_dnssrv" "$cluster_file"); then
+      echo "$cluster_info_lookup"
+      successful_lookups="false"
+    else
+      echo "Resolved cluster pods!"
+      pools_with_cluster=$(jq -r ".pools | to_entries[] | select(.value.servers == [ \"dnssrv:\"$cluster_dnssrv ]) | .key")
+      for pool in $pools_with_cluster; do
+        if ! jq_i ".pools.$pool = $(cat "$cluster_file")" "$conf_epoch_file"; then
+          successful_lookups="false"
+        fi
+      done
+    fi
+    echo ""
+    i=$(( i+1 ))
+  done
+
+  if [ "$successful_lookups" == "true" ]; then
+    if are_different "$CONFIG_OUTPUT" "$conf_epoch_file"; then
+      if [ -f "$CONFIG_OUTPUT" ]; then
+        echo "Changes detected, updating live configuration..."
+        diff --color "$CONFIG_OUTPUT" "$conf_epoch_file"
+      else
+        echo "New configuration generated:"
+        cat "$conf_epoch_file"
+      fi
+      mv -fv "$conf_epoch_file" "$CONFIG_OUTPUT"
+    else
+      echo "Configuration unchanged"
+    fi
+  else
+    echo "Errors while processing, aborting."
+  fi
+  echo ""
+
+  rm -rf "$workdir"
+  sleep "${WATCH_INTERVAL_SECONDS}"
+done