diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..e7e12e4 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +[*] +charset = utf-8 +end_of_line = lf +indent_size = 2 +indent_style = space +insert_final_newline = true +max_line_length = 120 +tab_width = 2 +trim_trailing_whitespace = true diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..23f18a2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +# See GitHub's docs for more information on this file: +# https://docs.github.com/en/free-pro-team@latest/github/administering-a-repository/configuration-options-for-dependency-updates +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every weekday + interval: "daily" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..70229d7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,73 @@ +on: + push: + branches: [ "*" ] + tags: [ "*" ] + schedule: + - cron: "0 10 * * 1" + +env: + REGISTRY: "ghcr.io" + IMAGE_NAME: "${{ github.repository }}" + DOCKER_LAYER_CACHE: "/tmp/.buildx-cache" + +jobs: + build: + strategy: + matrix: + MCROUTER_UPSTREAM_IMAGE_TAG: + - "2023.07.17.00-1-20240929" + - "latest" + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: ${{ env.DOCKER_LAYER_CACHE }} + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,event=push,enable=true,prefix=wiki-${{ matrix.MCROUTER_UPSTREAM_IMAGE_TAG }}-md-,format=short + type=raw,event=push,enable={{ is_default_branch }},value=wiki-${{ matrix.MCROUTER_UPSTREAM_IMAGE_TAG }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: "." + file: "./Dockerfile" + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64 #,linux/arm64/v8 - until we self-build or wikipedia publishes arm64 builds + provenance: false + pull: true + push: true + tags: ${{ steps.meta.outputs.tags }} + build-args: | + MCROUTER_UPSTREAM_IMAGE_TAG=${{ matrix.MCROUTER_UPSTREAM_IMAGE_TAG }} + cache-from: type=local,src=${{ env.DOCKER_LAYER_CACHE }} + cache-to: type=local,dest=${{ env.DOCKER_LAYER_CACHE }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fc9290c --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea +.venv +*.iml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..30b979e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +ARG MCROUTER_UPSTREAM_IMAGE_TAG="2023.07.17.00-1-20240929" +FROM docker-registry.wikimedia.org/mcrouter:${MCROUTER_UPSTREAM_IMAGE_TAG} AS upstream + +USER root +RUN apt -y update && \ + apt -y dist-upgrade && \ + apt -y install --no-install-recommends \ + dnsutils \ + jq \ + vim && \ + apt -y autoremove && \ + rm -rf /var/cache/* + +ENV PATH="/scripts:$PATH" +COPY --chown=root:root ./scripts /scripts + +USER mcrouter diff --git a/README.md b/README.md new file mode 100644 index 0000000..0a5fd73 --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +# mcrouter for Kubernetes + +## Summary + +Facebook's [mcrouter](https://github.com/facebook/mcrouter) is a cool project for anyone with the need to horizontally +scale caching clusters in dynamic environments like Kubernetes, where specific workload instances cannot expect +unchanging internal IP addresses. + +Finally, to achieve a truly dynamic setup is annoyingly verbose due to needing to regenerate JSON configuration files +and diff them on a timer after doing a bunch of DNS lookups per memcached cluster. + +## Usage + +Write your mcrouter configuration as you usually do, using `dnssrv:` prefix to get a dynamic list of servers in a pool +based on a headless Kubernetes service. + +For example, for headless service `foo-headless` in namespace `memcache`: + +```json +{ + "route": "PoolRoute|dynamic", + "pools": { + "dynamic": { + "servers": [ + "dnssrv:_memcache._tcp.foo-headless.memcache.svc.cluster.local" + ] + } + } +} +``` + +## Limitations + +1. You can have pools with static servers, but you cannot mix dynamic and static server entries in the same pool +2. in-cluster IPv6 addressing is not supported +3. These images are downstream + of [Wikipedia's images of mcrouter](https://docker-registry.wikimedia.org/mcrouter/tags/), as compiling mcrouter is + currently exceedingly difficult, slow and resource-intensive ( + see [PR #449](https://github.com/facebook/mcrouter/pull/449) and linked issues/PRs). Debated instead using a plain + image, but the extra size of a full image for 2 shell scripts seemed a little silly, when one probably wants to use a + readymade mcrouter image anyway de to how annoying it is to build. diff --git a/scripts/lib.sh b/scripts/lib.sh new file mode 100755 index 0000000..2304c36 --- /dev/null +++ b/scripts/lib.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# minimal IP address regex +# not there for security but rather to catch severe DNS misconfigurations if they don't raise error codes +# from https://stackoverflow.com/a/36760050 with minimal replacements to support whatever stupid limitations afflict crusty grep setups +REGEX_IP4="^((25[0-5]|(2[0-4]|1[0-9]|[1-9]?)[0-9]\.){3}(25[0-5]|(2[0-4]|1[0-9]|[1-9]?)[0-9]))$" +# REGEX_IP6="lol good luck" + +# in-place jq +function jq_i() { + local command=$1 + local file=$2 + + local tmpfile + tmpfile=$(mktemp) + + if jq "$command" "$file" > "$tmpfile"; then + mv -f "$tmpfile" "$file" + else + rm "$tmpfile" + return 1 + fi +} + +function resolve_cluster_pods_to() { + local cluster_dnssrv=$1 + local destination_file=$2 + + if [ -z "$cluster_dnssrv" ] || [ -z "$destination_file" ]; then + help + fi + + local srv_result + if ! srv_result=$(dig +short "$cluster_dnssrv" SRV); then + echo "Failed resolving SRV records for \"$cluster_dnssrv\" (dig(1) exit code $?)" + exit 1 + fi + + if [ -z "$srv_result" ]; then + echo "Empty response while looking up SRV records at \"$cluster_dnssrv\"" + exit 1 + fi + + # poor man's brittle-but-still-technically-consistent sorting + srv_result=$(echo "$srv_result" | sort -n -k4) + + local pods_json=() + while IFS= read -r line; do + # extract pod-specific DNS A record + if ! pod_a_record=$(echo "$line" | awk '{print $4}'); then + echo "Failed extracting pod A record from SRV lookup response" + exit 1 + fi + echo "> Resolving pod A record \"$pod_a_record\"" + + # grab the pod's name from the A record + if ! pod_name=$(echo "$pod_a_record" | cut -d '.' -f1); then + echo "Failed extracting pod's name from A record" + exit 1 + fi + + # resolve the A record to get the relevant pod's cluster IP + if ! pod_ip_lookup=$(dig +short "$pod_a_record"); then + echo "Failed resolving pod cluster IP from its A record \"$pod_a_record\" (dig(1) exit code $?)" + exit 1 + fi + + # extract IP from pod IP lookup + if ! pod_ip=$(echo "$pod_ip_lookup" | grep -Eio "$REGEX_IP4"); then + echo "Failed extracting pod IP address from A record response. Unexpected response format:" + echo "$pod_ip_lookup" + exit 1 + fi + + if ! pod_port=$(echo "$line" | cut -d ' ' -f3 | grep -Eio "^[1-9][0-9]*$"); then + echo "Failed extracting pod service port from SRV lookup response. Unexpected response format:" + echo "$line" + exit 1 + fi + + echo "<+ name=\"$pod_name\"" + echo " ip=\"$pod_ip\"" + echo " port=$pod_port" + + pods_json+=("{ \"name\": \"$pod_name\", \"ip\": \"$pod_ip\", \"port\": $pod_port }") + done <<< "$srv_result" + + echo "" + echo "Found the following cluster pods:" + + i=1 + ilen=${#pods_json[@]} + + printf "[\n" | tee "$destination_file" + for pod_json in "${pods_json[@]}"; do + printf " %s" "$pod_json" | tee -a "$destination_file" + if [ "$i" != "$ilen" ]; then + printf "," | tee -a "$destination_file" + fi + printf "\n" | tee -a "$destination_file" + i=$(( i+1 )) + done + echo "]" | tee -a "$destination_file" +} + +function are_different() { + local a="$1" + local b="$2" + + if ! [ -f "$a" ] && [ -f "$b" ]; then + return 0 + elif [ -f "$a" ] && ! [ -f "$b" ]; then + return 0 + fi + + if [ "$(sha256sum "$a" | cut -d ' ' -f1)" == "$(sha256sum "$b" | cut -d ' ' -f2)" ]; then + return 0 + else + return 1 + fi +} diff --git a/scripts/watch-config.sh b/scripts/watch-config.sh new file mode 100755 index 0000000..b4e0547 --- /dev/null +++ b/scripts/watch-config.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPTDIR="$(dirname "$0")" +source "$SCRIPTDIR/lib.sh" + +CONFIG_TEMPLATE="${1:-${CONFIG_TEMPLATE:-"/config/config.tpl.json"}}" +CONFIG_OUTPUT="${CONFIG_OUTPUT:-"/config/config.json"}" +WATCH_INTERVAL_SECONDS="${WATCH_INTERVAL_SECONDS:-5}" + +echo "----------------------------------" +echo "| mcrouter config watcher v0.0.1 |" +echo "----------------------------------" +echo "template: ${CONFIG_TEMPLATE}" +echo " output: ${CONFIG_OUTPUT}" +echo "interval: ${WATCH_INTERVAL_SECONDS}" +echo "" + +if ! [ -f "${CONFIG_TEMPLATE}" ]; then + echo "Configuration template file ${CONFIG_TEMPLATE} not found" + exit 1 +fi + +echo "Configuration template" +echo "----------------------------------" +cat "$CONFIG_TEMPLATE" +echo "" + +if ! jq . "${CONFIG_TEMPLATE}" >/dev/null; then + echo "Configuration template file is not valid Json." + exit 1 +fi + +while true; do + echo "Refreshing mcrouter configuration..." + workdir="$(mktemp -d)" + conf_epoch_file="$workdir/config.json" + cp "$CONFIG_TEMPLATE" "$conf_epoch_file" + + if ! template_clusters_raw=$(jq -r ".pools[].servers[]" "${CONFIG_TEMPLATE}" | sort -u | grep -Ei '^dnssrv\:') >/dev/null; then + echo "Unable to resolve the list of clusters used by the configuration template... Retrying in ${WATCH_INTERVAL_SECONDS}s..." + sleep "${WATCH_INTERVAL_SECONDS}" + continue + fi + + template_clusters=() + for template_cluster_rline in $template_clusters_raw; do + template_cluster="${template_cluster_rline/dnssrv:/}" + echo "+ queued $template_cluster" + template_clusters+=("$template_cluster") + done + + # if one cluster cannot be resolved, do not continue, but also don't just crash the pod + # instead, we just try again in the next round + successful_lookups="true" + + i=1 + ilen=${#template_clusters[@]} + for cluster_dnssrv in "${template_clusters[@]}"; do + cluster_file="$(mktemp -p "$workdir")" + echo "===== [$i/$ilen] - Processing $cluster_dnssrv... =====" + if ! cluster_info_lookup=$(resolve_cluster_pods_to "$cluster_dnssrv" "$cluster_file"); then + echo "$cluster_info_lookup" + successful_lookups="false" + else + echo "Resolved cluster pods!" + pools_with_cluster=$(jq -r ".pools | to_entries[] | select(.value.servers == [ \"dnssrv:\"$cluster_dnssrv ]) | .key") + for pool in $pools_with_cluster; do + if ! jq_i ".pools.$pool = $(cat "$cluster_file")" "$conf_epoch_file"; then + successful_lookups="false" + fi + done + fi + echo "" + i=$(( i+1 )) + done + + if [ "$successful_lookups" == "true" ]; then + if are_different "$CONFIG_OUTPUT" "$conf_epoch_file"; then + if [ -f "$CONFIG_OUTPUT" ]; then + echo "Changes detected, updating live configuration..." + diff --color "$CONFIG_OUTPUT" "$conf_epoch_file" + else + echo "New configuration generated:" + cat "$conf_epoch_file" + fi + mv -fv "$conf_epoch_file" "$CONFIG_OUTPUT" + else + echo "Configuration unchanged" + fi + else + echo "Errors while processing, aborting." + fi + echo "" + + rm -rf "$workdir" + sleep "${WATCH_INTERVAL_SECONDS}" +done