docker-compose.yml

version: "2.4"

x-runner-container:
  &runner-container
  privileged: true
  restart: unless-stopped
  labels:
    # Allow CI pipelines to load kernel modules on the host
    io.balena.features.kernel-modules: '1'
  tmpfs:
    - /tmp
    - /run
    - /scratch
  networks:
    - runner-network
  environment:
    ACTIONS_RUNNER_REGISTRATION_SLUG: enterprises/balena
    REGISTRY_MIRRORS: http://registry-cache:5000
    INSECURE_REGISTRIES: registry-cache:5000
    # container runners are only suitable for private repositories
    ACTIONS_RUNNER_GROUP: self-hosted-internal

x-runner-vm:
  &runner-vm
  cap_drop:
    - ALL
  cap_add:
    - NET_RAW # Required for VM networking
    - NET_ADMIN # Required for VM networking
    - CHOWN # Required to change ownership of files
    - DAC_OVERRIDE # Required by confd
  devices:
    - /dev/kvm # Required for Firecracker virtualization
    - /dev/net/tun # Required to create TAP/TUN device
  sysctls:
    - net.ipv4.ip_forward=1 # Required for VM networking
  tmpfs:
    - /tmp
  networks:
    - runner-network
  environment:
    ACTIONS_RUNNER_REGISTRATION_SLUG: enterprises/balena
    REGISTRY_MIRRORS: http://registry-cache:5000
    INSECURE_REGISTRIES: registry-cache:5000
    ACTIONS_RUNNER_GROUP: self-hosted

services:

  runner-container-1:
    <<: *runner-container
    image: ghcr.io/product-os/self-hosted-runners:6.2.20-jammy

  runner-focal-1:
    <<: *runner-vm
    image: ghcr.io/product-os/github-runner-vm:2.2.12-focal

  runner-jammy-1:
    <<: *runner-vm
    image: ghcr.io/product-os/github-runner-vm:2.2.12-jammy

  runner-jammy-2:
    <<: *runner-vm
    image: ghcr.io/product-os/github-runner-vm:2.2.12-jammy

  runner-jammy-3:
    <<: *runner-vm
    image: ghcr.io/product-os/github-runner-vm:2.2.12-jammy

  runner-noble-1:
    <<: *runner-vm
    image: ghcr.io/product-os/github-runner-vm:2.2.12-noble

  runner-noble-2:
    <<: *runner-vm
    image: ghcr.io/product-os/github-runner-vm:2.2.12-noble

  runner-noble-3:
    <<: *runner-vm
    image: ghcr.io/product-os/github-runner-vm:2.2.12-noble

  # https://distribution.github.io/distribution/recipes/mirror/
  registry-cache:
    image: registry:2.8.3
    volumes:
      - registry-data:/var/lib/registry
    tmpfs:
      - /tmp
      - /run
    networks:
      - runner-network
    environment:
      # Listen on the default bridge network on port 5000.
      # Do not expose this unsecured port on the host network!
      REGISTRY_HTTP_ADDR: 0.0.0.0:5000
      REGISTRY_PROXY_REMOTEURL: https://registry-1.docker.io
      # The proxy.ttl setting isn't supported until registry v3 but we can set it now and it will be ignored
      # https://github.com/distribution/distribution/blob/5d5c60f77c01b143b22be9d62af3b32843fbcb0c/configuration/configuration.go#L643-L653
      # https://github.com/distribution/distribution/blob/e1ec19ae60b8628b564d4fb01ce09ef81047987c/configuration/configuration.go#L628-L643
      REGISTRY_PROXY_TTL: 168h
      REGISTRY_STORAGE_DELETE_ENABLED: "true"
      # REGISTRY_PROXY_USERNAME: changeme
      # REGISTRY_PROXY_PASSWORD: changeme
      # REGISTRY_LOG_LEVEL: debug
      # Disable the blob descriptor cache by setting REGISTRY_STORAGE_CACHE_BLOBDESCRIPTOR=blah where blah is not inmemory or redis
      # https://github.com/distribution/distribution/issues/2367#issuecomment-1874449361
      REGISTRY_STORAGE_CACHE_BLOBDESCRIPTOR: blah

  # tag devices
  tag-sidecar:
    image: bash:alpine3.14
    restart: no
    entrypoint: ["/usr/local/bin/bash", "-c"]
    command:
      - |
          set -eu

          [[ "${VERBOSE:-false}" =~ on|On|Yes|yes|true|True ]] && set -x

          [[ $ENABLED == 'true' ]] || exit

          curl_with_opts() {
              curl --fail --silent --retry 3 --connect-timeout 3 --compressed "$@"
          }

          function get_robot() {
              local robot_json
              robot_json="$(curl_with_opts -u "${ROBOT_USER}:${ROBOT_PASS}" "${ROBOT_API}")"
              echo "${robot_json}"
          }

          which curl || apk add curl --no-cache
          which jq || apk add jq --no-cache

          device_id="$(curl_with_opts \
            "${BALENA_API_URL}/v6/device?\$filter=uuid%20eq%20'${BALENA_DEVICE_UUID}'" \
            -H "Content-Type: application/json" \
            -H "Authorization: Bearer ${BALENA_API_KEY}" | jq -r .d[].id)"

          myip="$(curl_with_opts https://ipinfo.io/ip)"

          flat_json="$(get_robot | jq -rc --arg ip "${myip}" '.[].server | select(.server_ip==$ip)' | sed 's/"/\\"/g')"

          if [[ -n $flat_json ]]; then
              curl_with_opts "${BALENA_API_URL}/v6/device_tag" \
                -H "Content-Type: application/json" \
                -H "Authorization: Bearer ${BALENA_API_KEY}" \
                --data "{\"device\":\"${device_id}\",\"tag_key\":\"hetzner-robot\",\"value\":\"${flat_json}\"}"
          fi

    environment:
      ENABLED: true
      ROBOT_API: https://robot-ws.your-server.de/server
    labels:
      io.balena.features.balena-api: '1'

  # enable IPv6
  enable-ipv6:
    image: bash:alpine3.14
    restart: no
    entrypoint: ["/usr/local/bin/bash", "-c"]
    command:
      - |
          set -eu

          [[ "${VERBOSE:-false}" =~ on|On|Yes|yes|true|True ]] && set -x

          [[ $ENABLED == 'true' ]] || exit

          curl_with_opts() {
              curl --fail --silent --retry 3 --connect-timeout 3 --compressed "$@"
          }

          function get_robot() {
              local robot_json
              robot_json="$(curl_with_opts -u "${ROBOT_USER}:${ROBOT_PASS}" "${ROBOT_API}")"
              echo "${robot_json}"
          }

          which curl || apk add curl --no-cache
          which jq || apk add jq --no-cache
          which nmcli || apk add networkmanager --no-cache

          myip="$(curl_with_opts https://ipinfo.io/ip)"
          flat_json="$(get_robot | jq -rc --arg ip "${myip}" '.[].server | select(.server_ip==$ip)')"
          ipv6_addresses="$(nmcli -f ipv6.addresses c s 'Wired connection 1' | awk '{print $2}')"

          if [[ -z $ipv6_addresses ]] || [[ $ipv6_addresses == '--' ]]; then
              ip="$(echo "${flat_json}" | jq -r '.subnet[].ip')"
              mask="$(echo "${flat_json}" | jq -r '.subnet[].mask')"
              ipv6_addresses="${ip}/${mask}"
              nmcli connection modify 'Wired connection 1' ipv6.addresses "${ipv6_addresses}"
              nmcli connection modify 'Wired connection 1' ipv6.gateway fe80::1
              nmcli connection modify 'Wired connection 1' ipv6.dns '2001:4860:4860::8888 2001:4860:4860::8844'
              nmcli connection modify 'Wired connection 1' ipv6.method manual
              nmcli connection up 'Wired connection 1'
          fi

    environment:
      ENABLED: true
      ROBOT_API: https://robot-ws.your-server.de/server
      DBUS_SYSTEM_BUS_ADDRESS: unix:path=/host/run/dbus/system_bus_socket
    labels:
      io.balena.features.dbus: '1'

  # create DNS record
  upsert-dns:
    image: bash:alpine3.14
    restart: no
    entrypoint: ["/usr/local/bin/bash", "-c"]
    command:
      - |
          set -eu

          [[ "${VERBOSE:-false}" =~ on|On|Yes|yes|true|True ]] && set -x

          [[ $ENABLED == 'true' ]] || exit

          curl_with_opts() {
              curl --fail --silent --retry 3 --connect-timeout 3 --compressed "$@"
          }

          function get_robot() {
              local robot_json
              robot_json="$(curl_with_opts -u "${ROBOT_USER}:${ROBOT_PASS}" "${ROBOT_API}/$1")"
              echo "${robot_json}"
          }

          function post_robot() {
              curl_with_opts -u "${ROBOT_USER}:${ROBOT_PASS}" "${ROBOT_API}/$1" -d "$2"
          }

          which curl || apk add curl --no-cache
          which jq || apk add jq --no-cache
          which dig || apk add bind-tools --no-cache

          myip="$(curl_with_opts https://ipinfo.io/ip)"
          flat_json="$(get_robot server | jq -rc --arg ip "${myip}" '.[].server | select(.server_ip==$ip)')"
          ipv4="$(echo "${flat_json}" | jq -r '.server_ip')"
          ipv6="$(echo "${flat_json}" | jq -r '.server_ipv6_net')"

          if curl_with_opts "https://api.cloudflare.com/client/v4/user/tokens/verify" \
            -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \
            -H "Content-Type:application/json"; then

              zone_id="$(curl_with_opts "https://api.cloudflare.com/client/v4/zones?name=${CLOUDFLARE_DNS_ZONE}&status=active&account.id" \
                -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \
                -H "Content-Type:application/json" | jq -r '.result[].id')"

              if [[ -n $zone_id ]]; then
                  dns_name="${BALENA_DEVICE_UUID::7}.runners"

                  if [[ -z "$(dig +short A "${dns_name}.${CLOUDFLARE_DNS_ZONE}")" ]]; then
                      curl_with_opts "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \
                        -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \
                        -H "Content-Type:application/json" \
                        --data "{\"type\":\"A\",\"name\":\"${dns_name}\",\"content\":\"${ipv4}\",\"ttl\":120,\"priority\":10,\"proxied\":false}"
                  fi

                  if [[ -z "$(dig +short AAAA "${dns_name}.${CLOUDFLARE_DNS_ZONE}")" ]]; then
                      curl_with_opts "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \
                        -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \
                        -H "Content-Type:application/json" \
                        --data "{\"type\":\"AAAA\",\"name\":\"${dns_name}\",\"content\":\"${ipv6}\",\"ttl\":120,\"priority\":10,\"proxied\":false}"
                  fi

                  server_number="$(get_robot "ip?server_ip=${myip}" | jq -r .[].ip.server_number)"
                  server_name="${dns_name}.${CLOUDFLARE_DNS_ZONE}"

                  if [[ -n $server_number ]]; then
                      post_robot "server/${server_number}" "server_name=${server_name}"
                  fi
              fi
          fi

    environment:
      ENABLED: true
      CLOUDFLARE_DNS_ZONE: product-os.io
      ROBOT_API: https://robot-ws.your-server.de

  # https://github.com/balenablocks/cert-manager
  # https://certbot.eff.org/docs/using.html
  # https://certbot-dns-cloudflare.readthedocs.io/
  cert-manager:
    build: cert-manager
    restart: unless-stopped
    volumes:
      - cert-manager:/etc/letsencrypt
      - certs:/certs
    labels:
      io.balena.features.balena-api: 1
      io.balena.features.supervisor-api: 1

  # https://github.com/balena-io/logs-to-vector
  # This service handles log collection from the balenaEngine/journald logs (AKA: logshipper)
  # .. sends logs to log-aggregator in product-os namespace over AWS/VPC peering link
  logs-to-vector:
    # FIXME: switch to ghcr.io or implement registry-proxy tag listing for updates
    image: ghcr.io/balena-io/logs-to-vector:1.10.13
    restart: unless-stopped
    environment:
      LOG: warn
    labels:
      io.balena.features.journal-logs: 1
    volumes:
      - logs-to-vector:/var/lib/logs-to-vector
      - certs:/certs
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8686/health"]

  # https://github.com/balena-io/metrics-agent
  metrics-agent:
    image: ghcr.io/balena-io/metrics-agent:1.0.1
    volumes:
      - certs:/certs:ro
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:9273/metrics"]
      interval: 30s
      timeout: 10s
      retries: 3
    labels:
      io.resin.features.balena-socket: '1'
    privileged: true

  minio:
    image: minio/minio:RELEASE.2024-10-29T16-01-48Z
    command: server /data --console-address ""
    environment:
      MINIO_BROWSER: off
      MINIO_ROOT_USER: minioadmin
      # Secret key minimum length is 8 characters so this will force us to set
      # new value(s) via fleet variables.
      # Secret keys can be generated with:
      # openssl rand -base64 30 | tr '+/' 'xx' | cut -c1-40
      MINIO_ROOT_PASSWORD: x
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
      interval: 5s
      timeout: 5s
      retries: 5
    networks:
      - runner-network
      - minio-network
    volumes:
      - minio-data:/data
    tmpfs:
      - /tmp

  minio-init:
    build: minio-init
    restart: on-failure
    environment:
      MINIO_SERVER_URL: http://minio:9000
      # These must match the root user credentials on minio
      MINIO_ROOT_USER: minioadmin
      # Secret key minimum length is 8 characters so this will force us to set
      # new value(s) via fleet variables.
      # Secret keys can be generated with:
      # openssl rand -base64 30 | tr '+/' 'xx' | cut -c1-40
      MINIO_ROOT_PASSWORD: x
      ACTIONS_CACHE_SECRET_KEY: x
      YOCTO_CACHE_SECRET_KEY: x
      ACTIONS_CACHE_EXPIRE_DAYS: 7
      YOCTO_CACHE_EXPIRE_DAYS: 1
    networks:
      - minio-network
    depends_on:
      - minio
    tmpfs:
      - /tmp

volumes:
  cert-manager:
  certs:
  logs-to-vector:
  registry-data:
  minio-data:

networks:
  runner-network: {}
  minio-network: {}