docker-compose.yml

---

version: "3.9"

# == Re-useable Definitions ==================================================

# Default service definition for all (incl. postgres/redis/...)
x-service-default: &service_default
  networks:
    - varfish
  restart: unless-stopped

# Default service definition for services with access to static data.
#
# We provide the /data directory as read-only to all services as this
# simplifies the configuration considerably.
x-service-varfish-static-data-default: &service_static_data_default
  <<: *service_default
  volumes:
    - type: bind
      source: ${volumes_basedir:-./.dev/volumes}/varfish-static/data
      target: /data
      read_only: true

# Default service definition for services with the `varfish-server` image.
x-service-varfish-server-default: &service_varfish_server_default
  <<: *service_default
  image: "${image_base-ghcr.io/varfish-org}/\
    ${image_varfish_name:-varfish-server}:${image_varfish_version:-main}"
  env_file:
    .env
  networks:
    - varfish
  restart: unless-stopped
  secrets:
    - db-url
    - varfish-server-django-secret-key
    - minio-varfish-password
  depends_on:  # start after all others
    - annonars
    - mehari
    - minio
    - postgres
    - redis
    - traefik
    - viguno

# Default service definition for VarFish celery services.
x-service-varfish-celery-default: &service_varfish_celery_default
  <<: *service_varfish_server_default
  depends_on:
    - postgres
    - redis
    - varfish-web


# == Services ================================================================

services:
  # -- Traefik ----------------------------------------------------------------
  #
  # We use traefik as the reverse proxy for all public services.
  #
  # This file only contains the bare minimal configuraton.  We provide example
  # override files as `docker-compose.override.yml-traefik-{cert,le}` for
  # using custom certificates or letsencrypt.

  traefik:
    <<: *service_default
    container_name: traefik
    hostname: traefik
    image: ${image_traefik_name:-traefik}:${image_traefik_version:-2.10}
    # Expose the default HTTP and HTTPS ports.
    ports:
      - "80:80"
      - "443:443"
    command:
      # Enable Docker provider and disable "exposed by default".
      - "--providers.docker=true"
      - "--providers.docker.exposedbydefault=false"
      # Define the "websecure" entrypoint with port 443.
      - "--entrypoints.websecure.address=:443"
      # Define the "web" entrypoint with port 80 and configure automated
      # permantent redirection from web to websecure (HTTP to HTTPS).
      - "--entrypoints.web.http.redirections.entryPoint.to=websecure"
      - "--entrypoints.web.http.redirections.entryPoint.scheme=https"
      - "--entrypoints.web.http.redirections.entrypoint.permanent=true"
      - "--entrypoints.web.address=:80"
    volumes:
      # Mount Docker socket into container so traefik can react to events.
      - "/var/run/docker.sock:/var/run/docker.sock:ro"

  # -- VarFish ---------------------------------------------------------------
  #
  # VarFish-related services.

  varfish-web:
    <<: *service_varfish_server_default
    container_name: varfish-web
    hostname: varfish-web
    labels:
      - "traefik.enable=true"
      - "traefik.http.middlewares.xforward.headers.customrequestheaders.X-Forwarded-Proto=https"
      - "traefik.http.routers.varfish-web.entrypoints=web,websecure"
      - "traefik.http.routers.varfish-web.middlewares=xforward"
      - "traefik.http.routers.varfish-web.rule=HostRegexp(`{catchall:.+}`)"
      - "traefik.http.services.varfish-web.loadbalancer.server.port=8080"
      - "traefik.http.routers.varfish-web.tls=true"

  varfish-celerybeat:
    <<: *service_varfish_server_default
    container_name: varfish-celerybeat
    hostname: varfish-celerybeat
    command:
      - celerybeat

  varfish-celeryd-default:
    <<: *service_varfish_server_default
    container_name: varfish-celeryd-default
    hostname: varfish-celeryd-default
    command: ["celeryd"]
    environment: ["CELERY_QUEUES=default", "CELERY_WORKERS=16"]

  varfish-celeryd-query:
    <<: *service_varfish_server_default
    container_name: varfish-celeryd-query
    hostname: varfish-celeryd-query
    command: ["celeryd"]
    environment: ["CELERY_QUEUES=query", "CELERY_WORKERS=16"]
    volumes:
      # Read-only static data.
      - type: bind
        source: ${volumes_basedir:-./.dev/volumes}/varfish-static/data
        target: /data/varfish-static/data
        read_only: true
      # Writeable data directory.
      - type: bind
        source: ${volumes_basedir:-./.dev/volumes}/varfish-dynamic/data
        target: /data/varfish-dynamic/data
        read_only: false

  varfish-celeryd-export:
    <<: *service_varfish_server_default
    container_name: varfish-celeryd-export
    hostname: varfish-celeryd-export
    command: ["celeryd"]
    environment: ["CELERY_QUEUES=export", "CELERY_WORKERS=16"]

  varfish-celeryd-import:
    <<: *service_varfish_server_default
    container_name: varfish-celeryd-import
    hostname: varfish-celeryd-import
    command: ["celeryd"]
    environment: ["CELERY_QUEUES=import", "CELERY_WORKERS=4"]

  varfish-celeryd-maintenance:
    <<: *service_varfish_server_default
    container_name: varfish-celeryd-maintenance
    hostname: varfish-celeryd-maintenance
    command: ["celeryd"]
    environment: ["CELERY_QUEUES=maintenance", "CELERY_WORKERS=4"]

  # -- nginx -----------------------------------------------------------------
  #
  # We serve static files such as browser tracks with nginx.

  nginx:
    <<: *service_default
    container_name: nginx
    hostname: nginx
    image: ${image_nginx_name:-nginx}:${image_nginx_version:-1}
    volumes:
      - type: bind
        source: ${volumes_basedir:-./.dev/volumes}/varfish-static/data
        target: /data
        read_only: true
      - type: bind
        source: ${config_basedir:-./.dev/config}/nginx/nginx.conf
        target: /etc/nginx/nginx.conf

  # -- Mehari ----------------------------------------------------------------
  #
  # Mehari provides the transcript-related information.

  mehari:
    <<: *service_static_data_default
    container_name: mehari
    hostname: mehari
    image: "${image_base:-ghcr.io/varfish-org}/${image_mehari_name:-mehari}:\
      ${image_mehari_version:-latest}"

  # -- Viguno ----------------------------------------------------------------
  #
  # Viguno provides the disease/phenotype/gene relationships and related
  # information..

  viguno:
    <<: *service_static_data_default
    container_name: viguno
    hostname: viguno
    image: "${image_base:-ghcr.io/varfish-org}/${image_viguno_name:-viguno}:\
      ${image_viguno_version:-latest}"

  # -- Annonars ---------------------------------------------------------------
  #
  # Annonars provides the variant information but also the gene information.

  annonars:
    <<: *service_static_data_default
    container_name: annonars
    hostname: annonars
    image: "${image_base:-ghcr.io/varfish-org}/\
      ${image_annonars_name:-annonars}:${image_annonars_version:-latest}"

  # -- PostgreSQL Server -----------------------------------------------------
  #
  # We use the default configuration, but mount a volume for the data for
  # persistent storage.

  postgres:
    <<: *service_default
    container_name: postgres
    hostname: postgres
    image: ${image_postgres_name:-postgres}:${image_postgres_version:-12}
    environment:
      POSTGRES_USER: varfish
      POSTGRES_PASSWORD_FILE: /run/secrets/db-password
      POSTGRES_DB: varfish
    secrets:
      - db-password
    volumes:
      - type: bind
        source: ${volumes_basedir:-./.dev/volumes}/postgres/data
        target: /var/lib/postgresql/data

  # -- Redis -----------------------------------------------------------------
  #
  # We use the default configuration, but mount a volume for the data for
  # persistent storage.

  redis:
    <<: *service_default
    container_name: redis
    hostname: redis
    image: ${image_redis_name:-redis}:${image_redis_version:-6}
    volumes:
      - type: bind
        source: ${volumes_basedir:-./.dev/volumes}/redis/data
        target: /data

  # -- Minio (Server) --------------------------------------------------------
  #
  # This runs the Minio that is for the internal usage by the VarFish server
  # only!  It is not exposed to the outside world.
  #
  # We configure Minio in standalone mode.  Redundancy of storage must be
  # ensured using RAID or similar (ZFS/BTRFS).

  minio:
    <<: *service_default
    container_name: minio
    hostname: minio
    image: "${image_minio_name:-quay.io/minio/minio}:\
      ${image_minio_version:-latest}"
    command:
      - server
      - /data
      - --console-address
      - ":9001"
    environment:
      # The administrator credentials are "minioadmin" with the password as
      # configured in the secret "minio-root-password".
      MINIO_ROOT_USER: minioadmin
      MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio-root-password
    secrets:
      - minio-root-password
    # Uncomment the following two lines (reminder: "HOST:CONTAINER") to
    # enable access to the console from the host.
    # ports:
    #   - "9001:9001"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3
    volumes:
      - type: bind
        source: ${volumes_basedir:-./.dev/volumes}/minio/data
        target: /data

  # -- Minio (Client) --------------------------------------------------------
  #
  # We will only run "sleep" here and pre-configure the credentials with the
  # "minio" server.

  minio-client:
    <<: *service_default
    container_name: minio-client
    hostname: minio-client
    depends_on:
      - minio  # start after the server
    image: "${image_mc_name:-bitnami/minio-client}:\
      ${image_mc_version:-latest}"
    # Run with custom entrypoint that sets the alias "minio" with admin
    # credentials, ensures that the "varfish" user and "varfish-server"
    # buckets are created, and then sleeps forever.
    entrypoint: bash -i /opt/minio-utils/entrypoint-override.sh
    secrets:
      - minio-root-password
      - minio-varfish-password
    volumes:
      - type: bind
        source: ./utils/minio-utils
        target: /opt/minio-utils
        read_only: true

# == Secrets ================================================================

secrets:
  # The PostgreSQL database password.
  db-password:
    file: ${secrets_basedir:-./.dev/secrets}/db-password
  # The Django database URL.
  db-url:
    file: ${secrets_basedir:-./.dev/secrets}/db-url
  # The Django secret key.
  varfish-server-django-secret-key:
    file: ${secrets_basedir:-./.dev/secrets}/varfish-server-django-secret-key
  # The secrets for the root (=minioadmin) user on the MinIO server.
  minio-root-password:
    file: ${secrets_basedir:-./.dev/secrets}/minio-root-password
  # The secrets for the varfish user on the MinIO server.
  minio-varfish-password:
    file: ${secrets_basedir:-./.dev/secrets}/minio-varfish-password


# == Networks ================================================================

networks:
  # Explicitely configure the "varfish" network so we can control its name.
  varfish:
    driver_opts:
      com.docker.network.bridge.name: br-varfish