From 8b12cbf30ce33cff4484518b7499748780694081 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 18 Mar 2026 14:04:59 -0700 Subject: [PATCH 1/5] feat: update staging compose for demo/preview deployments Update docker-compose.staging.yml to serve as the standard config for staging, demo, and branch preview environments: - Remove local Postgres (DB is always external via DATABASE_IP) - Add RabbitMQ container for Celery task broker - Add NATS container (was present but commented out in depends_on) - Add restart:always to all services - Switch from .envs/.local/.postgres to .envs/.production/.postgres - Remove hardcoded container_name on NATS (allows multiple instances) - Remove awscli service (backups handled by TeamCity) - RabbitMQ credentials configured via .envs/.production/.django, not hardcoded in compose Add compose/staging/docker-compose.db.yml as an optional convenience for running a local PostgreSQL container when no external DB is available (e.g., ood environment, local testing). Co-Authored-By: Claude Opus 4.6 (1M context) --- compose/staging/docker-compose.db.yml | 37 ++++++++++++++ docker-compose.staging.yml | 74 ++++++++++++++------------- 2 files changed, 75 insertions(+), 36 deletions(-) create mode 100644 compose/staging/docker-compose.db.yml diff --git a/compose/staging/docker-compose.db.yml b/compose/staging/docker-compose.db.yml new file mode 100644 index 000000000..f4f1183f1 --- /dev/null +++ b/compose/staging/docker-compose.db.yml @@ -0,0 +1,37 @@ +# Optional local PostgreSQL for staging environments. +# +# Use this when you don't have an external database (e.g., for local testing +# or isolated branch previews). Creates a containerized PostgreSQL instance +# that the staging compose stack connects to via the Docker network. +# +# Usage: +# # Start the database first +# docker compose -f compose/staging/docker-compose.db.yml up -d +# +# # Then start the app (DATABASE_IP points to the host's Docker bridge) +# docker compose -f docker-compose.staging.yml --env-file .envs/.production/.compose up -d +# +# The app's .envs/.production/.postgres should use POSTGRES_HOST=db and the +# DATABASE_IP in .envs/.production/.compose should be set to the host IP +# of the machine running this database container (e.g., 172.17.0.1 for the +# default Docker bridge, or the host's LAN IP). +# +# For ood-style setups where the DB runs on the same host, set: +# DATABASE_IP=172.17.0.1 (or use host.docker.internal on Docker Desktop) + +volumes: + staging_postgres_data: {} + +services: + postgres: + build: + context: ../../ + dockerfile: ./compose/local/postgres/Dockerfile + volumes: + - staging_postgres_data:/var/lib/postgresql/data + - ../../data/db/snapshots:/backups + env_file: + - ../../.envs/.production/.postgres + ports: + - "5432:5432" + restart: always diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 684e50e67..45b094c88 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -1,79 +1,81 @@ -# Identical to production.yml, but with the following differences: -# Uses the django production settings file, but staging .env file. -# Uses "local" database +# Staging / demo / branch preview deployment. # -# 1. The database is a service in the Docker Compose configuration rather than external as in production. -# 2. Redis is a service in the Docker Compose configuration rather than external as in production. -# 3. Port 5001 is exposed for the Django application. - -volumes: - ami_local_postgres_data: {} +# Like production, but runs Redis, RabbitMQ, and NATS as local containers +# instead of requiring external infrastructure services. +# Database is always external — set DATABASE_IP in .envs/.production/.compose. +# +# Usage: +# docker compose -f docker-compose.staging.yml --env-file .envs/.production/.compose up -d +# +# For a local database, see compose/staging/docker-compose.db.yml. +# +# Required env files: +# .envs/.production/.compose — DATABASE_IP +# .envs/.production/.django — Django settings, CELERY_BROKER_URL, NATS_URL, etc. +# .envs/.production/.postgres — POSTGRES_HOST=db, POSTGRES_DB, POSTGRES_USER, POSTGRES_PASSWORD services: django: &django build: context: . - # This is the most important setting to test the production configuration of Django. dockerfile: ./compose/production/django/Dockerfile - image: insectai/ami_backend depends_on: - - postgres - redis - # - nats + - rabbitmq + - nats env_file: - ./.envs/.production/.django - - ./.envs/.local/.postgres + - ./.envs/.production/.postgres volumes: - ./config:/app/config ports: - "5001:5000" + extra_hosts: + - "db:${DATABASE_IP}" command: /start restart: always - postgres: - build: - context: . - # There is not a local/staging version of the Postgres Dockerfile. - dockerfile: ./compose/local/postgres/Dockerfile - # Share the local Postgres image with the staging configuration. - # Production uses an external Postgres service. - volumes: - - ami_local_postgres_data:/var/lib/postgresql/data - - ./data/db/snapshots:/backups - env_file: - - ./.envs/.local/.postgres - restart: always - - redis: - image: redis:6 - restart: always - celeryworker: <<: *django scale: 1 ports: [] command: /start-celeryworker + restart: always celerybeat: <<: *django ports: [] command: /start-celerybeat + restart: always flower: <<: *django ports: - "5550:5555" command: /start-flower + restart: always + volumes: + - ./data/flower/:/data/ + + redis: + image: redis:6 + restart: always + + rabbitmq: + image: rabbitmq:3.13-management-alpine + hostname: rabbitmq + ports: + - "15672:15672" + restart: always nats: image: nats:2.10-alpine - container_name: ami_local_nats hostname: nats ports: - - "4222:4222" # Client port - - "8222:8222" # HTTP monitoring port - command: ["-js", "-m", "8222"] # Enable JetStream and monitoring + - "4222:4222" + - "8222:8222" + command: ["-js", "-m", "8222"] healthcheck: test: ["CMD", "wget", "--spider", "-q", "http://localhost:8222/healthz"] interval: 10s From b3d97716d06ac1ce16ce792c7dd2dbaf382c88e4 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 18 Mar 2026 14:50:00 -0700 Subject: [PATCH 2/5] =?UTF-8?q?fix:=20upgrade=20gunicorn=2020.1.0=20?= =?UTF-8?q?=E2=86=92=2023.0.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gunicorn 20.x requires pkg_resources from setuptools, which was removed in setuptools 82+. Fresh Docker image builds fail with ModuleNotFoundError on startup. gunicorn 23 drops the pkg_resources dependency entirely. Closes #1180 Co-Authored-By: Claude Opus 4.6 (1M context) --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 3b208e9df..037eeea17 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -98,5 +98,5 @@ pytest-django==4.5.2 # https://github.com/pytest-dev/pytest-django # ------------------------------------------------------------------------------ newrelic==9.6.0 -gunicorn==20.1.0 # https://github.com/benoitc/gunicorn +gunicorn==23.0.0 # https://github.com/benoitc/gunicorn # psycopg[c]==3.1.9 # https://github.com/psycopg/psycopg From 1bb26462c46e2af948d7e47db0a66a573b39c125 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 18 Mar 2026 15:09:30 -0700 Subject: [PATCH 3/5] fix: address PR review feedback - Add env_file to rabbitmq service so it picks up RABBITMQ_DEFAULT_USER/RABBITMQ_DEFAULT_PASS from .django env - Use ${DATABASE_IP:?} required-variable syntax for fail-fast on missing config - Bind local Postgres to 127.0.0.1 instead of 0.0.0.0 - Clarify DB compose comments: document host-bridge connectivity via DATABASE_IP, remove ambiguous "Docker network" wording Co-Authored-By: Claude Opus 4.6 (1M context) --- compose/staging/docker-compose.db.yml | 22 ++++++++++++---------- docker-compose.staging.yml | 4 +++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/compose/staging/docker-compose.db.yml b/compose/staging/docker-compose.db.yml index f4f1183f1..7ba5abbb3 100644 --- a/compose/staging/docker-compose.db.yml +++ b/compose/staging/docker-compose.db.yml @@ -1,23 +1,25 @@ # Optional local PostgreSQL for staging environments. # # Use this when you don't have an external database (e.g., for local testing -# or isolated branch previews). Creates a containerized PostgreSQL instance -# that the staging compose stack connects to via the Docker network. +# or isolated branch previews). Publishes PostgreSQL on localhost:5432. # # Usage: # # Start the database first # docker compose -f compose/staging/docker-compose.db.yml up -d # -# # Then start the app (DATABASE_IP points to the host's Docker bridge) +# # Then start the app stack # docker compose -f docker-compose.staging.yml --env-file .envs/.production/.compose up -d # -# The app's .envs/.production/.postgres should use POSTGRES_HOST=db and the -# DATABASE_IP in .envs/.production/.compose should be set to the host IP -# of the machine running this database container (e.g., 172.17.0.1 for the -# default Docker bridge, or the host's LAN IP). +# The app connects to the database via extra_hosts (db → DATABASE_IP). +# Set DATABASE_IP to the Docker bridge gateway so the app container can +# reach the host-published port: # -# For ood-style setups where the DB runs on the same host, set: -# DATABASE_IP=172.17.0.1 (or use host.docker.internal on Docker Desktop) +# .envs/.production/.compose: +# DATABASE_IP=172.17.0.1 # Linux (default Docker bridge gateway) +# DATABASE_IP=host-gateway # Docker Desktop (macOS/Windows) +# +# .envs/.production/.postgres: +# POSTGRES_HOST=db # resolves via extra_hosts to DATABASE_IP volumes: staging_postgres_data: {} @@ -33,5 +35,5 @@ services: env_file: - ../../.envs/.production/.postgres ports: - - "5432:5432" + - "127.0.0.1:5432:5432" restart: always diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 45b094c88..052f63d6d 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -32,7 +32,7 @@ services: ports: - "5001:5000" extra_hosts: - - "db:${DATABASE_IP}" + - "db:${DATABASE_IP:?Set DATABASE_IP in .envs/.production/.compose}" command: /start restart: always @@ -65,6 +65,8 @@ services: rabbitmq: image: rabbitmq:3.13-management-alpine hostname: rabbitmq + env_file: + - ./.envs/.production/.django ports: - "15672:15672" restart: always From d5e8f6f1c7128a89512427aa66aa34965984116a Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 18 Mar 2026 15:11:34 -0700 Subject: [PATCH 4/5] fix: remove host port bindings from internal services MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Internal services (Redis, RabbitMQ, NATS) don't need host port exposure — only the app containers talk to them via the Docker network. Removing host ports means multiple instances (branch previews, worktrees) never conflict on these ports. Django and Flower ports are now configurable via DJANGO_PORT and FLOWER_PORT env vars (default 5001 and 5550). Also use host-gateway (works on all platforms) instead of platform-specific Docker bridge IPs in DB compose docs. Co-Authored-By: Claude Opus 4.6 (1M context) --- compose/staging/docker-compose.db.yml | 3 +-- docker-compose.staging.yml | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/compose/staging/docker-compose.db.yml b/compose/staging/docker-compose.db.yml index 7ba5abbb3..1e94bcfc0 100644 --- a/compose/staging/docker-compose.db.yml +++ b/compose/staging/docker-compose.db.yml @@ -15,8 +15,7 @@ # reach the host-published port: # # .envs/.production/.compose: -# DATABASE_IP=172.17.0.1 # Linux (default Docker bridge gateway) -# DATABASE_IP=host-gateway # Docker Desktop (macOS/Windows) +# DATABASE_IP=host-gateway # Recommended (resolves to host on all platforms) # # .envs/.production/.postgres: # POSTGRES_HOST=db # resolves via extra_hosts to DATABASE_IP diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 052f63d6d..ba4935498 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -9,6 +9,18 @@ # # For a local database, see compose/staging/docker-compose.db.yml. # +# Multiple instances: This compose file can run multiple instances on the same +# host (e.g., branch previews, worktrees) by setting a unique project name and +# overriding the published ports: +# +# DJANGO_PORT=5002 FLOWER_PORT=5551 \ +# docker compose -p my-preview -f docker-compose.staging.yml \ +# --env-file .envs/.production/.compose up -d +# +# Internal services (Redis, RabbitMQ, NATS) do not publish host ports, so they +# never conflict between instances. Each compose project gets its own isolated +# Docker network. +# # Required env files: # .envs/.production/.compose — DATABASE_IP # .envs/.production/.django — Django settings, CELERY_BROKER_URL, NATS_URL, etc. @@ -30,7 +42,7 @@ services: volumes: - ./config:/app/config ports: - - "5001:5000" + - "${DJANGO_PORT:-5001}:5000" extra_hosts: - "db:${DATABASE_IP:?Set DATABASE_IP in .envs/.production/.compose}" command: /start @@ -52,7 +64,7 @@ services: flower: <<: *django ports: - - "5550:5555" + - "${FLOWER_PORT:-5550}:5555" command: /start-flower restart: always volumes: @@ -67,16 +79,11 @@ services: hostname: rabbitmq env_file: - ./.envs/.production/.django - ports: - - "15672:15672" restart: always nats: image: nats:2.10-alpine hostname: nats - ports: - - "4222:4222" - - "8222:8222" command: ["-js", "-m", "8222"] healthcheck: test: ["CMD", "wget", "--spider", "-q", "http://localhost:8222/healthz"] From 7f75f9826c39ee1c4ce013db8623f4c77a541388 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 18 Mar 2026 15:13:11 -0700 Subject: [PATCH 5/5] docs: add staging deployment guide Setup instructions for single and multi-instance staging deployments, covering environment configuration, database options, migrations, sample data, and port management for running multiple instances on the same host. Co-Authored-By: Claude Opus 4.6 (1M context) --- compose/staging/README.md | 170 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 compose/staging/README.md diff --git a/compose/staging/README.md b/compose/staging/README.md new file mode 100644 index 000000000..473639696 --- /dev/null +++ b/compose/staging/README.md @@ -0,0 +1,170 @@ +# Staging Deployment + +Deploy the Antenna platform with local Redis, RabbitMQ, and NATS containers. +The database is always external — either a dedicated server, a managed service, +or the optional local Postgres container included here. + +## Quick Start (single instance) + +### 1. Configure environment files + +Copy the examples and fill in the values: + +```bash +# Django settings +cp .envs/.production/.django-example .envs/.production/.django + +# Database credentials +cat > .envs/.production/.postgres << 'EOF' +POSTGRES_HOST=db +POSTGRES_PORT=5432 +POSTGRES_DB=antenna_staging +POSTGRES_USER=antenna +POSTGRES_PASSWORD= +EOF + +# Database host IP +cat > .envs/.production/.compose << 'EOF' +DATABASE_IP=host-gateway +EOF +``` + +Key settings to configure in `.envs/.production/.django`: + +| Variable | Example | Notes | +|---|---|---| +| `DJANGO_SECRET_KEY` | `` | Generate with `python -c "from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())"` | +| `DJANGO_ALLOWED_HOSTS` | `*` or `api.staging.example.com` | | +| `REDIS_URL` | `redis://redis:6379/0` | Always use `redis` hostname (local container) | +| `CELERY_BROKER_URL` | `amqp://antenna:password@rabbitmq:5672/` | Always use `rabbitmq` hostname | +| `RABBITMQ_DEFAULT_USER` | `antenna` | Must match the user in `CELERY_BROKER_URL` | +| `RABBITMQ_DEFAULT_PASS` | `` | Must match the password in `CELERY_BROKER_URL` | +| `NATS_URL` | `nats://nats:4222` | Always use `nats` hostname | +| `CELERY_FLOWER_USER` | `flower` | Basic auth for the Flower web UI | +| `CELERY_FLOWER_PASSWORD` | `` | | +| `SENDGRID_API_KEY` | `placeholder` | Set a real key to enable email, or any non-empty string to skip | +| `DJANGO_AWS_STORAGE_BUCKET_NAME` | `my-bucket` | S3-compatible object storage for media/static files | +| `DJANGO_SUPERUSER_EMAIL` | `admin@example.com` | Used by `create_demo_project` command | +| `DJANGO_SUPERUSER_PASSWORD` | `` | Used by `create_demo_project` command | + +### 2. Start the database + +If you have an external database, set `DATABASE_IP` in `.envs/.production/.compose` +to its IP address and skip this step. + +For a local database container: + +```bash +docker compose -f compose/staging/docker-compose.db.yml up -d + +# Set DATABASE_IP to reach the host-published port from app containers +echo "DATABASE_IP=host-gateway" > .envs/.production/.compose +``` + +Verify the database is ready: + +```bash +docker compose -f compose/staging/docker-compose.db.yml logs +# Should show: "database system is ready to accept connections" +``` + +### 3. Build and start the app + +```bash +docker compose -f docker-compose.staging.yml \ + --env-file .envs/.production/.compose build django + +docker compose -f docker-compose.staging.yml \ + --env-file .envs/.production/.compose up -d +``` + +### 4. Run migrations and create an admin user + +```bash +# Shorthand for the compose command +COMPOSE="docker compose -f docker-compose.staging.yml --env-file .envs/.production/.compose" + +# Apply database migrations +$COMPOSE run --rm django python manage.py migrate + +# Create demo project with sample data and admin user +$COMPOSE run --rm django python manage.py create_demo_project + +# Or just create an admin user without sample data +$COMPOSE run --rm django python manage.py createsuperuser --noinput +``` + +### 5. Verify + +```bash +# API root +curl http://localhost:5001/api/v2/ + +# Django admin +# Open http://localhost:5001/admin/ in a browser + +# Flower (Celery monitoring) +# Open http://localhost:5550/ in a browser + +# NATS health (internal, but reachable via docker exec) +docker compose -f docker-compose.staging.yml \ + --env-file .envs/.production/.compose \ + exec nats wget -qO- http://localhost:8222/healthz +``` + +## Multiple Instances on the Same Host + +Internal services (Redis, RabbitMQ, NATS) don't publish host ports, so they +never conflict between instances. Each compose project gets its own isolated +Docker network. + +Only Django and Flower publish host ports. Override them with environment +variables and use a unique project name (`-p`): + +```bash +# Instance A (defaults: Django on 5001, Flower on 5550) +docker compose -p antenna-main \ + -f docker-compose.staging.yml \ + --env-file .envs/.production/.compose up -d + +# Instance B (custom ports) +DJANGO_PORT=5002 FLOWER_PORT=5551 \ + docker compose -p antenna-feature-xyz \ + -f docker-compose.staging.yml \ + --env-file path/to/other/.compose up -d +``` + +Each instance needs its own: +- `.envs/.production/.compose` (can share `DATABASE_IP` if using the same DB server) +- `.envs/.production/.postgres` (use a different `POSTGRES_DB` per instance) +- `.envs/.production/.django` (can share most settings, but use unique `DJANGO_SECRET_KEY`) + +If using the local database container, each instance needs its own DB container +too (or share one by creating multiple databases in it). + +## Stopping and Cleaning Up + +```bash +# Stop the app stack +docker compose -f docker-compose.staging.yml \ + --env-file .envs/.production/.compose down + +# Stop the local database (data is preserved in a Docker volume) +docker compose -f compose/staging/docker-compose.db.yml down + +# Remove everything including database data +docker compose -f compose/staging/docker-compose.db.yml down -v +``` + +## Database Options + +The staging compose supports any PostgreSQL database reachable by IP: + +| Option | `DATABASE_IP` | Notes | +|---|---|---| +| Local container | `host-gateway` | Use `compose/staging/docker-compose.db.yml` | +| Dedicated VM | `` | Best performance for shared environments | +| Managed service | `` | Cloud-hosted PostgreSQL | + +Set `POSTGRES_HOST=db` in `.envs/.production/.postgres` — the `extra_hosts` +directive in the compose file maps `db` to whatever `DATABASE_IP` resolves to.