Break docker-compose.yml into smaller compose.yml files (#4114)

WordPress · Apr 16, 2024 · 82282a0 · 82282a0
1 parent c617c85
commit 82282a0
Show file tree

Hide file tree

Showing 9 changed files with 270 additions and 345 deletions.
diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml
@@ -366,7 +366,9 @@ jobs:
           setup_images: upstream_db ingestion_server
 
       - name: Run ingestion-server tests
-        run: just ingestion_server/test-local
+        run: |
+          just env
+          just ingestion_server/test-local
 
       - name: Print ingestion-server test logs
         run: |

diff --git a/api/compose.yml b/api/compose.yml
@@ -0,0 +1,60 @@
+services:
+  cache:
+    profiles:
+      - api
+    image: docker.io/redis:7.2.4
+    ports:
+      - "50263:6379"
+
+  web:
+    profiles:
+      - api
+    build:
+      target: api
+      args: # Automatically inferred from env vars, unless specified
+        - SEMANTIC_VERSION=${SEMANTIC_VERSION:-v1.0.0}
+        - API_PY_VERSION
+    image: openverse-api
+    volumes:
+      - .:/api:z
+    ports:
+      - "50280:50280"
+    depends_on:
+      - db
+      - es
+      - cache
+    env_file:
+      - env.docker
+      - .env
+    stdin_open: true
+    tty: true
+
+  nginx:
+    profiles:
+      - api
+    build:
+      target: nginx
+      args: # Automatically inferred from env vars, unless specified
+        - SEMANTIC_VERSION=${SEMANTIC_VERSION:-v1.0.0}
+        - API_PY_VERSION
+    ports:
+      - "50270:8080"
+    environment:
+      DJANGO_NGINX_UPSTREAM_URL: web:50280
+    depends_on:
+      - web
+
+  proxy:
+    profiles:
+      - api
+    image: docker.io/nginx:alpine
+    ports:
+      - "50200:9080"
+      - "50243:9443"
+    environment:
+      HTTPS_PORT: 50243 # See `ports` mapping above.
+    depends_on:
+      - web
+    volumes:
+      - ../docker/nginx/templates:/etc/nginx/templates:z
+      - ../docker/nginx/certs:/etc/nginx/certs:z
diff --git a/catalog/compose.yml b/catalog/compose.yml
@@ -0,0 +1,106 @@
+# Common build configuration for Airflow
+# Extension field, see https://docs.docker.com/compose/compose-file/compose-file-v3/#extension-fields
+x-airflow-common: &airflow-common
+  profiles:
+    - catalog
+  restart: on-failure
+  depends_on:
+    - postgres
+    - s3
+  image: openverse_catalog
+  env_file:
+    - .env
+  build:
+    target: cat
+    args: # Automatically inferred from env vars, unless specified
+      - REQUIREMENTS_FILE=requirements-dev.txt
+      - CATALOG_PY_VERSION
+      - CATALOG_AIRFLOW_VERSION
+  volumes:
+    - .:/opt/airflow/catalog:z
+    - catalog-cache:/home/airflow/.cache
+
+services:
+  # Dev changes for the scheduler
+  scheduler:
+    <<: *airflow-common
+    depends_on:
+      - upstream_db
+      - s3
+    command: scheduler
+    expose:
+      - "8793" # Used for fetching logs
+    environment:
+      # Upgrade the DB on startup
+      _AIRFLOW_DB_MIGRATE: "true"
+      _AIRFLOW_WWW_USER_CREATE: "true"
+      _AIRFLOW_WWW_USER_USERNAME: airflow
+      _AIRFLOW_WWW_USER_PASSWORD: airflow
+      _AIRFLOW_WWW_USER_FIRSTNAME: Air
+      _AIRFLOW_WWW_USER_LASTNAME: Flow
+      _AIRFLOW_WWW_USER_EMAIL: airflow@example.com
+
+  # Dev changes for the triggerer
+  triggerer:
+    <<: *airflow-common
+    depends_on:
+      - upstream_db
+      - s3
+    expose:
+      - "8794" # Used for logs
+    command: triggerer
+
+  # Dev changes for the webserver container
+  webserver:
+    <<: *airflow-common
+    depends_on:
+      - upstream_db
+      - s3
+      - scheduler
+      - triggerer
+    command: webserver
+    ports:
+      - "${AIRFLOW_PORT}:8080"
+
+  s3:
+    profiles:
+      - catalog_dependencies
+      - catalog
+    image: docker.io/minio/minio:latest
+    ports:
+      - "5010:5000"
+      - "5011:5001"
+    env_file:
+      - .env
+      - ../docker/minio/.env
+    command: minio server /data --address :5000 --console-address :5001
+    volumes:
+      - minio:/data
+      - ../docker/minio/s3_entrypoint.sh:/opt/minio/s3_entrypoint.sh:ro,z
+    entrypoint: /opt/minio/s3_entrypoint.sh
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:5010/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  load_to_s3:
+    profiles:
+      - catalog_dependencies
+      - catalog
+    image: docker.io/minio/mc:latest
+    env_file:
+      - .env
+      - ../docker/minio/.env
+    depends_on:
+      - s3
+    volumes:
+      # Buckets for testing provider data imported from s3 are subdirectories under
+      # /tests/s3-data/
+      - ./tests/s3-data:/data:rw,z
+      - ../docker/minio/load_to_s3_entrypoint.sh:/opt/minio/load_to_s3_entrypoint.sh:ro,z
+    entrypoint: /opt/minio/load_to_s3_entrypoint.sh
+
+volumes:
+  catalog-cache:
+  minio: