dev: add community compose (#2326)

* dev: add community compose
ansible · Oct 24, 2024 · c18aff0 · c18aff0
1 parent b530dd2
commit c18aff0
Show file tree

Hide file tree

Showing 2 changed files with 322 additions and 5 deletions.
diff --git a/dev/compose/README.md b/dev/compose/README.md
@@ -4,7 +4,7 @@ Profiles:
 
 - `aap` - Run galaxy_ng for integration with Ansible Automation Platform and Resource Server
 - `community` - Run galaxy_ng for galaxy.ansible.com development
-- `cloud` - Run galaxy_ng for console.redhat.com development
+- `insights` - Run galaxy_ng for console.redhat.com development
 
 ## Requirements
 
@@ -76,6 +76,8 @@ $ docker compose -f dev/compose/aap.yaml down
 # add -v to stop and remove volumes
 ```
 
+> [!TIP]
+> Stop with Ctrl + C if running without `-d` and then execute the `down` command.
 
 ## API Access
 
@@ -87,8 +89,10 @@ AAP UI and API will be available only if started separately on:
 
 [https://localhost](https://localhost)
 
-Ansible Hub UI can be started separately as a standalone `npm` run.
 
+## Running UI for community development
+
+Ansible Hub UI can be started separately as a standalone `npm` run.
 
 ```console
 # Assuming galaxy_ng is running on community compose.
@@ -110,7 +114,7 @@ On the second terminal:
 ```console
 cd ansible-hub-ui
 npm install
-API_PROXY_PORT=5001 API_BASE_PATH=/api/galaxy/ npm run start-standalone
+npm run start-community
 ```
 
 UI will be available on http://localhost:8002 and API on http://localhost:5001
@@ -153,8 +157,10 @@ Optionally it can be informed in a single line:
 $ DEV_SOURCE_PATH="dynaconf:pulp_ansible:galaxy_ng" docker compose -f dev/compose/app.yaml up --build
 ```
 
-> **NOTE** if passing on the call line, remember to repass the same variable every time you interact with
->`docker compose` for exec, logs, down etc.
+> [!NOTE]
+> if passing on the call line, remember to repass the same variable every time you interact with
+>`docker compose` using the `run` command, usually `exec,logs,stats` doesn't require, but commands
+> that starts the service container from scratch needs the variables.
 
 Now when changes are detected on `.py` and `.yaml` files on any of the `DEV_SOURCE_PATH`
 directories it will trigger reload of `api`, `worker`, and `content` services.
@@ -185,6 +191,18 @@ Solution 2:
 - Ensure `LOCK_REQUIREMENTS` is set to `0`
 - Ensure all your local checkouts are checked out to compatible branches
 
+### LLB definition error
+
+```bash
+failed to solve: rpc error: code = Unknown desc = failed to solve with frontend dockerfile.v0: failed to create LLB definition: failed to do request: Head "http://localhost/v2/galaxy_ng/galaxy_ng/manifests/base": dial tcp [::1]:80: connect: connection refused
+```
+
+Solution
+
+```bash
+export DOCKER_BUILDKIT=0
+export COMPOSE_DOCKER_CLI_BUILD=0
+```
 
 ## Tips and Tricks.
 

diff --git a/dev/compose/community.yaml b/dev/compose/community.yaml
@@ -0,0 +1,299 @@
+x-common-env: &common-env
+
+  GNUPGHOME: /root/.gnupg/
+  GALAXY_IMPORTER_CONFIG: /src/galaxy_ng/profiles/community/galaxy-importer/galaxy-importer.cfg
+
+  DJANGO_SUPERUSER_USERNAME: admin
+  DJANGO_SUPERUSER_EMAIL: admin@example.com
+  DJANGO_SUPERUSER_PASSWORD: admin
+
+  POSTGRES_USER: galaxy_ng
+  POSTGRES_PASSWORD: galaxy_ng
+  POSTGRES_DB: galaxy_ng
+
+  # no spying
+  PULP_ANALYTICS: 'false'
+
+  # normally goes into settings.py ...
+  PULP_DATABASES__default__ENGINE: django.db.backends.postgresql
+  PULP_DATABASES__default__NAME: galaxy_ng
+  PULP_DATABASES__default__USER: galaxy_ng
+  PULP_DATABASES__default__PASSWORD: galaxy_ng
+  PULP_DATABASES__default__HOST: postgres
+  PULP_DATABASES__default__PORT: 5432
+
+  PULP_DEBUG: 1
+  PULP_GALAXY_DEPLOYMENT_MODE: 'standalone'
+  PULP_DEFAULT_FILE_STORAGE: "pulpcore.app.models.storage.FileSystem"
+  PULP_REDIRECT_TO_OBJECT_STORAGE: 'false'
+
+  # Hostname and prefix has to be correct
+  PULP_GALAXY_API_PATH_PREFIX: '/api/'
+  PULP_CONTENT_PATH_PREFIX: '/pulp/content/'
+  PULP_ANSIBLE_API_HOSTNAME: 'https://localhost'
+  PULP_ANSIBLE_CONTENT_HOSTNAME: "https://localhost"
+  PULP_CONTENT_ORIGIN: "https://localhost"
+  PULP_CSRF_TRUSTED_ORIGINS: "['https://localhost']"
+
+  # auth ...
+  PULP_GALAXY_AUTHENTICATION_CLASSES: "['galaxy_ng.app.auth.session.SessionAuthentication', 'rest_framework.authentication.TokenAuthentication', 'rest_framework.authentication.BasicAuthentication', 'django.contrib.auth.backends.ModelBackend']"
+  PULP_GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_ACCESS: 'true'
+  PULP_GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_DOWNLOAD: 'true'
+
+  PULP_SOCIAL_AUTH_LOGIN_REDIRECT_URL: "/"
+  SOCIAL_AUTH_GITHUB_BASE_URL: 'http://localhost:8082'
+  SOCIAL_AUTH_GITHUB_API_URL: 'http://localhost:8082'
+  SOCIAL_AUTH_GITHUB_KEY: 'abcd1234'
+  SOCIAL_AUTH_GITHUB_SECRET: 'abcd1234'
+
+  PULP_GALAXY_ENABLE_LEGACY_ROLES: 'true'
+  PULP_GALAXY_FEATURE_FLAGS__execution_environments: 'false'
+  PULP_GALAXY_FEATURE_FLAGS__ai_deny_index: 'true'
+  PULP_GALAXY_FEATURE_FLAGS__display_repositories: 'false'
+  PULP_GALAXY_FEATURE_FLAGS__dab_resource_registry: 'false'
+  PULP_GALAXY_FEATURE_FLAGS__external_authentication: 'true'
+
+  PULP_GALAXY_CONTAINER_SIGNING_SERVICE: '@none None'
+  PULP_GALAXY_COLLECTION_SIGNING_SERVICE: '@none None'
+  PULP_GALAXY_REQUIRE_CONTENT_APPROVAL: 'false'
+  PULP_GALAXY_AUTO_SIGN_COLLECTIONS: 'false'
+
+  # disable user/group modifications
+  PULP_ALLOW_LOCAL_RESOURCE_MANAGEMENT: 'false'
+
+  # role content workaround
+  PULP_ANSIBLE_BASE_ROLES_REQUIRE_VIEW: 'false'
+
+  # Resource server
+  # This disables the attempt for resource syncing
+  PULP_RESOURCE_SERVER_SYNC_ENABLED: 'false'
+  # Set above to 'true' if/when RESOURCE_SERVER is configured
+  # The next variables must be configured to enable resource sync
+  # PULP_RESOURCE_SERVER__URL='https://localhost'
+  # PULP_RESOURCE_SERVER__SECRET_KEY='?'
+  # PULP_RESOURCE_SERVER__VALIDATE_HTTPS='false'
+
+  # Integration test settings
+  HUB_TEST_AUTHENTICATION_BACKEND: "community"
+  HUB_TEST_MARKS: "deployment_community"
+
+  # Unpin dependencies on setup.py if set to 0
+  LOCK_REQUIREMENTS: 0
+
+  # DEV EDITABLE STUFF
+  # To enable editable to install local checkouts set DEV_SOURCE_PATH keeping the ordering as follows:
+  # "dynaconf:pulpcore:galaxy_importer:pulp_ansible:pulp_container:galaxy_ng:django-ansible-base"
+  # This can be done as part of the `docker compose` call:
+  # $ DEV_SOURCE_PATH="pulp_container:galaxy_ng" docker compose -f dev/compose/aap.yaml up
+  DEV_SOURCE_PATH:
+
+
+services:
+  base_img:
+    build:
+      context: ../../
+      dockerfile: Dockerfile
+    image: "localhost/galaxy_ng/galaxy_ng:base"
+
+  base_img_dev:  # Extends base_img with extra files and dev tools
+    depends_on:
+      - base_img
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+      args:
+        <<: *common-env
+    image: "localhost/galaxy_ng/galaxy_ng:dev"
+
+  github:
+    build:
+      context: '../../profiles/community/github_mock'
+    ports:
+      - "8082:8082"
+    environment:
+      UPSTREAM_PROTO: 'http'
+      UPSTREAM_HOST: api
+      UPSTREAM_PORT: 8000
+    volumes:
+      - '../../profiles/community/github_mock:/app:ro'
+
+  redis:
+    image: "redis:5"
+
+  postgres:
+    image: "postgres:13"
+    ports:
+      - '5433:5432'
+    environment:
+      <<: *common-env
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "galaxy_ng"]
+      interval: 10s
+      retries: 5
+    # Uncomment below to spam out every DB statement to the service stderr
+    # WARNING: enabling log_statement=all makes database slower
+    # command: ["postgres", "-c", "log_statement=ddl", "-c", "log_destination=stderr"]
+
+  migrations:
+    image: "localhost/galaxy_ng/galaxy_ng:dev"
+    depends_on:
+      - base_img_dev
+      - postgres
+    volumes:
+      - "etc_pulp_certs:/etc/pulp/certs"
+      - "var_lib_pulp:/var/lib/pulp"
+      - "../../../:/src"
+      - "../../:/app"
+    environment:
+      <<: *common-env
+    user: root
+    command: |
+      bash -c "
+        set -e;
+        rm -rf /var/lib/pulp/.migrated;
+        /src/galaxy_ng/dev/compose/bin/devinstall;
+
+        pulpcore-manager check --database default;
+        pulpcore-manager migrate;
+        pulpcore-manager shell < /src/galaxy_ng/dev/common/setup_test_data.py;
+        pulpcore-manager createsuperuser --noinput || true;
+
+        touch /var/lib/pulp/.migrated;
+      "
+
+  api:
+    image: "localhost/galaxy_ng/galaxy_ng:dev"
+    depends_on:
+      - base_img_dev
+      - postgres
+      - migrations
+    volumes:
+      - "etc_pulp_certs:/etc/pulp/certs"
+      - "var_lib_pulp:/var/lib/pulp"
+      - "../../../:/src"
+      - "../../:/app"
+    environment:
+      <<: *common-env
+    extra_hosts:
+      localhost: "host-gateway"
+    networks:
+      - default
+      - service-mesh
+    user: root
+    command: |
+      bash -c "
+        /src/galaxy_ng/dev/compose/bin/wait /var/lib/pulp/.migrated;
+        /src/galaxy_ng/dev/compose/bin/devinstall;
+        /src/galaxy_ng/dev/compose/bin/reloader /venv/bin/pulpcore-api
+      "
+
+  content:
+    image: "localhost/galaxy_ng/galaxy_ng:dev"
+    depends_on:
+      - base_img_dev
+      - postgres
+      - migrations
+    volumes:
+      - "etc_pulp_certs:/etc/pulp/certs"
+      - "var_lib_pulp:/var/lib/pulp"
+      - "../../../:/src"
+      - "../../:/app"
+    environment:
+      <<: *common-env
+    extra_hosts:
+      localhost: "host-gateway"
+    networks:
+      - default
+      - service-mesh
+    user: root
+    command: |
+      bash -c "
+        /src/galaxy_ng/dev/compose/bin/wait /var/lib/pulp/.migrated;
+        /src/galaxy_ng/dev/compose/bin/devinstall;
+        /src/galaxy_ng/dev/compose/bin/reloader /venv/bin/pulpcore-content
+      "
+
+  worker:
+    image: "localhost/galaxy_ng/galaxy_ng:dev"
+    depends_on:
+      - base_img_dev
+      - postgres
+      - migrations
+    volumes:
+      - "etc_pulp_certs:/etc/pulp/certs"
+      - "var_lib_pulp:/var/lib/pulp"
+      - "../../../:/src"
+      - "../../:/app"
+    environment:
+      <<: *common-env
+    user: root
+    command: |
+      bash -c "
+        /src/galaxy_ng/dev/compose/bin/wait /var/lib/pulp/.migrated;
+        /src/galaxy_ng/dev/compose/bin/devinstall;
+        /src/galaxy_ng/dev/compose/bin/reloader /venv/bin/pulpcore-worker
+      "
+
+  manager:
+    image: "localhost/galaxy_ng/galaxy_ng:dev"
+    depends_on:
+      - base_img_dev
+      - postgres
+      - migrations
+      - worker
+    volumes:
+      - "etc_pulp_certs:/etc/pulp/certs"
+      - "var_lib_pulp:/var/lib/pulp"
+      - "../../../:/src"
+      - "../../:/app"
+    environment:
+      <<: *common-env
+    user: root
+    command: |
+      bash -c "
+        /src/galaxy_ng/dev/compose/bin/wait /var/lib/pulp/.migrated;
+        /src/galaxy_ng/dev/compose/bin/devinstall;
+
+        # Give some time for API to start;
+        sleep 5;
+
+        echo ' ';
+        echo '###################### API ROOT ##############################';
+        curl -s http://api:24817/api/ | python -m json.tool;
+        echo '################### DEV_SOURCE_PATH ##########################';
+        echo $$DEV_SOURCE_PATH;
+        echo ' ';
+        echo '######################## READY ###############################';
+        echo ' ';
+        echo 'Credentials:  ' $$DJANGO_SUPERUSER_USERNAME:$$DJANGO_SUPERUSER_PASSWORD;
+        echo 'API Spec:      http://localhost:5001/api/v3/swagger-ui/';
+        echo 'Django Admin:  docker compose -f dev/compose/aap.yaml exec manager pulpcore-manager';
+        echo 'Settings list: docker compose -f dev/compose/aap.yaml exec manager dynaconf list';
+        echo 'Docs:          https://github.com/ansible/galaxy_ng/blob/master/dev/compose/README.md';
+        echo '##############################################################';
+
+        # Keep it running indefinitely to enable `docker compose -f ... exec manager /bin/bash`;
+        tail -f /dev/null
+      "
+
+  nginx:
+    image: "nginx:latest"
+    depends_on:
+      - postgres
+      - migrations
+      - api
+      - content
+    ports:
+      - '5001:5001'
+    volumes:
+      - '../nginx/nginx.conf:/etc/nginx/nginx.conf:ro'
+
+volumes:
+  var_lib_pulp:
+    name: var_lib_pulp
+  etc_pulp_certs:
+    name: etc_pulp_certs
+
+networks:
+  service-mesh:
+    name: service-mesh