Community tests in docker compose CI

.github/workflows/ci-docker-compose-integration.yml

@@ -0,0 +1,74 @@
+---
+name: Docker Compose Integration
+on:
+  pull_request:
+    branches:
+      - '**'
+#  push:
+#    branches:
+#      - '**'
+  workflow_dispatch:
+
+jobs:
+  integration:
+    strategy:
+      fail-fast: false
+      matrix:
+        env:
+#          - TEST_PROFILE: ldap
+#          - TEST_PROFILE: keycloak
+#          - TEST_PROFILE: standalone
+#          # - TEST_PROFILE: rbac
+#          - TEST_PROFILE: rbac_parallel_group_1
+#          - TEST_PROFILE: rbac_parallel_group_2
+#          - TEST_PROFILE: certified-sync
+#          - TEST_PROFILE: insights
+#          - TEST_PROFILE: iqe_rbac
+#          - TEST_PROFILE: x_repo_search
+          - TEST_PROFILE: community
+#          - TEST_PROFILE: dab_jwt
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: (Linux) Install docker compose
+        run: |
+          curl -L -o /tmp/docker-compose https://github.com/docker/compose/releases/download/v2.29.1/docker-compose-linux-x86_64
+          install /tmp/docker-compose /usr/local/bin/
+
+      - name: Spin up dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml
+        run: |
+          docker compose -f dev/compose/community.yaml up --detach
+
+      - name: Export environment variables to host
+        run: |
+          docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml exec manager /bin/bash -c 'env | grep -v -w "HOME"' >> $GITHUB_ENV
+          cat $GITHUB_ENV
+
+      - name: Wait for API online status
+        run: |
+          max_runs=10
+          for i in $(seq 1 $max_runs); do
+            echo "$i: checking api status..."
+            RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -u "${{ env.DJANGO_SUPERUSER_USERNAME }}:${{ env.DJANGO_SUPERUSER_PASSWORD }}" -LI http://localhost:5001/api/galaxy/pulp/api/v3/status/)
+            if [ "$RESPONSE" -ne 200 ]; then
+              echo "API is down. Retrying in 10 seconds..."
+              sleep 10
+            else
+              echo "API online."
+              exit 0
+            fi
+          done
+
+      - name: Install integration requirements
+        run: |
+          pip install -r integration_requirements.txt
+
+      - name: Run integration tests
+        run: |
+          pytest -v -r sx --color=yes -m '${{ env.HUB_TEST_MARKS }}' galaxy_ng/tests/integration

.github/workflows/ci_automation_hub_collection.yml

@@ -1,16 +1,16 @@
 ---
 name: Galaxy Collection CI
 on:
-  pull_request:
-    branches:
-      - "*"
-    paths-ignore:
-      - "CHANGES/**"
-      - "docs/**"
-      - "mkdocs.yml"
-  push:
-    branches:
-      - "*"
+#  pull_request:
+#    branches:
+#      - "*"
+#    paths-ignore:
+#      - "CHANGES/**"
+#      - "docs/**"
+#      - "mkdocs.yml"
+#  push:
+#    branches:
+#      - "*"
   workflow_dispatch:
 
 jobs:

.github/workflows/ci_full.yml

@@ -1,8 +1,10 @@
 ---
 name: galaxy_ng/ci
 on:
-  - push
-  - pull_request
+  workflow_dispatch:
+
+#  - push
+#  - pull_request
 
 jobs:
 

.github/workflows/ci_oci-env-integration.yml

@@ -1,12 +1,12 @@
 ---
 name: OCI Env Integration
 on:
-  pull_request:
-    branches:
-      - '**'
-  push:
-    branches:
-      - '**'
+#  pull_request:
+#    branches:
+#      - '**'
+#  push:
+#    branches:
+#      - '**'
   workflow_dispatch:
 
 jobs:

.github/workflows/ci_tdd.yml

@@ -1,9 +1,9 @@
 ---
 name: TDD
 on:
-  pull_request:
-    branches:
-      - '*'
+#  pull_request:
+#    branches:
+#      - '*'
   workflow_dispatch:
 
 jobs:

.github/workflows/sonar-pr.yaml

@@ -4,8 +4,8 @@
 name: SonarCloud
 on:
   workflow_run:
-    workflows:
-      - galaxy_ng/ci
+#    workflows:
+#      - galaxy_ng/ci
     types:
       - completed
 jobs:

.gitignore

@@ -110,6 +110,7 @@ gng_unit_testing/
 
 # Galaxy NG
 db_snapshots/
+temp_roles/*
 
 # oci-env profile files
 .compiled/

dev/compose/community.yaml

@@ -30,9 +30,9 @@ x-common-env: &common-env
   # Hostname and prefix has to be correct
   PULP_GALAXY_API_PATH_PREFIX: '/api/'
   PULP_CONTENT_PATH_PREFIX: '/pulp/content/'
-  PULP_ANSIBLE_API_HOSTNAME: 'https://localhost'
-  PULP_ANSIBLE_CONTENT_HOSTNAME: "https://localhost"
-  PULP_CONTENT_ORIGIN: "https://localhost"
+  PULP_ANSIBLE_API_HOSTNAME: 'http://localhost:5001'
+  PULP_ANSIBLE_CONTENT_HOSTNAME: "http://localhost:5001"
+  PULP_CONTENT_ORIGIN: "http://localhost:5001"
   PULP_CSRF_TRUSTED_ORIGINS: "['https://localhost']"
 
   # auth ...
@@ -59,7 +59,7 @@ x-common-env: &common-env
   PULP_GALAXY_AUTO_SIGN_COLLECTIONS: 'false'
 
   # disable user/group modifications
-  PULP_ALLOW_LOCAL_RESOURCE_MANAGEMENT: 'false'
+  PULP_ALLOW_LOCAL_RESOURCE_MANAGEMENT: 'true'
 
   # role content workaround
   PULP_ANSIBLE_BASE_ROLES_REQUIRE_VIEW: 'false'
@@ -74,8 +74,9 @@ x-common-env: &common-env
   # PULP_RESOURCE_SERVER__VALIDATE_HTTPS='false'
 
   # Integration test settings
-  HUB_TEST_AUTHENTICATION_BACKEND: "community"
-  HUB_TEST_MARKS: "deployment_community"
+  HUB_TEST_AUTHENTICATION_BACKEND: 'community'
+  HUB_TEST_MARKS: 'deployment_community'
+  HUB_API_ROOT: 'http://localhost:5001/api/'
 
   # Unpin dependencies on setup.py if set to 0
   LOCK_REQUIREMENTS: 0

docker/etc/settings.py

@@ -39,7 +39,10 @@
 REDIS_HOST = os.environ.get('PULP_REDIS_HOST')
 REDIS_PORT = os.environ.get('PULP_REDIS_PORT')
 
-REST_FRAMEWORK__DEFAULT_RENDERER_CLASSES = ['rest_framework.renderers.JSONRenderer']
+REST_FRAMEWORK__DEFAULT_RENDERER_CLASSES = [
+    'rest_framework.renderers.JSONRenderer',
+    'galaxy_ng.app.renderers.CustomBrowsableAPIRenderer'
+]
 
 _enabled_handlers = ['console']
 _extra_handlers = {}

galaxy_ng/tests/integration/api/test_ui_paths.py

@@ -1,6 +1,5 @@
 import random
 import json
-import subprocess
 from http import HTTPStatus
 
 import pytest
@@ -874,7 +873,7 @@ def build_upload_wait(tags):
 
 # /api/automation-hub/_ui/v1/tags/roles/
 @pytest.mark.deployment_community
-def test_api_ui_v1_tags_roles(ansible_config):
+def test_api_ui_v1_tags_roles(ansible_config, docker_compose_exec):
     """Test endpoint's sorting and filtering"""
 
     def _sync_role(github_user, role_name):
@@ -886,11 +885,7 @@ def _sync_role(github_user, role_name):
         wait_for_v1_task(resp=resp, api_client=api_admin_client)
 
     def _populate_tags_cmd():
-        proc = subprocess.run(
-            "django-admin populate-role-tags",
-            shell=True,
-            capture_output=True,
-        )
+        proc = docker_compose_exec('django-admin populate-role-tags')
         assert proc.returncode == 0
 
     config = ansible_config("basic_user")

galaxy_ng/tests/integration/cli/test_community_sync.py

@@ -4,7 +4,6 @@
 import json
 import pytest
 import requests
-import subprocess
 
 from ansible.galaxy.api import GalaxyError
 
@@ -15,7 +14,7 @@
 
 
 @pytest.mark.deployment_community
-def test_community_collection_download_count_sync(ansible_config):
+def test_community_collection_download_count_sync(ansible_config, docker_compose_exec):
     """ Test collection download count sync command """
 
     # FIXME(jctanner): once beta switches over, this test is no longer needed.
@@ -83,11 +82,7 @@ def test_community_collection_download_count_sync(ansible_config):
     wait_for_task(api_client, sync_task)
 
     # run the django command
-    pid = subprocess.run(
-        'pulpcore-manager sync-collection-download-counts',
-        shell=True,
-        capture_output=True,
-    )
+    pid = docker_compose_exec('pulpcore-manager sync-collection-download-counts')
     assert pid.returncode == 0
 
     # check the counter in the api

galaxy_ng/tests/integration/community/test_community_api.py

@@ -813,10 +813,10 @@ def test_v1_role_versions(ansible_config):
     id = resp["results"][0]["id"]
     versions = resp["results"][0]["summary_fields"]["versions"]
 
-    resp = api_client(f'/api/v1/roles/{id}/versions')
+    resp = api_client(f'/api/v1/roles/{id}/versions/')
     assert resp["count"] >= len(versions)
 
     with pytest.raises(AnsibleError) as html:
-        api_client(f"v1/roles/{id}/versions", headers={"Accept": "text/html"})
+        api_client(f"v1/roles/{id}/versions/", headers={"Accept": "text/html"})
     assert not isinstance(html.value, dict)
     assert "results" in str(html.value)

galaxy_ng/tests/integration/community/test_role_import_overrides.py

@@ -39,11 +39,10 @@
             'alternate_namespace_name': None,
             'alternate_role_name': None,
         }
-
     ]
 )
 @pytest.mark.deployment_community
-def test_role_import_overrides(ansible_config, spec):
+def test_role_import_overrides(ansible_config, spec, docker_compose_exec):
     """" Validate setting namespace in meta/main.yml does the right thing """
 
     admin_config = ansible_config("admin")
@@ -83,10 +82,12 @@ def test_role_import_overrides(ansible_config, spec):
         'meta_namespace': spec['meta_namespace'],
         'meta_name': spec['meta_name'],
     }
-    lr = LegacyRoleGitRepoBuilder(**builder_kwargs)
+    lr = LegacyRoleGitRepoBuilder(**builder_kwargs, docker_compose_exec=docker_compose_exec)
+
+    local_tmp_dir = lr.role_cont_dir
 
     # run the import
-    payload = {'alternate_clone_url': lr.role_dir}
+    payload = {'alternate_clone_url': local_tmp_dir}
     for key in ['github_user', 'github_repo', 'alternate_namespace_name', 'alternate_role_name']:
         if spec.get(key):
             payload[key] = spec[key]

galaxy_ng/tests/integration/community/test_v1_api.py

@@ -133,7 +133,7 @@ def test_custom_browsable_format(ansible_config):
         require_auth=True,
     )
 
-    resp = api_client("v1/namespaces")
+    resp = api_client("v1/namespaces/")
     assert isinstance(resp, dict)
     assert "results" in resp
 
@@ -142,7 +142,7 @@ def test_custom_browsable_format(ansible_config):
     assert "results" in resp
 
     with pytest.raises(AnsibleError) as html:
-        api_client("v1/namespaces", headers={"Accept": "text/html"})
+        api_client("v1/namespaces/", headers={"Accept": "text/html"})
     assert not isinstance(html.value, dict)
     assert "results" in str(html.value)
 
@@ -154,7 +154,7 @@ def test_custom_browsable_format(ansible_config):
         require_auth=True,
     )
 
-    resp = api_client("v1/namespaces")
+    resp = api_client("v1/namespaces/")
     assert isinstance(resp, dict)
     assert "results" in resp
 
@@ -163,6 +163,6 @@ def test_custom_browsable_format(ansible_config):
     assert "results" in resp
 
     with pytest.raises(AnsibleError) as html:
-        api_client("v1/namespaces", headers={"Accept": "text/html"})
+        api_client("v1/namespaces/", headers={"Accept": "text/html"})
     assert not isinstance(html.value, dict)
     assert "results" in str(html.value)

galaxy_ng/tests/integration/conftest.py

@@ -3,6 +3,7 @@
 import os
 import shutil
 import yaml
+import subprocess
 
 import pytest
 from orionutils.utils import increment_version
@@ -726,3 +727,20 @@ def skip_if_require_signature_for_approval():
 def skip_if_not_require_signature_for_approval():
     if not require_signature_for_approval():
         pytest.skip("This test needs refactoring to work with signatures required on move.")
+
+
+@pytest.fixture
+def docker_compose_exec():
+    def _exec(cmd: str, cwd=None):
+        cd = ''
+        if cwd is not None:
+            cd = f'cd {cwd};'
+
+        proc = subprocess.run(
+            f"docker exec compose-manager-1 /bin/bash -c '{cd}{cmd}'",
+            shell=True,
+            capture_output=True,
+        )
+        return proc
+
+    return _exec