[docker-compose] certified-sync CI (#2407)

refactor sync tests
ansible · Jan 22, 2025 · e6ed459 · e6ed459
1 parent d77427f
commit e6ed459
Show file tree

Hide file tree

Showing 9 changed files with 76 additions and 135 deletions.
diff --git a/.github/workflows/ci-docker-compose-integration-extra.yml b/.github/workflows/ci-docker-compose-integration-extra.yml
diff --git a/.github/workflows/ci-docker-compose-integration.yml b/.github/workflows/ci-docker-compose-integration.yml
@@ -15,18 +15,14 @@ jobs:
       fail-fast: false
       matrix:
         env:
-          # - TEST_PROFILE: ldap
-          # - TEST_PROFILE: keycloak
-          - TEST_PROFILE: standalone
-          # - TEST_PROFILE: rbac
-          # - TEST_PROFILE: rbac_parallel_group_1
-          # - TEST_PROFILE: rbac_parallel_group_2
-          # - TEST_PROFILE: certified-sync
-          - TEST_PROFILE: insights
-          # - TEST_PROFILE: iqe_rbac
-          # - TEST_PROFILE: x_repo_search
-          - TEST_PROFILE: community
-          # - TEST_PROFILE: dab_jwt
+          - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "deployment_standalone or all" }
+          - { TEST_PROFILE: insights, HUB_TEST_MARKS: "deployment_cloud or all" }
+          - { TEST_PROFILE: community, HUB_TEST_MARKS: "deployment_community" }
+          - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "x_repo_search" }
+          - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "iqe_rbac_test" }
+          - { TEST_PROFILE: certified-sync, HUB_TEST_MARKS: "sync" }
+          - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "rbac_parallel_group_1" }
+          - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "rbac_parallel_group_2" }
 
     runs-on: ubuntu-latest
     steps:
@@ -54,9 +50,16 @@ jobs:
           docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml up --detach
 
       - name: Export environment variables to host
+        if: ${{ matrix.env.TEST_PROFILE != 'certified-sync' }}
         run: |
           docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml exec manager /bin/bash -c 'env | grep -v -w "HOME"' >> $GITHUB_ENV
-          cat $GITHUB_ENV
+
+      - name: Export environment variables to host (certified-sync)
+        if: ${{ matrix.env.TEST_PROFILE == 'certified-sync' }}
+        run: |
+          docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml exec standalone-manager /bin/bash -c 'env | grep -v -w "HOME"' >> $GITHUB_ENV
+
+      - run: cat $GITHUB_ENV
 
       - name: Wait for API online status
         run: |
@@ -79,7 +82,7 @@ jobs:
 
       - name: Run integration tests
         run: |
-          pytest -v -r sx --color=yes -m '${{ env.HUB_TEST_MARKS }}' galaxy_ng/tests/integration
+          pytest -v -r sx --color=yes -m '${{ matrix.env.HUB_TEST_MARKS }}' galaxy_ng/tests/integration
         env:
           GALAXYKIT_SLEEP_SECONDS_POLLING: ".5"
           GALAXYKIT_SLEEP_SECONDS_ONETIME: ".5"

diff --git a/.github/workflows/ci_oci-env-integration.yml b/.github/workflows/ci_oci-env-integration.yml
@@ -21,7 +21,7 @@ jobs:
           # - TEST_PROFILE: rbac
           # - TEST_PROFILE: rbac_parallel_group_1
           # - TEST_PROFILE: rbac_parallel_group_2
-          - TEST_PROFILE: certified-sync
+          # - TEST_PROFILE: certified-sync
           # - TEST_PROFILE: insights
           # - TEST_PROFILE: iqe_rbac
           # - TEST_PROFILE: x_repo_search

diff --git a/dev/compose/certified-sync.yaml b/dev/compose/certified-sync.yaml
@@ -148,7 +148,7 @@ services:
       - standalone-postgres
     volumes:
       - "etc_pulp_certs:/etc/pulp/certs"
-      - "var_lib_pulp:/var/lib/pulp"
+      - "var_lib_pulp_standalone:/var/lib/pulp"
       - "../../../:/src"
       - "../../:/app"
     environment:
@@ -177,7 +177,7 @@ services:
       - standalone-migrations
     volumes:
       - "etc_pulp_certs:/etc/pulp/certs"
-      - "var_lib_pulp:/var/lib/pulp"
+      - "var_lib_pulp_standalone:/var/lib/pulp"
       - "../../../:/src"
       - "../../:/app"
     environment:
@@ -204,7 +204,7 @@ services:
       - standalone-migrations
     volumes:
       - "etc_pulp_certs:/etc/pulp/certs"
-      - "var_lib_pulp:/var/lib/pulp"
+      - "var_lib_pulp_standalone:/var/lib/pulp"
       - "../../../:/src"
       - "../../:/app"
     environment:
@@ -231,7 +231,7 @@ services:
       - standalone-migrations
     volumes:
       - "etc_pulp_certs:/etc/pulp/certs"
-      - "var_lib_pulp:/var/lib/pulp"
+      - "var_lib_pulp_standalone:/var/lib/pulp"
       - "../../../:/src"
       - "../../:/app"
     environment:
@@ -258,7 +258,7 @@ services:
       - standalone-worker
     volumes:
       - "etc_pulp_certs:/etc/pulp/certs"
-      - "var_lib_pulp:/var/lib/pulp"
+      - "var_lib_pulp_standalone:/var/lib/pulp"
       - "../../../:/src"
       - "../../:/app"
     environment:
@@ -345,11 +345,13 @@ services:
       until pg_isready -h insights-postgres -p 5432; do
         sleep 2;
       done;
+
+      tar --overwrite -xzf /db_data/insights-fixture.tar.gz -C /var/lib/pulp/;
       PGPASSWORD="galaxy_ng" pg_restore -h insights-postgres -U galaxy_ng -d galaxy_ng --no-owner /var/lib/pulp/pulp_db.backup;
       "
     volumes:
       - "./data:/db_data"
-      - "var_lib_pulp:/var/lib/pulp"
+      - "var_lib_pulp_insights:/var/lib/pulp"
 
   insights-migrations:
     image: "localhost/galaxy_ng/galaxy_ng:dev"
@@ -359,7 +361,7 @@ services:
       - load-sync-data
     volumes:
       - "etc_pulp_certs:/etc/pulp/certs"
-      - "var_lib_pulp:/var/lib/pulp"
+      - "var_lib_pulp_insights:/var/lib/pulp"
       - "../../../:/src"
       - "../../:/app"
     environment:
@@ -374,8 +376,6 @@ services:
 
         pulpcore-manager check --database default;
 
-        tar --overwrite -xzf dev/compose/data/insights-fixture.tar.gz -C /var/lib/pulp/;
-
         pulpcore-manager migrate;
 
         pulpcore-manager shell < /src/galaxy_ng/dev/common/setup_test_data.py;
@@ -404,7 +404,7 @@ services:
       - insights-migrations
     volumes:
       - "etc_pulp_certs:/etc/pulp/certs"
-      - "var_lib_pulp:/var/lib/pulp"
+      - "var_lib_pulp_insights:/var/lib/pulp"
       - "../../../:/src"
       - "../../:/app"
     environment:
@@ -431,7 +431,7 @@ services:
       - insights-migrations
     volumes:
       - "etc_pulp_certs:/etc/pulp/certs"
-      - "var_lib_pulp:/var/lib/pulp"
+      - "var_lib_pulp_insights:/var/lib/pulp"
       - "../../../:/src"
       - "../../:/app"
     environment:
@@ -458,7 +458,7 @@ services:
       - insights-migrations
     volumes:
       - "etc_pulp_certs:/etc/pulp/certs"
-      - "var_lib_pulp:/var/lib/pulp"
+      - "var_lib_pulp_insights:/var/lib/pulp"
       - "../../../:/src"
       - "../../:/app"
     environment:
@@ -485,7 +485,7 @@ services:
       - insights-worker
     volumes:
       - "etc_pulp_certs:/etc/pulp/certs"
-      - "var_lib_pulp:/var/lib/pulp"
+      - "var_lib_pulp_insights:/var/lib/pulp"
       - "../../../:/src"
       - "../../:/app"
     environment:
@@ -536,8 +536,10 @@ services:
       - '../nginx/certified-sync/insights.conf:/etc/nginx/nginx.conf:ro'
 
 volumes:
-  var_lib_pulp:
-    name: var_lib_pulp
+  var_lib_pulp_standalone:
+    name: var_lib_pulp_standalone
+  var_lib_pulp_insights:
+    name: var_lib_pulp_insights
   etc_pulp_certs:
     name: etc_pulp_certs
 

diff --git a/dev/oci_env_integration/test_fixtures/insights-fixture.tar.gz b/dev/oci_env_integration/test_fixtures/insights-fixture.tar.gz
diff --git a/dev/oci_env_integration/test_fixtures/insights-fixture.tar.gz b/dev/oci_env_integration/test_fixtures/insights-fixture.tar.gz
@@ -0,0 +1 @@
+dev/compose/data/insights-fixture.tar.gz
diff --git a/galaxy_ng/tests/integration/api/test_certified_sync.py b/galaxy_ng/tests/integration/api/test_certified_sync.py
@@ -104,7 +104,7 @@ def _assert_sync(manifest, client):
 
 def _assert_namespace_sync(pah_client, crc_client, namespace):
     crc_ns = crc_client(f"v3/namespaces/{namespace['name']}/")
-    pah_ns = pah_client(f"v3/plugin/ansible/content/rh-certified/namespaces/{namespace['name']}")
+    pah_ns = pah_client(f"v3/plugin/ansible/content/rh-certified/namespaces/{namespace['name']}/")
     pah_galaxy_ns = pah_client(f"v3/namespaces/{namespace['name']}/")
 
     # test the fields
@@ -134,7 +134,8 @@ def test_basic_sync(sync_instance_crc, ansible_config):
     manifest, crc_config = sync_instance_crc
 
     pah_client = get_client(
-        config=config
+        config=config,
+        request_token=False,
     )
 
     clear_certified(pah_client)
@@ -156,7 +157,7 @@ def test_synclist_sync(sync_instance_crc, ansible_config):
 
     crc_client = get_client(
         config=crc_config,
-        request_token=True,
+        request_token=False,
         require_auth=True
     )
 
@@ -203,16 +204,15 @@ def test_signed_only_sync(sync_instance_crc, ansible_config):
     _assert_sync(expected_manifest, pah_client)
 
 
-# @pytest.mark.skip("broken by python 3.11 ... ?")
 @pytest.mark.sync
 def test_namespace_sync(sync_instance_crc, ansible_config):
     pah_config = ansible_config(profile="admin")
     manifest, crc_config = sync_instance_crc
 
     crc_config.profile = "admin"
 
-    pah_client = get_client(pah_config)
-    crc_client = get_client(crc_config)
+    pah_client = get_client(pah_config, request_token=False)
+    crc_client = get_client(crc_config, request_token=False)
 
     ns_data = {
         "name": "ansible",
@@ -273,8 +273,6 @@ def test_namespace_sync(sync_instance_crc, ansible_config):
         method="PUT",
     )
 
-    clear_certified(pah_client)
-
     perform_sync(pah_client, crc_config)
     _assert_namespace_sync(pah_client, crc_client, ns)
 

diff --git a/galaxy_ng/tests/integration/api/test_custom_repo_sync.py b/galaxy_ng/tests/integration/api/test_custom_repo_sync.py
@@ -1,5 +1,5 @@
 import logging
-
+import os
 import pytest
 
 from galaxykit.namespaces import create_namespace
@@ -21,6 +21,15 @@
 
 @pytest.mark.min_hub_version("4.7dev")
 class TestCustomReposSync:
+    remote_url = os.getenv(
+        "TEST_CRC_API_ROOT_SERVICE",
+        "http://insights-proxy:8080/api/automation-hub/"
+    )
+    remote_auth_url = os.getenv(
+        "TEST_CRC_AUTH_URL_SERVICE",
+        "http://insights-proxy:8080/auth/realms/redhat-external/protocol/openid-connect/token"
+    )
+
     @pytest.mark.sync
     def test_basic_sync_custom_repo_with_req_file(self, sync_instance_crc, galaxy_client):
         """
@@ -29,7 +38,6 @@ def test_basic_sync_custom_repo_with_req_file(self, sync_instance_crc, galaxy_cl
         """
         # this is the insights mode instance (source hub)
         _, custom_config = sync_instance_crc
-        url = custom_config["url"]
         galaxy_kit_client = GalaxyKitClient(custom_config)
         source_client = galaxy_kit_client.gen_authorized_client(
             {"username": "notifications_admin", "password": "redhat"}, basic_token=True
@@ -65,12 +73,12 @@ def test_basic_sync_custom_repo_with_req_file(self, sync_instance_crc, galaxy_cl
         gc = galaxy_client("iqe_admin")
         test_remote_name = f"remote-test-{generate_random_string()}"
         params = {
-            "auth_url": custom_config["auth_url"],
+            "auth_url": self.remote_auth_url,
             "token": custom_config["token"],
             "requirements_file": f"---\ncollections:\n- {artifact.namespace}.{artifact.name}",
         }
         create_remote(
-            gc, test_remote_name, f"{url}content/{test_repo_name_1}/", params=params
+            gc, test_remote_name, f"{self.remote_url}content/{test_repo_name_1}/", params=params
         )
         create_repo_and_dist(gc, test_repo_name_1, remote=test_remote_name)
 
@@ -102,7 +110,6 @@ def test_basic_sync_custom_repo_mirror(self, sync_instance_crc, galaxy_client):
         it's removed from the local repo
         """
         _, custom_config = sync_instance_crc
-        url = custom_config["url"]
         galaxy_kit_client = GalaxyKitClient(custom_config)
         source_client = galaxy_kit_client.gen_authorized_client(
             {"username": "notifications_admin", "password": "redhat"}, basic_token=True
@@ -126,11 +133,11 @@ def test_basic_sync_custom_repo_mirror(self, sync_instance_crc, galaxy_client):
         gc = galaxy_client("iqe_admin")
         test_remote_name = f"remote-test-{generate_random_string()}"
         params = {
-            "auth_url": custom_config["auth_url"],
+            "auth_url": self.remote_auth_url,
             "token": custom_config["token"],
         }
         create_remote(
-            gc, test_remote_name, f"{url}content/{test_repo_name_1}/", params=params
+            gc, test_remote_name, f"{self.remote_url}content/{test_repo_name_1}/", params=params
         )
         pulp_href = create_repo_and_dist(gc, test_repo_name_1, remote=test_remote_name)