diff --git a/.github/workflows/ci-docker-compose-integration-extra.yml b/.github/workflows/ci-docker-compose-integration-extra.yml deleted file mode 100644 index 92014dadeb..0000000000 --- a/.github/workflows/ci-docker-compose-integration-extra.yml +++ /dev/null @@ -1,82 +0,0 @@ ---- -name: Docker Compose Integration -on: - pull_request: - branches: - - '**' - push: - branches: - - '**' - workflow_dispatch: - -jobs: - integration: - strategy: - fail-fast: false - matrix: - env: - - - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "x_repo_search" } - - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "iqe_rbac_test" } - - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "rbac_parallel_group_1" } - - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "rbac_parallel_group_2" } - - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: (Linux) Install docker compose - run: | - curl -L -o /tmp/docker-compose https://github.com/docker/compose/releases/download/v2.29.1/docker-compose-linux-x86_64 - install /tmp/docker-compose /usr/local/bin/ - - - name: sudo apt update - run: | - sudo apt update - - - name: Install python requirements and podman - run: | - sudo apt-get install -y libsasl2-dev python-dev-is-python3 libldap2-dev libssl-dev podman - - - name: Spin up dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml - run: | - docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml up --detach - - - name: Export environment variables to host - run: | - docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml exec manager /bin/bash -c 'env | grep -v -w "HOME"' >> $GITHUB_ENV - cat $GITHUB_ENV - - - name: Wait for API online status - run: | - max_runs=10 - for i in $(seq 1 $max_runs); do - echo "$i: checking api status..." - RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -u "${{ env.DJANGO_SUPERUSER_USERNAME }}:${{ env.DJANGO_SUPERUSER_PASSWORD }}" -LI http://localhost:5001/api/galaxy/pulp/api/v3/status/) - if [ "$RESPONSE" -ne 200 ]; then - echo "API is down. Retrying in 10 seconds..." - sleep 10 - else - echo "API online." - exit 0 - fi - done - - - name: Install integration requirements - run: | - pip install -r integration_requirements.txt - - - name: Run integration tests - run: | - pytest -v -r sx --color=yes -m '${{ matrix.env.HUB_TEST_MARKS }}' galaxy_ng/tests/integration - env: - GALAXYKIT_SLEEP_SECONDS_POLLING: ".5" - GALAXYKIT_SLEEP_SECONDS_ONETIME: ".5" - GALAXYKIT_POLLING_MAX_ATTEMPTS: "50" - GALAXY_SLEEP_SECONDS_POLLING: ".5" - GALAXY_SLEEP_SECONDS_ONETIME: ".5" - GALAXY_POLLING_MAX_ATTEMPTS: "50" diff --git a/.github/workflows/ci-docker-compose-integration.yml b/.github/workflows/ci-docker-compose-integration.yml index 5c9448d3b1..8d43010991 100644 --- a/.github/workflows/ci-docker-compose-integration.yml +++ b/.github/workflows/ci-docker-compose-integration.yml @@ -15,18 +15,14 @@ jobs: fail-fast: false matrix: env: - # - TEST_PROFILE: ldap - # - TEST_PROFILE: keycloak - - TEST_PROFILE: standalone - # - TEST_PROFILE: rbac - # - TEST_PROFILE: rbac_parallel_group_1 - # - TEST_PROFILE: rbac_parallel_group_2 - # - TEST_PROFILE: certified-sync - - TEST_PROFILE: insights - # - TEST_PROFILE: iqe_rbac - # - TEST_PROFILE: x_repo_search - - TEST_PROFILE: community - # - TEST_PROFILE: dab_jwt + - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "deployment_standalone or all" } + - { TEST_PROFILE: insights, HUB_TEST_MARKS: "deployment_cloud or all" } + - { TEST_PROFILE: community, HUB_TEST_MARKS: "deployment_community" } + - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "x_repo_search" } + - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "iqe_rbac_test" } + - { TEST_PROFILE: certified-sync, HUB_TEST_MARKS: "sync" } + - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "rbac_parallel_group_1" } + - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "rbac_parallel_group_2" } runs-on: ubuntu-latest steps: @@ -54,9 +50,16 @@ jobs: docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml up --detach - name: Export environment variables to host + if: ${{ matrix.env.TEST_PROFILE != 'certified-sync' }} run: | docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml exec manager /bin/bash -c 'env | grep -v -w "HOME"' >> $GITHUB_ENV - cat $GITHUB_ENV + + - name: Export environment variables to host (certified-sync) + if: ${{ matrix.env.TEST_PROFILE == 'certified-sync' }} + run: | + docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml exec standalone-manager /bin/bash -c 'env | grep -v -w "HOME"' >> $GITHUB_ENV + + - run: cat $GITHUB_ENV - name: Wait for API online status run: | @@ -79,7 +82,7 @@ jobs: - name: Run integration tests run: | - pytest -v -r sx --color=yes -m '${{ env.HUB_TEST_MARKS }}' galaxy_ng/tests/integration + pytest -v -r sx --color=yes -m '${{ matrix.env.HUB_TEST_MARKS }}' galaxy_ng/tests/integration env: GALAXYKIT_SLEEP_SECONDS_POLLING: ".5" GALAXYKIT_SLEEP_SECONDS_ONETIME: ".5" diff --git a/.github/workflows/ci_oci-env-integration.yml b/.github/workflows/ci_oci-env-integration.yml index 9f5dfd0eac..15451bb692 100644 --- a/.github/workflows/ci_oci-env-integration.yml +++ b/.github/workflows/ci_oci-env-integration.yml @@ -21,7 +21,7 @@ jobs: # - TEST_PROFILE: rbac # - TEST_PROFILE: rbac_parallel_group_1 # - TEST_PROFILE: rbac_parallel_group_2 - - TEST_PROFILE: certified-sync + # - TEST_PROFILE: certified-sync # - TEST_PROFILE: insights # - TEST_PROFILE: iqe_rbac # - TEST_PROFILE: x_repo_search diff --git a/dev/compose/certified-sync.yaml b/dev/compose/certified-sync.yaml index 3375e7a0e0..5a92c5da42 100644 --- a/dev/compose/certified-sync.yaml +++ b/dev/compose/certified-sync.yaml @@ -148,7 +148,7 @@ services: - standalone-postgres volumes: - "etc_pulp_certs:/etc/pulp/certs" - - "var_lib_pulp:/var/lib/pulp" + - "var_lib_pulp_standalone:/var/lib/pulp" - "../../../:/src" - "../../:/app" environment: @@ -177,7 +177,7 @@ services: - standalone-migrations volumes: - "etc_pulp_certs:/etc/pulp/certs" - - "var_lib_pulp:/var/lib/pulp" + - "var_lib_pulp_standalone:/var/lib/pulp" - "../../../:/src" - "../../:/app" environment: @@ -204,7 +204,7 @@ services: - standalone-migrations volumes: - "etc_pulp_certs:/etc/pulp/certs" - - "var_lib_pulp:/var/lib/pulp" + - "var_lib_pulp_standalone:/var/lib/pulp" - "../../../:/src" - "../../:/app" environment: @@ -231,7 +231,7 @@ services: - standalone-migrations volumes: - "etc_pulp_certs:/etc/pulp/certs" - - "var_lib_pulp:/var/lib/pulp" + - "var_lib_pulp_standalone:/var/lib/pulp" - "../../../:/src" - "../../:/app" environment: @@ -258,7 +258,7 @@ services: - standalone-worker volumes: - "etc_pulp_certs:/etc/pulp/certs" - - "var_lib_pulp:/var/lib/pulp" + - "var_lib_pulp_standalone:/var/lib/pulp" - "../../../:/src" - "../../:/app" environment: @@ -345,11 +345,13 @@ services: until pg_isready -h insights-postgres -p 5432; do sleep 2; done; + + tar --overwrite -xzf /db_data/insights-fixture.tar.gz -C /var/lib/pulp/; PGPASSWORD="galaxy_ng" pg_restore -h insights-postgres -U galaxy_ng -d galaxy_ng --no-owner /var/lib/pulp/pulp_db.backup; " volumes: - "./data:/db_data" - - "var_lib_pulp:/var/lib/pulp" + - "var_lib_pulp_insights:/var/lib/pulp" insights-migrations: image: "localhost/galaxy_ng/galaxy_ng:dev" @@ -359,7 +361,7 @@ services: - load-sync-data volumes: - "etc_pulp_certs:/etc/pulp/certs" - - "var_lib_pulp:/var/lib/pulp" + - "var_lib_pulp_insights:/var/lib/pulp" - "../../../:/src" - "../../:/app" environment: @@ -374,8 +376,6 @@ services: pulpcore-manager check --database default; - tar --overwrite -xzf dev/compose/data/insights-fixture.tar.gz -C /var/lib/pulp/; - pulpcore-manager migrate; pulpcore-manager shell < /src/galaxy_ng/dev/common/setup_test_data.py; @@ -404,7 +404,7 @@ services: - insights-migrations volumes: - "etc_pulp_certs:/etc/pulp/certs" - - "var_lib_pulp:/var/lib/pulp" + - "var_lib_pulp_insights:/var/lib/pulp" - "../../../:/src" - "../../:/app" environment: @@ -431,7 +431,7 @@ services: - insights-migrations volumes: - "etc_pulp_certs:/etc/pulp/certs" - - "var_lib_pulp:/var/lib/pulp" + - "var_lib_pulp_insights:/var/lib/pulp" - "../../../:/src" - "../../:/app" environment: @@ -458,7 +458,7 @@ services: - insights-migrations volumes: - "etc_pulp_certs:/etc/pulp/certs" - - "var_lib_pulp:/var/lib/pulp" + - "var_lib_pulp_insights:/var/lib/pulp" - "../../../:/src" - "../../:/app" environment: @@ -485,7 +485,7 @@ services: - insights-worker volumes: - "etc_pulp_certs:/etc/pulp/certs" - - "var_lib_pulp:/var/lib/pulp" + - "var_lib_pulp_insights:/var/lib/pulp" - "../../../:/src" - "../../:/app" environment: @@ -536,8 +536,10 @@ services: - '../nginx/certified-sync/insights.conf:/etc/nginx/nginx.conf:ro' volumes: - var_lib_pulp: - name: var_lib_pulp + var_lib_pulp_standalone: + name: var_lib_pulp_standalone + var_lib_pulp_insights: + name: var_lib_pulp_insights etc_pulp_certs: name: etc_pulp_certs diff --git a/dev/oci_env_integration/test_fixtures/insights-fixture.tar.gz b/dev/oci_env_integration/test_fixtures/insights-fixture.tar.gz deleted file mode 100644 index 50f33f10c9..0000000000 Binary files a/dev/oci_env_integration/test_fixtures/insights-fixture.tar.gz and /dev/null differ diff --git a/dev/oci_env_integration/test_fixtures/insights-fixture.tar.gz b/dev/oci_env_integration/test_fixtures/insights-fixture.tar.gz new file mode 120000 index 0000000000..ef543119e1 --- /dev/null +++ b/dev/oci_env_integration/test_fixtures/insights-fixture.tar.gz @@ -0,0 +1 @@ +dev/compose/data/insights-fixture.tar.gz \ No newline at end of file diff --git a/galaxy_ng/tests/integration/api/test_certified_sync.py b/galaxy_ng/tests/integration/api/test_certified_sync.py index 1549e45c2a..e1648cebd8 100644 --- a/galaxy_ng/tests/integration/api/test_certified_sync.py +++ b/galaxy_ng/tests/integration/api/test_certified_sync.py @@ -104,7 +104,7 @@ def _assert_sync(manifest, client): def _assert_namespace_sync(pah_client, crc_client, namespace): crc_ns = crc_client(f"v3/namespaces/{namespace['name']}/") - pah_ns = pah_client(f"v3/plugin/ansible/content/rh-certified/namespaces/{namespace['name']}") + pah_ns = pah_client(f"v3/plugin/ansible/content/rh-certified/namespaces/{namespace['name']}/") pah_galaxy_ns = pah_client(f"v3/namespaces/{namespace['name']}/") # test the fields @@ -134,7 +134,8 @@ def test_basic_sync(sync_instance_crc, ansible_config): manifest, crc_config = sync_instance_crc pah_client = get_client( - config=config + config=config, + request_token=False, ) clear_certified(pah_client) @@ -156,7 +157,7 @@ def test_synclist_sync(sync_instance_crc, ansible_config): crc_client = get_client( config=crc_config, - request_token=True, + request_token=False, require_auth=True ) @@ -203,7 +204,6 @@ def test_signed_only_sync(sync_instance_crc, ansible_config): _assert_sync(expected_manifest, pah_client) -# @pytest.mark.skip("broken by python 3.11 ... ?") @pytest.mark.sync def test_namespace_sync(sync_instance_crc, ansible_config): pah_config = ansible_config(profile="admin") @@ -211,8 +211,8 @@ def test_namespace_sync(sync_instance_crc, ansible_config): crc_config.profile = "admin" - pah_client = get_client(pah_config) - crc_client = get_client(crc_config) + pah_client = get_client(pah_config, request_token=False) + crc_client = get_client(crc_config, request_token=False) ns_data = { "name": "ansible", @@ -273,8 +273,6 @@ def test_namespace_sync(sync_instance_crc, ansible_config): method="PUT", ) - clear_certified(pah_client) - perform_sync(pah_client, crc_config) _assert_namespace_sync(pah_client, crc_client, ns) diff --git a/galaxy_ng/tests/integration/api/test_custom_repo_sync.py b/galaxy_ng/tests/integration/api/test_custom_repo_sync.py index 72f228b7dc..1ef09e61dd 100644 --- a/galaxy_ng/tests/integration/api/test_custom_repo_sync.py +++ b/galaxy_ng/tests/integration/api/test_custom_repo_sync.py @@ -1,5 +1,5 @@ import logging - +import os import pytest from galaxykit.namespaces import create_namespace @@ -21,6 +21,15 @@ @pytest.mark.min_hub_version("4.7dev") class TestCustomReposSync: + remote_url = os.getenv( + "TEST_CRC_API_ROOT_SERVICE", + "http://insights-proxy:8080/api/automation-hub/" + ) + remote_auth_url = os.getenv( + "TEST_CRC_AUTH_URL_SERVICE", + "http://insights-proxy:8080/auth/realms/redhat-external/protocol/openid-connect/token" + ) + @pytest.mark.sync def test_basic_sync_custom_repo_with_req_file(self, sync_instance_crc, galaxy_client): """ @@ -29,7 +38,6 @@ def test_basic_sync_custom_repo_with_req_file(self, sync_instance_crc, galaxy_cl """ # this is the insights mode instance (source hub) _, custom_config = sync_instance_crc - url = custom_config["url"] galaxy_kit_client = GalaxyKitClient(custom_config) source_client = galaxy_kit_client.gen_authorized_client( {"username": "notifications_admin", "password": "redhat"}, basic_token=True @@ -65,12 +73,12 @@ def test_basic_sync_custom_repo_with_req_file(self, sync_instance_crc, galaxy_cl gc = galaxy_client("iqe_admin") test_remote_name = f"remote-test-{generate_random_string()}" params = { - "auth_url": custom_config["auth_url"], + "auth_url": self.remote_auth_url, "token": custom_config["token"], "requirements_file": f"---\ncollections:\n- {artifact.namespace}.{artifact.name}", } create_remote( - gc, test_remote_name, f"{url}content/{test_repo_name_1}/", params=params + gc, test_remote_name, f"{self.remote_url}content/{test_repo_name_1}/", params=params ) create_repo_and_dist(gc, test_repo_name_1, remote=test_remote_name) @@ -102,7 +110,6 @@ def test_basic_sync_custom_repo_mirror(self, sync_instance_crc, galaxy_client): it's removed from the local repo """ _, custom_config = sync_instance_crc - url = custom_config["url"] galaxy_kit_client = GalaxyKitClient(custom_config) source_client = galaxy_kit_client.gen_authorized_client( {"username": "notifications_admin", "password": "redhat"}, basic_token=True @@ -126,11 +133,11 @@ def test_basic_sync_custom_repo_mirror(self, sync_instance_crc, galaxy_client): gc = galaxy_client("iqe_admin") test_remote_name = f"remote-test-{generate_random_string()}" params = { - "auth_url": custom_config["auth_url"], + "auth_url": self.remote_auth_url, "token": custom_config["token"], } create_remote( - gc, test_remote_name, f"{url}content/{test_repo_name_1}/", params=params + gc, test_remote_name, f"{self.remote_url}content/{test_repo_name_1}/", params=params ) pulp_href = create_repo_and_dist(gc, test_repo_name_1, remote=test_remote_name) diff --git a/galaxy_ng/tests/integration/conftest.py b/galaxy_ng/tests/integration/conftest.py index efa272dc7a..85aa419808 100644 --- a/galaxy_ng/tests/integration/conftest.py +++ b/galaxy_ng/tests/integration/conftest.py @@ -333,10 +333,10 @@ def sync_instance_crc(): dev/data/insights-fixture.tar.gz """ - url = os.getenv("TEST_CRC_API_ROOT", "http://localhost:38080/api/automation-hub/") + url = os.getenv("TEST_CRC_API_ROOT", "http://localhost:8080/api/automation-hub/") auth_url = os.getenv( "TEST_CRC_AUTH_URL", - "http://localhost:38080/auth/realms/redhat-external/protocol/openid-connect/token" + "http://localhost:8080/auth/realms/redhat-external/protocol/openid-connect/token" ) config = AnsibleConfigFixture(url=url, auth_url=auth_url, profile="org_admin") @@ -344,7 +344,7 @@ def sync_instance_crc(): client = get_client( config=config, - request_token=True, + request_token=False, require_auth=True ) diff --git a/galaxy_ng/tests/integration/utils/sync.py b/galaxy_ng/tests/integration/utils/sync.py index 613fd82fa9..72975b8b9d 100644 --- a/galaxy_ng/tests/integration/utils/sync.py +++ b/galaxy_ng/tests/integration/utils/sync.py @@ -1,3 +1,4 @@ +import os from ..utils import wait_for_task, cleanup_namespace, iterate_all @@ -31,10 +32,18 @@ def clear_certified(api_client): def perform_sync(api_client, crc_config, repo=None, remote_params=None): """Perform a sync against the crc_client defined by crc_config. """ + remote_url = os.getenv( + "TEST_CRC_API_ROOT_SERVICE", + "http://insights-proxy:8080/api/automation-hub/" + ) + remote_auth_url = os.getenv( + "TEST_CRC_AUTH_URL_SERVICE", + "http://insights-proxy:8080/auth/realms/redhat-external/protocol/openid-connect/token" + ) remote_params = remote_params or {} - url = crc_config["url"] + if repo: - url = url + f"/content/{repo}/" + remote_url = remote_url + f"content/{repo}/" # pulp_ansible will only perform a sync if the remote source is updated # or if the remote itself is modified. Since the remote source doesn't @@ -55,8 +64,8 @@ def perform_sync(api_client, crc_config, repo=None, remote_params=None): "content/rh-certified/v3/sync/config/", method="PUT", args={ - "url": url, - "auth_url": crc_config["auth_url"], + "url": remote_url, + "auth_url": remote_auth_url, "token": crc_config["token"], **remote_params, } @@ -65,7 +74,11 @@ def perform_sync(api_client, crc_config, repo=None, remote_params=None): # Launch sync r = api_client( "content/rh-certified/v3/sync/", - method="POST" + method="POST", + args={ + "mirror": True, + "optimize": False, + } ) resp = {