From 767254dff8778ce50e678eaaec9ec937d08c9e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Je=C5=99=C3=A1bek=20=28Jiri=20Jerabek=29?= Date: Thu, 28 Nov 2024 20:09:59 +0100 Subject: [PATCH] Community tests in docker compose CI (#2373) * init ci docker compose integration file with community profile * dont run other CI for now * dont run sonar as well * add django browsable api and fix test * fix tests with browsable api * run commands inside docker compose profile * run less CI * fix browsable api mistake * remove comment * first checkin * fix test_role_import_overrides * cleanup * remove unused and revert * run ruff and flake * dont run community in oci env --- .../ci-docker-compose-integration.yml | 74 +++++++++++++++++++ .github/workflows/ci_oci-env-integration.yml | 2 +- .gitignore | 2 + dev/compose/community.yaml | 13 ++-- docker/etc/settings.py | 5 +- .../tests/integration/api/test_ui_paths.py | 9 +-- .../integration/cli/test_community_sync.py | 9 +-- .../community/test_community_api.py | 4 +- .../community/test_role_import_overrides.py | 12 ++- .../integration/community/test_v1_api.py | 8 +- galaxy_ng/tests/integration/conftest.py | 18 +++++ galaxy_ng/tests/integration/utils/legacy.py | 50 +++++++++---- integration_requirements.txt | 3 +- 13 files changed, 163 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/ci-docker-compose-integration.yml diff --git a/.github/workflows/ci-docker-compose-integration.yml b/.github/workflows/ci-docker-compose-integration.yml new file mode 100644 index 0000000000..2cf2d5b4b6 --- /dev/null +++ b/.github/workflows/ci-docker-compose-integration.yml @@ -0,0 +1,74 @@ +--- +name: Docker Compose Integration +on: + pull_request: + branches: + - '**' + push: + branches: + - '**' + workflow_dispatch: + +jobs: + integration: + strategy: + fail-fast: false + matrix: + env: +# - TEST_PROFILE: ldap +# - TEST_PROFILE: keycloak +# - TEST_PROFILE: standalone +# # - TEST_PROFILE: rbac +# - TEST_PROFILE: rbac_parallel_group_1 +# - TEST_PROFILE: rbac_parallel_group_2 +# - TEST_PROFILE: certified-sync +# - TEST_PROFILE: insights +# - TEST_PROFILE: iqe_rbac +# - TEST_PROFILE: x_repo_search + - TEST_PROFILE: community +# - TEST_PROFILE: dab_jwt + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: (Linux) Install docker compose + run: | + curl -L -o /tmp/docker-compose https://github.com/docker/compose/releases/download/v2.29.1/docker-compose-linux-x86_64 + install /tmp/docker-compose /usr/local/bin/ + + - name: Spin up dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml + run: | + docker compose -f dev/compose/community.yaml up --detach + + - name: Export environment variables to host + run: | + docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml exec manager /bin/bash -c 'env | grep -v -w "HOME"' >> $GITHUB_ENV + cat $GITHUB_ENV + + - name: Wait for API online status + run: | + max_runs=10 + for i in $(seq 1 $max_runs); do + echo "$i: checking api status..." + RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -u "${{ env.DJANGO_SUPERUSER_USERNAME }}:${{ env.DJANGO_SUPERUSER_PASSWORD }}" -LI http://localhost:5001/api/galaxy/pulp/api/v3/status/) + if [ "$RESPONSE" -ne 200 ]; then + echo "API is down. Retrying in 10 seconds..." + sleep 10 + else + echo "API online." + exit 0 + fi + done + + - name: Install integration requirements + run: | + pip install -r integration_requirements.txt + + - name: Run integration tests + run: | + pytest -v -r sx --color=yes -m '${{ env.HUB_TEST_MARKS }}' galaxy_ng/tests/integration \ No newline at end of file diff --git a/.github/workflows/ci_oci-env-integration.yml b/.github/workflows/ci_oci-env-integration.yml index 45e6d3b654..8558fa0c56 100644 --- a/.github/workflows/ci_oci-env-integration.yml +++ b/.github/workflows/ci_oci-env-integration.yml @@ -25,7 +25,7 @@ jobs: - TEST_PROFILE: insights - TEST_PROFILE: iqe_rbac - TEST_PROFILE: x_repo_search - - TEST_PROFILE: community + # - TEST_PROFILE: community - TEST_PROFILE: dab_jwt runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 0d2485c730..9584dba2f2 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,8 @@ gng_unit_testing/ # Galaxy NG db_snapshots/ +temp_roles/* +.gitconfig # oci-env profile files .compiled/ diff --git a/dev/compose/community.yaml b/dev/compose/community.yaml index 72984732aa..9ff409f59e 100644 --- a/dev/compose/community.yaml +++ b/dev/compose/community.yaml @@ -30,9 +30,9 @@ x-common-env: &common-env # Hostname and prefix has to be correct PULP_GALAXY_API_PATH_PREFIX: '/api/' PULP_CONTENT_PATH_PREFIX: '/pulp/content/' - PULP_ANSIBLE_API_HOSTNAME: 'https://localhost' - PULP_ANSIBLE_CONTENT_HOSTNAME: "https://localhost" - PULP_CONTENT_ORIGIN: "https://localhost" + PULP_ANSIBLE_API_HOSTNAME: 'http://localhost:5001' + PULP_ANSIBLE_CONTENT_HOSTNAME: "http://localhost:5001" + PULP_CONTENT_ORIGIN: "http://localhost:5001" PULP_CSRF_TRUSTED_ORIGINS: "['https://localhost']" # auth ... @@ -59,7 +59,7 @@ x-common-env: &common-env PULP_GALAXY_AUTO_SIGN_COLLECTIONS: 'false' # disable user/group modifications - PULP_ALLOW_LOCAL_RESOURCE_MANAGEMENT: 'false' + PULP_ALLOW_LOCAL_RESOURCE_MANAGEMENT: 'true' # role content workaround PULP_ANSIBLE_BASE_ROLES_REQUIRE_VIEW: 'false' @@ -74,8 +74,9 @@ x-common-env: &common-env # PULP_RESOURCE_SERVER__VALIDATE_HTTPS='false' # Integration test settings - HUB_TEST_AUTHENTICATION_BACKEND: "community" - HUB_TEST_MARKS: "deployment_community" + HUB_TEST_AUTHENTICATION_BACKEND: 'community' + HUB_TEST_MARKS: 'deployment_community' + HUB_API_ROOT: 'http://localhost:5001/api/' # Unpin dependencies on setup.py if set to 0 LOCK_REQUIREMENTS: 0 diff --git a/docker/etc/settings.py b/docker/etc/settings.py index 7d2f22728c..a5c1912361 100644 --- a/docker/etc/settings.py +++ b/docker/etc/settings.py @@ -39,7 +39,10 @@ REDIS_HOST = os.environ.get('PULP_REDIS_HOST') REDIS_PORT = os.environ.get('PULP_REDIS_PORT') -REST_FRAMEWORK__DEFAULT_RENDERER_CLASSES = ['rest_framework.renderers.JSONRenderer'] +REST_FRAMEWORK__DEFAULT_RENDERER_CLASSES = [ + 'rest_framework.renderers.JSONRenderer', + 'galaxy_ng.app.renderers.CustomBrowsableAPIRenderer' +] _enabled_handlers = ['console'] _extra_handlers = {} diff --git a/galaxy_ng/tests/integration/api/test_ui_paths.py b/galaxy_ng/tests/integration/api/test_ui_paths.py index 787044b152..b28efce596 100644 --- a/galaxy_ng/tests/integration/api/test_ui_paths.py +++ b/galaxy_ng/tests/integration/api/test_ui_paths.py @@ -1,6 +1,5 @@ import random import json -import subprocess from http import HTTPStatus import pytest @@ -874,7 +873,7 @@ def build_upload_wait(tags): # /api/automation-hub/_ui/v1/tags/roles/ @pytest.mark.deployment_community -def test_api_ui_v1_tags_roles(ansible_config): +def test_api_ui_v1_tags_roles(ansible_config, docker_compose_exec): """Test endpoint's sorting and filtering""" def _sync_role(github_user, role_name): @@ -886,11 +885,7 @@ def _sync_role(github_user, role_name): wait_for_v1_task(resp=resp, api_client=api_admin_client) def _populate_tags_cmd(): - proc = subprocess.run( - "django-admin populate-role-tags", - shell=True, - capture_output=True, - ) + proc = docker_compose_exec('django-admin populate-role-tags') assert proc.returncode == 0 config = ansible_config("basic_user") diff --git a/galaxy_ng/tests/integration/cli/test_community_sync.py b/galaxy_ng/tests/integration/cli/test_community_sync.py index a2d00f9365..289231e7e3 100644 --- a/galaxy_ng/tests/integration/cli/test_community_sync.py +++ b/galaxy_ng/tests/integration/cli/test_community_sync.py @@ -4,7 +4,6 @@ import json import pytest import requests -import subprocess from ansible.galaxy.api import GalaxyError @@ -15,7 +14,7 @@ @pytest.mark.deployment_community -def test_community_collection_download_count_sync(ansible_config): +def test_community_collection_download_count_sync(ansible_config, docker_compose_exec): """ Test collection download count sync command """ # FIXME(jctanner): once beta switches over, this test is no longer needed. @@ -83,11 +82,7 @@ def test_community_collection_download_count_sync(ansible_config): wait_for_task(api_client, sync_task) # run the django command - pid = subprocess.run( - 'pulpcore-manager sync-collection-download-counts', - shell=True, - capture_output=True, - ) + pid = docker_compose_exec('pulpcore-manager sync-collection-download-counts') assert pid.returncode == 0 # check the counter in the api diff --git a/galaxy_ng/tests/integration/community/test_community_api.py b/galaxy_ng/tests/integration/community/test_community_api.py index e538cb87fd..8a6354494d 100644 --- a/galaxy_ng/tests/integration/community/test_community_api.py +++ b/galaxy_ng/tests/integration/community/test_community_api.py @@ -813,10 +813,10 @@ def test_v1_role_versions(ansible_config): id = resp["results"][0]["id"] versions = resp["results"][0]["summary_fields"]["versions"] - resp = api_client(f'/api/v1/roles/{id}/versions') + resp = api_client(f'/api/v1/roles/{id}/versions/') assert resp["count"] >= len(versions) with pytest.raises(AnsibleError) as html: - api_client(f"v1/roles/{id}/versions", headers={"Accept": "text/html"}) + api_client(f"v1/roles/{id}/versions/", headers={"Accept": "text/html"}) assert not isinstance(html.value, dict) assert "results" in str(html.value) diff --git a/galaxy_ng/tests/integration/community/test_role_import_overrides.py b/galaxy_ng/tests/integration/community/test_role_import_overrides.py index 3798ba726b..ad03832729 100644 --- a/galaxy_ng/tests/integration/community/test_role_import_overrides.py +++ b/galaxy_ng/tests/integration/community/test_role_import_overrides.py @@ -39,11 +39,10 @@ 'alternate_namespace_name': None, 'alternate_role_name': None, } - ] ) @pytest.mark.deployment_community -def test_role_import_overrides(ansible_config, spec): +def test_role_import_overrides(ansible_config, spec, docker_compose_exec): """" Validate setting namespace in meta/main.yml does the right thing """ admin_config = ansible_config("admin") @@ -83,10 +82,12 @@ def test_role_import_overrides(ansible_config, spec): 'meta_namespace': spec['meta_namespace'], 'meta_name': spec['meta_name'], } - lr = LegacyRoleGitRepoBuilder(**builder_kwargs) + lr = LegacyRoleGitRepoBuilder(**builder_kwargs, docker_compose_exec=docker_compose_exec) + + local_tmp_dir = lr.role_cont_dir # run the import - payload = {'alternate_clone_url': lr.role_dir} + payload = {'alternate_clone_url': local_tmp_dir} for key in ['github_user', 'github_repo', 'alternate_namespace_name', 'alternate_role_name']: if spec.get(key): payload[key] = spec[key] @@ -102,3 +103,6 @@ def test_role_import_overrides(ansible_config, spec): assert roles_search['results'][0]['name'] == spec['name'] assert roles_search['results'][0]['github_user'] == spec['github_user'] assert roles_search['results'][0]['github_repo'] == spec['github_repo'] + + # cleanup + lr.local_roles_cleanup() diff --git a/galaxy_ng/tests/integration/community/test_v1_api.py b/galaxy_ng/tests/integration/community/test_v1_api.py index dac5e2c5d9..9b2d9f064e 100644 --- a/galaxy_ng/tests/integration/community/test_v1_api.py +++ b/galaxy_ng/tests/integration/community/test_v1_api.py @@ -133,7 +133,7 @@ def test_custom_browsable_format(ansible_config): require_auth=True, ) - resp = api_client("v1/namespaces") + resp = api_client("v1/namespaces/") assert isinstance(resp, dict) assert "results" in resp @@ -142,7 +142,7 @@ def test_custom_browsable_format(ansible_config): assert "results" in resp with pytest.raises(AnsibleError) as html: - api_client("v1/namespaces", headers={"Accept": "text/html"}) + api_client("v1/namespaces/", headers={"Accept": "text/html"}) assert not isinstance(html.value, dict) assert "results" in str(html.value) @@ -154,7 +154,7 @@ def test_custom_browsable_format(ansible_config): require_auth=True, ) - resp = api_client("v1/namespaces") + resp = api_client("v1/namespaces/") assert isinstance(resp, dict) assert "results" in resp @@ -163,6 +163,6 @@ def test_custom_browsable_format(ansible_config): assert "results" in resp with pytest.raises(AnsibleError) as html: - api_client("v1/namespaces", headers={"Accept": "text/html"}) + api_client("v1/namespaces/", headers={"Accept": "text/html"}) assert not isinstance(html.value, dict) assert "results" in str(html.value) diff --git a/galaxy_ng/tests/integration/conftest.py b/galaxy_ng/tests/integration/conftest.py index a6d44e53e9..efa272dc7a 100644 --- a/galaxy_ng/tests/integration/conftest.py +++ b/galaxy_ng/tests/integration/conftest.py @@ -3,6 +3,7 @@ import os import shutil import yaml +import subprocess import pytest from orionutils.utils import increment_version @@ -726,3 +727,20 @@ def skip_if_require_signature_for_approval(): def skip_if_not_require_signature_for_approval(): if not require_signature_for_approval(): pytest.skip("This test needs refactoring to work with signatures required on move.") + + +@pytest.fixture +def docker_compose_exec(): + def _exec(cmd: str, cwd=None): + cd = '' + if cwd is not None: + cd = f'cd {cwd};' + + proc = subprocess.run( + f"docker exec compose-manager-1 /bin/bash -c '{cd}{cmd}'", + shell=True, + capture_output=True, + ) + return proc + + return _exec diff --git a/galaxy_ng/tests/integration/utils/legacy.py b/galaxy_ng/tests/integration/utils/legacy.py index 3a0c9603bc..bf7a56e7fa 100644 --- a/galaxy_ng/tests/integration/utils/legacy.py +++ b/galaxy_ng/tests/integration/utils/legacy.py @@ -234,32 +234,51 @@ def __init__( namespace=None, name=None, meta_namespace=None, - meta_name=None + meta_name=None, + docker_compose_exec=None ): self.namespace = namespace self.name = name self.meta_namespace = meta_namespace self.meta_name = meta_name - self.workdir = tempfile.mkdtemp(prefix='gitrepo_') + self.docker_compose_exec = docker_compose_exec + + # local tmp dir for roles + self.temp_roles = 'temp_roles/' + self.local_roles_cleanup() + if not os.path.exists(self.temp_roles): + os.makedirs(self.temp_roles) + + self.workdir = tempfile.mkdtemp(prefix='gitrepo_', dir=self.temp_roles) + path_parts = self.workdir.partition(self.temp_roles) + + # should be equal to HOME=/app + # TODO(jjerabek): better way to get env var from container? + pid = self.docker_compose_exec('printenv HOME') + home = pid.stdout.decode('utf-8').strip() or '/app' + + self.workdir_cont = os.path.join(home, path_parts[1], path_parts[2]) + self.role_dir = None + self.role_cont_dir = None self.role_init() self.role_edit() self.git_init() self.git_commit() - self.fix_perms() - - def fix_perms(self): - subprocess.run(f'chown -R pulp:pulp {self.workdir}', shell=True) - def role_init(self): cmd = f'ansible-galaxy role init {self.namespace}.{self.name}' self.role_dir = os.path.join(self.workdir, self.namespace + '.' + self.name) - pid = subprocess.run(cmd, shell=True, cwd=self.workdir) + self.role_cont_dir = os.path.join(self.workdir_cont, self.namespace + '.' + self.name) + + pid = subprocess.run(cmd, shell=True, cwd=self.workdir, capture_output=True) + assert pid.returncode == 0 assert os.path.exists(self.role_dir) + ag_init_stdout = f'- Role {self.namespace}.{self.name} was created successfully' + assert pid.stdout.decode('utf-8').strip() == ag_init_stdout def role_edit(self): if self.meta_namespace or self.meta_name: @@ -277,12 +296,17 @@ def role_edit(self): f.write(yaml.dump(meta)) def git_init(self): - subprocess.run('git init', shell=True, cwd=self.role_dir) + self.docker_compose_exec('git init', cwd=self.role_cont_dir) + + # hack to make git inside git dir work + self.docker_compose_exec(f'git config --global --add safe.directory {self.role_cont_dir}') def git_commit(self): + self.docker_compose_exec('git config --global user.email "root@localhost"') + self.docker_compose_exec('git config --global user.name "root at localhost"') - subprocess.run('git config --global user.email "root@localhost"', shell=True) - subprocess.run('git config --global user.name "root at localhost"', shell=True) + self.docker_compose_exec('git add *', cwd=self.role_cont_dir) + self.docker_compose_exec('git commit -m "first checkin"', cwd=self.role_cont_dir) - subprocess.run('git add *', shell=True, cwd=self.role_dir) - subprocess.run('git commit -m "first checkin"', shell=True, cwd=self.role_dir) + def local_roles_cleanup(self): + self.docker_compose_exec(f'rm -rf {self.temp_roles}') diff --git a/integration_requirements.txt b/integration_requirements.txt index f48183403e..3a7ed8ecf0 100644 --- a/integration_requirements.txt +++ b/integration_requirements.txt @@ -11,4 +11,5 @@ importlib_resources galaxykit @ git+https://github.com/ansible/galaxykit pyyaml pytest-subtests -ldap3 \ No newline at end of file +ldap3 +setuptools \ No newline at end of file