From 124da3dcef0d95a6f6aa76ed849f47ada25b83e2 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 1 Jul 2024 15:11:21 +0530 Subject: [PATCH 1/5] Replace deprecated macos CI runners Replace macos-11 runners with macos-14 runners. Reference: https://github.com/actions/runner-images?tab=readme-ov-file#available-images Reference: https://github.com/nexB/skeleton/issues/89 Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 373b78cd..c2a3b522 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -25,24 +25,24 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: macos11_cpython - image_name: macOS-11 + job_name: macos12_cpython + image_name: macOS-12 python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: - job_name: macos12_cpython - image_name: macOS-12 + job_name: macos13_cpython + image_name: macOS-13 python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: - job_name: macos13_cpython - image_name: macOS-13 + job_name: macos14_cpython_arm64 + image_name: macOS-14 python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -50,8 +50,8 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos14_cpython - image_name: macOS-14 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + image_name: macOS-14-large + python_versions: ['3.8', '3.8', '3.9', '3.10', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs From be4e14d414cf4f7112b529dc71f7abccc9dcf24a Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 1 Jul 2024 16:00:40 +0530 Subject: [PATCH 2/5] Update minimum required python version to 3.8 Signed-off-by: Ayan Sinha Mahapatra --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index bd0e58a7..a8e20c5d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ zip_safe = false setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.7 +python_requires = >=3.8 install_requires = From ccb6e1cd2fd0aaf37989acebbebf9c0ab63fbe11 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 6 Aug 2024 16:06:41 -0700 Subject: [PATCH 3/5] Update azure-pipelines files #520 Signed-off-by: Jono Yang --- azure-pipelines.yml | 68 ++++++++++-------------------------------- etc/ci/azure-posix.yml | 7 +++-- 2 files changed, 20 insertions(+), 55 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c2a3b522..55430a9c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,68 +5,32 @@ # These jobs are using VMs with Azure-provided Python builds ################################################################################ +resources: + containers: + - container: postgres + image: postgres + env: + POSTGRES_DB: packagedb + POSTGRES_USER: packagedb + POSTGRES_PASSWORD: packagedb + POSTGRES_INITDB_ARGS: "--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8" + ports: + - 5432:5432 + jobs: - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.10', '3.11', '3.12'] test_suites: - all: venv/bin/pytest -n 2 -vvs + all: make test - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu22_cpython image_name: ubuntu-22.04 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos12_cpython - image_name: macOS-12 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos13_cpython - image_name: macOS-13 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos14_cpython_arm64 - image_name: macOS-14 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos14_cpython - image_name: macOS-14-large - python_versions: ['3.8', '3.8', '3.9', '3.10', '3.12'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-win.yml - parameters: - job_name: win2019_cpython - image_name: windows-2019 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] - test_suites: - all: venv\Scripts\pytest -n 2 -vvs - - - template: etc/ci/azure-win.yml - parameters: - job_name: win2022_cpython - image_name: windows-2022 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.10', '3.11', '3.12'] test_suites: - all: venv\Scripts\pytest -n 2 -vvs + all: make test diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml index 9fdc7f15..e626e032 100644 --- a/etc/ci/azure-posix.yml +++ b/etc/ci/azure-posix.yml @@ -30,9 +30,10 @@ jobs: displayName: '${{ pyver }} - Install Python' - script: | - python${{ pyver }} --version - echo "python${{ pyver }}" > PYTHON_EXECUTABLE - ./configure --clean && ./configure --dev + make dev + make envfile_testing + sudo mkdir /etc/scancodeio + sudo cp .env /etc/scancodeio displayName: '${{ pyver }} - Configure' - script: $(test_suite) From 3eaeacc2f9d25fe2230da24e6ea675c0eb68c021 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 7 Aug 2024 14:03:03 -0700 Subject: [PATCH 4/5] Update envfile_testing Make command #520 Signed-off-by: Jono Yang --- Makefile | 13 ++++++------- azure-pipelines.yml | 16 +++++++--------- etc/ci/azure-posix.yml | 5 +++++ purldb_project/settings.py | 1 - 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 691ba6fc..10afe268 100644 --- a/Makefile +++ b/Makefile @@ -50,12 +50,11 @@ envfile: @mkdir -p $(shell dirname ${ENV_FILE}) && touch ${ENV_FILE} @echo SECRET_KEY=\"${GET_SECRET_KEY}\" > ${ENV_FILE} -envfile_testing: - @echo "-> Create the .env file and generate a secret key" - @if test -f ${ENV_FILE}; then echo ".env file exists already"; exit 1; fi - @mkdir -p $(shell dirname ${ENV_FILE}) && touch ${ENV_FILE} - @echo SECRET_KEY=\"${GET_SECRET_KEY}\" >> ${ENV_FILE} - @echo SCANCODEIO_DB_PORT=\"5433\" >> ${ENV_FILE} +envfile_testing: envfile + @echo PACKAGEDB_DB_USER=\"postgres\" >> ${ENV_FILE} + @echo PACKAGEDB_DB_PASSWORD=\"postgres\" >> ${ENV_FILE} + @echo SCANCODEIO_DB_USER=\"postgres\" >> ${ENV_FILE} + @echo SCANCODEIO_DB_PASSWORD=\"postgres\" >> ${ENV_FILE} isort: @echo "-> Apply isort changes to ensure proper imports ordering" @@ -126,7 +125,7 @@ run_map: test: @echo "-> Run the test suite" - ${ACTIVATE} DJANGO_SETTINGS_MODULE=purldb_project.settings ${PYTHON_EXE} -m pytest -vvs --ignore matchcode_pipeline --ignore matchcode_project --ignore purldb-toolkit --ignore packagedb/tests/test_throttling.py + ${ACTIVATE} DJANGO_SETTINGS_MODULE=purldb_project.settings ${PYTHON_EXE} -m pytest -x -vvs --ignore matchcode_pipeline --ignore matchcode_project --ignore purldb-toolkit --ignore packagedb/tests/test_throttling.py ${ACTIVATE} DJANGO_SETTINGS_MODULE=purldb_project.settings ${PYTHON_EXE} -m pytest -vvs packagedb/tests/test_throttling.py ${ACTIVATE} DJANGO_SETTINGS_MODULE=matchcode_project.settings ${PYTHON_EXE} -m pytest -vvs matchcode_pipeline ${ACTIVATE} ${PYTHON_EXE} -m pytest -vvs purldb-toolkit/ diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 55430a9c..23faf951 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,15 +7,13 @@ resources: containers: - - container: postgres - image: postgres - env: - POSTGRES_DB: packagedb - POSTGRES_USER: packagedb - POSTGRES_PASSWORD: packagedb - POSTGRES_INITDB_ARGS: "--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8" - ports: - - 5432:5432 + - container: postgres + image: postgres:13 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 jobs: diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml index e626e032..b139a66c 100644 --- a/etc/ci/azure-posix.yml +++ b/etc/ci/azure-posix.yml @@ -18,6 +18,9 @@ jobs: test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} + services: + postgres: postgres + steps: - checkout: self fetchDepth: 10 @@ -32,6 +35,8 @@ jobs: - script: | make dev make envfile_testing + sudo mkdir /etc/purldb + sudo cp .env /etc/purldb sudo mkdir /etc/scancodeio sudo cp .env /etc/scancodeio displayName: '${{ pyver }} - Configure' diff --git a/purldb_project/settings.py b/purldb_project/settings.py index 7651b0a0..a9592932 100644 --- a/purldb_project/settings.py +++ b/purldb_project/settings.py @@ -107,7 +107,6 @@ ) # Database - DATABASES = { 'default': { 'ENGINE': env.str('PACKAGEDB_DB_ENGINE', 'django.db.backends.postgresql'), From 9a6f988be8d3598a6374f303952d83d1bf9d7c70 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 7 Aug 2024 17:22:06 -0700 Subject: [PATCH 5/5] Do not run tests using github actions #520 Signed-off-by: Jono Yang --- .github/workflows/purldb-tests.yml | 71 ------------------------------ Makefile | 2 +- 2 files changed, 1 insertion(+), 72 deletions(-) delete mode 100644 .github/workflows/purldb-tests.yml diff --git a/.github/workflows/purldb-tests.yml b/.github/workflows/purldb-tests.yml deleted file mode 100644 index a88f4026..00000000 --- a/.github/workflows/purldb-tests.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: PurlDB Tests CI - -on: [push, pull_request] - -env: - POSTGRES_DB: packagedb - POSTGRES_USER: packagedb - POSTGRES_PASSWORD: packagedb - POSTGRES_INITDB_ARGS: --encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 - -jobs: - build: - runs-on: ubuntu-20.04 - - services: - postgres1: - image: postgres:13 - env: - POSTGRES_DB: ${{ env.POSTGRES_DB }} - POSTGRES_USER: ${{ env.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }} - POSTGRES_INITDB_ARGS: ${{ env.POSTGRES_INITDB_ARGS }} - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - postgres2: - image: postgres:13 - env: - POSTGRES_DB: matchcodeio - POSTGRES_USER: matchcodeio - POSTGRES_PASSWORD: matchcodeio - POSTGRES_INITDB_ARGS: ${{ env.POSTGRES_INITDB_ARGS }} - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5433:5432 - - strategy: - max-parallel: 4 - matrix: - python-version: ["3.10", "3.11"] - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - working-directory: . - run: | - make dev - - - name: Run tests - working-directory: . - run: | - make envfile_testing - sudo mkdir /etc/scancodeio - sudo cp .env /etc/scancodeio - make test diff --git a/Makefile b/Makefile index 10afe268..d4554cc7 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,7 @@ run_map: test: @echo "-> Run the test suite" - ${ACTIVATE} DJANGO_SETTINGS_MODULE=purldb_project.settings ${PYTHON_EXE} -m pytest -x -vvs --ignore matchcode_pipeline --ignore matchcode_project --ignore purldb-toolkit --ignore packagedb/tests/test_throttling.py + ${ACTIVATE} DJANGO_SETTINGS_MODULE=purldb_project.settings ${PYTHON_EXE} -m pytest -vvs --ignore matchcode_pipeline --ignore matchcode_project --ignore purldb-toolkit --ignore packagedb/tests/test_throttling.py ${ACTIVATE} DJANGO_SETTINGS_MODULE=purldb_project.settings ${PYTHON_EXE} -m pytest -vvs packagedb/tests/test_throttling.py ${ACTIVATE} DJANGO_SETTINGS_MODULE=matchcode_project.settings ${PYTHON_EXE} -m pytest -vvs matchcode_pipeline ${ACTIVATE} ${PYTHON_EXE} -m pytest -vvs purldb-toolkit/