From cb24731426f899368669f66cdc2490ebbe160595 Mon Sep 17 00:00:00 2001 From: bbrzyski Date: Tue, 17 Sep 2024 14:34:34 +0200 Subject: [PATCH] Run all python version dependant tests in parallel Remove stage for buidling docs Internal-tag: [#65799] Signed-off-by: bbrzyski --- .ci.yml | 59 +++++++++++++++++++----- .github/scripts/ci.sh | 12 ++--- .github/workflows/pipeline.yml | 82 +++++++++++++++++++++++++++++++--- noxfile.py | 11 ++--- 4 files changed, 136 insertions(+), 28 deletions(-) diff --git a/.ci.yml b/.ci.yml index 9c0a0609..f7bed684 100644 --- a/.ci.yml +++ b/.ci.yml @@ -3,36 +3,66 @@ stages: - test - - build_docs - deploy_docs variables: DEBIAN_FRONTEND: noninteractive - SCALENODE_RAM: 12288 - SCALENODE_CPU: 6 + SCALENODE_RAM: 10240 + SCALENODE_CPU: 8 + SCALENODE_DISK: 20 lint: variables: - SCALENODE_RAM: 2048 - SCALENODE_CPU: 2 + SCALENODE_RAM: 4096 + SCALENODE_CPU: 4 stage: test image: debian:bookworm script: - ./.github/scripts/ci.sh lint -tests: +test_python_3_8: + stage: test + image: debian:bookworm + script: + - ./.github/scripts/ci.sh test_python 3.8 + +test_python_3_9: + stage: test + image: debian:bookworm + script: + - ./.github/scripts/ci.sh test_python 3.9 + +test_python_3_10: stage: test image: debian:bookworm script: - - ./.github/scripts/ci.sh tests + - ./.github/scripts/ci.sh test_python 3.10 + +test_python_3_11: + stage: test + image: debian:bookworm + script: + - ./.github/scripts/ci.sh test_python 3.11 + +test_python_3_12: + stage: test + image: debian:bookworm + script: + - ./.github/scripts/ci.sh test_python 3.12 package_test: + variables: + SCALENODE_CPU: 4 + SCALENODE_RAM: 8192 stage: test image: debian:bookworm script: - ./.github/scripts/ci.sh package_dist examples: + variables: + SCALENODE_CPU: 6 + SCALENODE_RAM: 8192 stage: test image: debian:bookworm script: @@ -42,6 +72,9 @@ examples: - examples/**/build package_cores: + variables: + SCALENODE_CPU: 4 + SCALENODE_RAM: 8192 stage: test image: debian:bookworm script: @@ -52,8 +85,8 @@ package_cores: pyright_check: variables: - SCALENODE_RAM: 12288 - SCALENODE_CPU: 2 + SCALENODE_RAM: 6144 + SCALENODE_CPU: 4 stage: test image: debian:bookworm script: @@ -62,10 +95,10 @@ pyright_check: build_docs: variables: - SCALENODE_RAM: 2048 - SCALENODE_CPU: 2 + SCALENODE_RAM: 18432 + SCALENODE_CPU: 5 image: debian:bookworm - stage: build_docs + stage: test script: - ./.github/scripts/ci.sh docs - tar cf "$CI_DOCS_ARCHIVE" -C docs/build/html/ . @@ -78,6 +111,8 @@ deploy_docs: image: debian:bookworm variables: GIT_STRATEGY: none + SCALENODE_RAM: 2048 + SCALENODE_CPU: 1 dependencies: [ build_docs ] stage: deploy_docs tags: ['docs'] diff --git a/.github/scripts/ci.sh b/.github/scripts/ci.sh index 029867f5..f8930f9e 100755 --- a/.github/scripts/ci.sh +++ b/.github/scripts/ci.sh @@ -119,15 +119,15 @@ run_lint() { end_command_group } -run_tests() { +run_python_tests() { install_common_system_packages install_topwrap_system_deps install_interconnect_test_system_deps install_nox install_pyenv - begin_command_group "Run Python tests" - log_cmd nox -s tests_in_env + begin_command_group "Run Python $1 tests" + log_cmd nox -s tests_in_env -- "$1" end_command_group } @@ -203,8 +203,8 @@ case "$1" in lint) run_lint ;; -tests) - run_tests +test_python) + run_python_tests "$2" ;; examples) generate_examples @@ -222,6 +222,6 @@ docs) generate_docs ;; *) - echo "Usage: $0 {lint|tests|examples|package_cores|package_dist|docs|pyright_check}" + echo "Usage: $0 {lint|test_python|examples|package_cores|package_dist|docs|pyright_check}" ;; esac diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index c04c43a5..7f838da4 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -3,7 +3,7 @@ name: Pipeline -on: +on: pull_request: push: workflow_dispatch: @@ -57,11 +57,11 @@ jobs: run: | ./.github/scripts/ci.sh pyright_check - tests: + test_python_3_8: runs-on: ubuntu-latest container: image: debian:bookworm - name: "Run tests" + name: "Run tests on python 3.8" steps: - name: Install git package @@ -71,9 +71,81 @@ jobs: - uses: actions/checkout@v4 - - name: Run Python tests + - name: Run Python tests on python 3.8 run: | - ./.github/scripts/ci.sh tests + ./.github/scripts/ci.sh test_python 3.8 + + test_python_3_9: + runs-on: ubuntu-latest + container: + image: debian:bookworm + name: "Run tests on python 3.9" + + steps: + - name: Install git package + run: | + apt-get update -qq + apt-get install -y git + + - uses: actions/checkout@v4 + + - name: Run Python tests on python 3.9 + run: | + ./.github/scripts/ci.sh test_python 3.9 + + test_python_3_10: + runs-on: ubuntu-latest + container: + image: debian:bookworm + name: "Run tests on python 3.10" + + steps: + - name: Install git package + run: | + apt-get update -qq + apt-get install -y git + + - uses: actions/checkout@v4 + + - name: Run Python tests on python 3.10 + run: | + ./.github/scripts/ci.sh test_python 3.10 + + test_python_3_11: + runs-on: ubuntu-latest + container: + image: debian:bookworm + name: "Run tests on python 3.11" + + steps: + - name: Install git package + run: | + apt-get update -qq + apt-get install -y git + + - uses: actions/checkout@v4 + + - name: Run Python tests on python 3.11 + run: | + ./.github/scripts/ci.sh test_python 3.11 + + test_python_3_12: + runs-on: ubuntu-latest + container: + image: debian:bookworm + name: "Run tests on python 3.12" + + steps: + - name: Install git package + run: | + apt-get update -qq + apt-get install -y git + + - uses: actions/checkout@v4 + + - name: Run Python tests on python 3.12 + run: | + ./.github/scripts/ci.sh test_python 3.12 package_test: runs-on: ubuntu-latest diff --git a/noxfile.py b/noxfile.py index 0a9d9e62..41b5d66e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -61,7 +61,7 @@ def update_test_data(session: nox.Session) -> None: session.run("python3", "tests/update_test_data.py", *tests_to_update) -def prepare_pyenv(session: nox.Session) -> dict: +def prepare_pyenv(session: nox.Session, python_versions: List[str]) -> Dict[str, str]: env = os.environ.copy() path = env.get("PATH") @@ -82,14 +82,14 @@ def prepare_pyenv(session: nox.Session) -> dict: ) # Install required Python versions if these don't exist - for ver in PYTHON_VERSIONS: + for ver in python_versions: if not shutil.which(f"python{ver}", path=path): session.log(f"Installing Python {ver}") session.run("pyenv", "install", ver, external=True, env=env) # Detect which versions are provided by Pyenv pythons_in_pyenv = [] - for ver in PYTHON_VERSIONS: + for ver in python_versions: if shutil.which(f"python{ver}", path=pyenv_shims): pythons_in_pyenv += [ver] @@ -103,8 +103,9 @@ def prepare_pyenv(session: nox.Session) -> dict: @nox.session def tests_in_env(session: nox.Session) -> None: - env = prepare_pyenv(session) - session.run("nox", "-s", "tests", external=True, env=env) + python_versions = PYTHON_VERSIONS if not session.posargs else session.posargs + env = prepare_pyenv(session, python_versions) + session.run("nox", "-s", "tests", "-p", *python_versions, external=True, env=env) @nox.session