From b44e3e99ddcc745c661bc175534b09022eaabf1b Mon Sep 17 00:00:00 2001 From: dafnapension <46454972+dafnapension@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:46:46 +0200 Subject: [PATCH] Make GitHub Actions tests faster and clearer * commit again same changes needed Signed-off-by: dafnapension * maybe Signed-off-by: dafnapension * maybe Signed-off-by: dafnapension * maybe Signed-off-by: dafnapension * removed the reporter following job Signed-off-by: dafnapension * Add minor change to github action Signed-off-by: elronbandel * Make sure there are no more then one job running for each pull request at the same time Signed-off-by: elronbandel * Fix Signed-off-by: elronbandel * Update Signed-off-by: elronbandel * Update Signed-off-by: elronbandel * Shorten jobs names for better visability Signed-off-by: elronbandel --------- Signed-off-by: dafnapension Signed-off-by: elronbandel Co-authored-by: elronbandel --- .github/workflows/catalog_consistency.yml | 6 +- .github/workflows/catalog_preparation.yml | 70 +++++++++++-------- .github/workflows/docs.yml | 8 ++- .github/workflows/examples_tests.yml | 6 +- .github/workflows/inference_tests.yml | 6 +- .../library_eager_execution_tests.yml | 6 +- .github/workflows/library_tests.yml | 6 +- .github/workflows/performance.yml | 6 +- .github/workflows/quality.yml | 6 +- .github/workflows/test_helm.yml | 6 +- tests/catalog/test_preparation.py | 25 +++++-- 11 files changed, 108 insertions(+), 43 deletions(-) diff --git a/.github/workflows/catalog_consistency.yml b/.github/workflows/catalog_consistency.yml index 20de8d87f..0cb796086 100644 --- a/.github/workflows/catalog_consistency.yml +++ b/.github/workflows/catalog_consistency.yml @@ -6,8 +6,12 @@ on: pull_request: branches: [ main ] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-catalog_consistency.yml + cancel-in-progress: true + jobs: - run-catalog-consistency: + consistency: runs-on: ubuntu-latest env: diff --git a/.github/workflows/catalog_preparation.yml b/.github/workflows/catalog_preparation.yml index 3968d8309..54c45e2eb 100644 --- a/.github/workflows/catalog_preparation.yml +++ b/.github/workflows/catalog_preparation.yml @@ -1,34 +1,48 @@ name: Test Catalog Preparation on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - run-catalog-preparation: - - runs-on: ubuntu-latest - env: - OS: ubuntu-latest - UNITXT_DEFAULT_VERBOSITY: error - DATASETS_VERBOSITY: error - HF_HUB_VERBOSITY: error - HF_DATASETS_DISABLE_PROGRESS_BARS: "True" - TQDM_DISABLE: "True" + push: + branches: [ main ] + pull_request: + branches: [ main ] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-catalog_preparation.yml + cancel-in-progress: true - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - run: curl -LsSf https://astral.sh/uv/install.sh | sh - - run: uv pip install --system ".[tests]" - - run: huggingface-cli login --token ${{ secrets.UNITXT_READ_HUGGINGFACE_HUB_FOR_TESTS }} +jobs: + preparation: + + runs-on: ubuntu-latest + env: + OS: ubuntu-latest + UNITXT_DEFAULT_VERBOSITY: error + DATASETS_VERBOSITY: error + HF_HUB_VERBOSITY: error + HF_DATASETS_DISABLE_PROGRESS_BARS: "True" + TQDM_DISABLE: "True" + + strategy: + matrix: + modulo: [0,1,2,3,4,5,6,7] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - run: curl -LsSf https://astral.sh/uv/install.sh | sh + - run: uv pip install --system ".[tests]" + - run: huggingface-cli login --token ${{ secrets.UNITXT_READ_HUGGINGFACE_HUB_FOR_TESTS }} + + - name: Run Tests + run: | + modulo="${{ matrix.modulo }}" + echo "modulo=${modulo}" >> $GITHUB_STEP_SUMMARY + echo "sed -i 's/^num_par = 1 /num_par = 8 /' tests/catalog/test_preparation.py" > sedit.sh + echo "sed -i 's/^modulo = 0/modulo = ${modulo}/' tests/catalog/test_preparation.py" >> sedit.sh + sh sedit.sh + python -m unittest tests.catalog.test_preparation - - name: Run Tests - run: python -m unittest discover -s tests/catalog -p "test_*.py" \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 764a4d399..735e54638 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,8 +6,12 @@ on: pull_request: branches: [ main ] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-docs.yml + cancel-in-progress: true + jobs: - compile-docs: + docs: runs-on: ubuntu-latest env: @@ -20,7 +24,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.9' - + - run: curl -LsSf https://astral.sh/uv/install.sh | sh - run: uv pip install --system ".[tests,docs]" diff --git a/.github/workflows/examples_tests.yml b/.github/workflows/examples_tests.yml index d9d9704a3..1d98c0f47 100644 --- a/.github/workflows/examples_tests.yml +++ b/.github/workflows/examples_tests.yml @@ -9,8 +9,12 @@ on: release: types: [published] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-examples_tests.yml + cancel-in-progress: true + jobs: - run-full-tests-suit: + examples: runs-on: ubuntu-latest env: diff --git a/.github/workflows/inference_tests.yml b/.github/workflows/inference_tests.yml index 7361ebdf9..7bdb8bebd 100644 --- a/.github/workflows/inference_tests.yml +++ b/.github/workflows/inference_tests.yml @@ -10,8 +10,12 @@ on: release: types: [published] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-inference_tests.yml + cancel-in-progress: true + jobs: - run-full-tests-suit: + inference: runs-on: ubuntu-latest env: diff --git a/.github/workflows/library_eager_execution_tests.yml b/.github/workflows/library_eager_execution_tests.yml index 77ad90c64..e662b5a5e 100644 --- a/.github/workflows/library_eager_execution_tests.yml +++ b/.github/workflows/library_eager_execution_tests.yml @@ -6,8 +6,12 @@ on: pull_request: branches: [ main ] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-library_eager_execution.yml + cancel-in-progress: true + jobs: - run-full-tests-suit: + eager: runs-on: ubuntu-latest env: diff --git a/.github/workflows/library_tests.yml b/.github/workflows/library_tests.yml index 9f913a29e..50cd0600c 100644 --- a/.github/workflows/library_tests.yml +++ b/.github/workflows/library_tests.yml @@ -6,8 +6,12 @@ on: pull_request: branches: [ main ] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-library_tests.yml + cancel-in-progress: true + jobs: - run-full-tests-suit: + unittests: runs-on: ubuntu-latest env: diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 3e2c18a0e..6c55e75f1 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -6,8 +6,12 @@ on: pull_request: branches: [ main ] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-performance.yml + cancel-in-progress: true + jobs: - run-performance-tests: + performance: runs-on: ubuntu-latest env: diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 00abbb864..bbb341e7b 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -6,8 +6,12 @@ on: pull_request: branches: [ main ] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-quality.yml + cancel-in-progress: true + jobs: - verify-pre-commit-executed: + quality: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test_helm.yml b/.github/workflows/test_helm.yml index 9912509c6..086e524a3 100644 --- a/.github/workflows/test_helm.yml +++ b/.github/workflows/test_helm.yml @@ -6,8 +6,12 @@ on: pull_request: branches: [ main ] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-test_helm.yml + cancel-in-progress: true + jobs: - test-helm: + helm: runs-on: ubuntu-latest env: diff --git a/tests/catalog/test_preparation.py b/tests/catalog/test_preparation.py index caec4d2d3..d00e5d801 100644 --- a/tests/catalog/test_preparation.py +++ b/tests/catalog/test_preparation.py @@ -20,16 +20,30 @@ ) glob_query = os.path.join(project_dir, "prepare", "**", "*.py") all_preparation_files = glob.glob(glob_query, recursive=True) +# Make sure the order in which the tests are run is deterministic +# Having a different order for local testing and github testing may cause diffs in results. +all_preparation_files.sort() +num_par = 1 # num of parallel executions +logger.critical( + f"Over all, {len(all_preparation_files)} files will now be tested over {num_par} parallel processes." +) +# the following should be any of modulo num_par: 0,1,2,3,4,5,6,7,8,.. num_par-1 +modulo = 0 +all_preparation_files = [ + file for i, file in enumerate(all_preparation_files) if i % num_par == modulo +] class TestCatalogPreparation(UnitxtCatalogPreparationTestCase): def test_preparations(self): logger.info(glob_query) - logger.critical(f"Testing preparation files: {all_preparation_files}") - # Make sure the order in which the tests are run is deterministic - # Having a different order for local testing and github testing may cause diffs in results. + all_preparation_files_as_string = "\n".join( + [file.split("prepare")[-1] for file in all_preparation_files] + ) + logger.critical( + f"Testing {len(all_preparation_files)} preparation files: \n{all_preparation_files_as_string}\n" + ) times = {} - all_preparation_files.sort() for file in all_preparation_files: logger.info( "\n_____________________________________________\n" @@ -50,6 +64,7 @@ def test_preparations(self): f"Skipping file {file} due to ignored error {e}" ) continue + self.assertTrue(False) raise logger.info(f"Testing preparation file: {file} passed") self.assertTrue(True) @@ -70,6 +85,6 @@ def test_preparations(self): logger.critical(f"Testing preparation file '{file}' failed:") raise e - logger.critical("Preparation times table:") + logger.critical(f"Preparation times table for {len(times)} files:") times = dict(sorted(times.items(), key=lambda item: item[1], reverse=True)) print_dict(times, log_level="critical")