Make GitHub Actions tests faster and clearer

* commit again same changes needed Signed-off-by: dafnapension <dafnashein@yahoo.com> * maybe Signed-off-by: dafnapension <dafnashein@yahoo.com> * maybe Signed-off-by: dafnapension <dafnashein@yahoo.com> * maybe Signed-off-by: dafnapension <dafnashein@yahoo.com> * removed the reporter following job Signed-off-by: dafnapension <dafnashein@yahoo.com> * Add minor change to github action Signed-off-by: elronbandel <elronbandel@gmail.com> * Make sure there are no more then one job running for each pull request at the same time Signed-off-by: elronbandel <elronbandel@gmail.com> * Fix Signed-off-by: elronbandel <elronbandel@gmail.com> * Update Signed-off-by: elronbandel <elronbandel@gmail.com> * Update Signed-off-by: elronbandel <elronbandel@gmail.com> * Shorten jobs names for better visability Signed-off-by: elronbandel <elronbandel@gmail.com> --------- Signed-off-by: dafnapension <dafnashein@yahoo.com> Signed-off-by: elronbandel <elronbandel@gmail.com> Co-authored-by: elronbandel <elronbandel@gmail.com>
IBM · Nov 14, 2024 · b44e3e9 · b44e3e9
1 parent fbbbebe
commit b44e3e9
Show file tree

Hide file tree

Showing 11 changed files with 108 additions and 43 deletions.
diff --git a/.github/workflows/catalog_consistency.yml b/.github/workflows/catalog_consistency.yml
@@ -6,8 +6,12 @@ on:
    pull_request:
      branches: [ main ]
 
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-catalog_consistency.yml
+    cancel-in-progress: true
+
 jobs:
-   run-catalog-consistency:
+   consistency:
 
      runs-on: ubuntu-latest
      env:

diff --git a/.github/workflows/catalog_preparation.yml b/.github/workflows/catalog_preparation.yml
@@ -1,34 +1,48 @@
 name: Test Catalog Preparation
 
 on:
-   push:
-     branches: [ main ]
-   pull_request:
-     branches: [ main ]
-
-jobs:
-   run-catalog-preparation:
-
-     runs-on: ubuntu-latest
-     env:
-       OS: ubuntu-latest
-       UNITXT_DEFAULT_VERBOSITY: error
-       DATASETS_VERBOSITY: error
-       HF_HUB_VERBOSITY: error
-       HF_DATASETS_DISABLE_PROGRESS_BARS: "True"
-       TQDM_DISABLE: "True"
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
 
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-catalog_preparation.yml
+    cancel-in-progress: true
 
-     steps:
-     - uses: actions/checkout@v4
-
-     - uses: actions/setup-python@v5
-       with:
-         python-version: '3.9'
-
-     - run: curl -LsSf https://astral.sh/uv/install.sh | sh
-     - run: uv pip install --system ".[tests]"
-     - run: huggingface-cli login --token ${{ secrets.UNITXT_READ_HUGGINGFACE_HUB_FOR_TESTS }}
+jobs:
+  preparation:
+
+    runs-on: ubuntu-latest
+    env:
+      OS: ubuntu-latest
+      UNITXT_DEFAULT_VERBOSITY: error
+      DATASETS_VERBOSITY: error
+      HF_HUB_VERBOSITY: error
+      HF_DATASETS_DISABLE_PROGRESS_BARS: "True"
+      TQDM_DISABLE: "True"
+
+    strategy:
+      matrix:
+        modulo: [0,1,2,3,4,5,6,7]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+
+    - run: curl -LsSf https://astral.sh/uv/install.sh | sh
+    - run: uv pip install --system ".[tests]"
+    - run: huggingface-cli login --token ${{ secrets.UNITXT_READ_HUGGINGFACE_HUB_FOR_TESTS }}
+
+    - name: Run Tests
+      run: |
+        modulo="${{ matrix.modulo }}"
+        echo "modulo=${modulo}" >> $GITHUB_STEP_SUMMARY
+        echo "sed -i 's/^num_par = 1 /num_par = 8 /' tests/catalog/test_preparation.py" > sedit.sh
+        echo "sed -i 's/^modulo = 0/modulo = ${modulo}/' tests/catalog/test_preparation.py" >> sedit.sh
+        sh sedit.sh
+        python -m unittest tests.catalog.test_preparation
 
-     - name: Run Tests
-       run: python -m unittest discover -s tests/catalog -p "test_*.py"
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -6,8 +6,12 @@ on:
     pull_request:
       branches: [ main ]
 
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-docs.yml
+    cancel-in-progress: true
+
 jobs:
-    compile-docs:
+    docs:
 
       runs-on: ubuntu-latest
       env:
@@ -20,7 +24,7 @@ jobs:
         - uses: actions/setup-python@v5
           with:
             python-version: '3.9'
-            
+
         - run: curl -LsSf https://astral.sh/uv/install.sh | sh
         - run: uv pip install --system ".[tests,docs]"
 

diff --git a/.github/workflows/examples_tests.yml b/.github/workflows/examples_tests.yml
@@ -9,8 +9,12 @@ on:
   release:
     types: [published]
 
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-examples_tests.yml
+    cancel-in-progress: true
+
 jobs:
-   run-full-tests-suit:
+   examples:
 
      runs-on: ubuntu-latest
      env:

diff --git a/.github/workflows/inference_tests.yml b/.github/workflows/inference_tests.yml
@@ -10,8 +10,12 @@ on:
   release:
     types: [published]
 
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-inference_tests.yml
+    cancel-in-progress: true
+
 jobs:
-   run-full-tests-suit:
+   inference:
 
      runs-on: ubuntu-latest
      env:

diff --git a/.github/workflows/library_eager_execution_tests.yml b/.github/workflows/library_eager_execution_tests.yml
@@ -6,8 +6,12 @@ on:
   pull_request:
     branches: [ main ]
 
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-library_eager_execution.yml
+    cancel-in-progress: true
+
 jobs:
-  run-full-tests-suit:
+  eager:
 
     runs-on: ubuntu-latest
     env:

diff --git a/.github/workflows/library_tests.yml b/.github/workflows/library_tests.yml
@@ -6,8 +6,12 @@ on:
   pull_request:
     branches: [ main ]
 
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-library_tests.yml
+    cancel-in-progress: true
+
 jobs:
-  run-full-tests-suit:
+  unittests:
 
     runs-on: ubuntu-latest
     env:

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
@@ -6,8 +6,12 @@ on:
   pull_request:
     branches: [ main ]
 
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-performance.yml
+    cancel-in-progress: true
+
 jobs:
-  run-performance-tests:
+  performance:
 
     runs-on: ubuntu-latest
     env:

diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml
@@ -6,8 +6,12 @@ on:
   pull_request:
     branches: [ main ]
 
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-quality.yml
+    cancel-in-progress: true
+
 jobs:
-  verify-pre-commit-executed:
+  quality:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/test_helm.yml b/.github/workflows/test_helm.yml
@@ -6,8 +6,12 @@ on:
     pull_request:
       branches: [ main ]
 
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-test_helm.yml
+    cancel-in-progress: true
+
 jobs:
-    test-helm:
+    helm:
 
       runs-on: ubuntu-latest
       env:

diff --git a/tests/catalog/test_preparation.py b/tests/catalog/test_preparation.py
@@ -20,16 +20,30 @@
 )
 glob_query = os.path.join(project_dir, "prepare", "**", "*.py")
 all_preparation_files = glob.glob(glob_query, recursive=True)
+# Make sure the order in which the tests are run is deterministic
+# Having a different order for local testing and github testing may cause diffs in results.
+all_preparation_files.sort()
+num_par = 1  # num of parallel executions
+logger.critical(
+    f"Over all, {len(all_preparation_files)} files will now be tested over {num_par} parallel processes."
+)
+# the following should be any of modulo num_par: 0,1,2,3,4,5,6,7,8,.. num_par-1
+modulo = 0
+all_preparation_files = [
+    file for i, file in enumerate(all_preparation_files) if i % num_par == modulo
+]
 
 
 class TestCatalogPreparation(UnitxtCatalogPreparationTestCase):
     def test_preparations(self):
         logger.info(glob_query)
-        logger.critical(f"Testing preparation files: {all_preparation_files}")
-        # Make sure the order in which the tests are run is deterministic
-        # Having a different order for local testing and github testing may cause diffs in results.
+        all_preparation_files_as_string = "\n".join(
+            [file.split("prepare")[-1] for file in all_preparation_files]
+        )
+        logger.critical(
+            f"Testing {len(all_preparation_files)} preparation files: \n{all_preparation_files_as_string}\n"
+        )
         times = {}
-        all_preparation_files.sort()
         for file in all_preparation_files:
             logger.info(
                 "\n_____________________________________________\n"
@@ -50,6 +64,7 @@ def test_preparations(self):
                                 f"Skipping file {file} due to ignored error {e}"
                             )
                             continue
+                        self.assertTrue(False)
                         raise
                     logger.info(f"Testing preparation file: {file} passed")
                     self.assertTrue(True)
@@ -70,6 +85,6 @@ def test_preparations(self):
                 logger.critical(f"Testing preparation file '{file}' failed:")
                 raise e
 
-        logger.critical("Preparation times table:")
+        logger.critical(f"Preparation times table for {len(times)} files:")
         times = dict(sorted(times.items(), key=lambda item: item[1], reverse=True))
         print_dict(times, log_level="critical")