Test only: Only run certain test cases if the folder has changed

.github/scripts/generate_test_module.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+
+"""Generate test modules list to be utilized through gihub actions
+
+Will output a list of modules needs run test cases:
+*
+"""
+
+import argparse
+import os
+import sys
+from typing import List
+
+import regex
+
+MAX_FOLDER_DEPTH = 3
+
+# TODO: discuss with Naren for a better mapping
+Folder_To_TestModules_Dict = {
+    "py/torch_tensorrt/dynamo": [
+        "dynamo_frontend",
+        "dynamo_core",
+        "dynamo_converter",
+        "dynamo_serde",
+        "torch_compile_backend",
+    ],
+    "py/torch_tensorrt/ts": ["ts_frontend"],
+    "py/torch_tensorrt": ["py_core"],
+}
+
+# The following folder files change will trigger the build but will skip all the test modules
+Folder_To_Skip_TestModules = {
+    "docker/*",
+    "docs/*",
+    "docsrc/*",
+    "examples/*",
+    "notenooks/*",
+    "packaging/*",
+    "third_party/*",
+    "toolchains/*",
+    "tools/*",
+}
+
+# TODO: discuss with Naren for a basic set of tests here
+# this is just an example only
+Base_Test_Modules = [
+    "cpp_core_conversion",
+    "cpp_core_lowering",
+    "cpp_core_partitioning",
+    "cpp_core_runtime",
+]
+Full_Test_Modules = [
+    "ts_frontend",
+    "py_core",
+    "dynamo_frontend",
+    "dynamo_core",
+    "dynamo_converter",
+    "dynamo_serde",
+    "torch_compile_backend",
+]
+
+
+def filter_files_to_folders(
+    files: str,
+) -> List[str]:
+    fileList = files.split(" ")
+    folders = []
+
+    for file in fileList:
+        if file == "":
+            continue
+        filePath = os.path.dirname(file)
+        splits = filePath.split("/", MAX_FOLDER_DEPTH)
+        if len(splits) > MAX_FOLDER_DEPTH:
+            splits = splits[: len(splits) - 1]
+        folder = "/".join(splits)
+        folders.append(folder)
+
+    ret = list(dict.fromkeys(folders))
+    return ret
+
+
+def generate_test_modules(
+    folders: List[str],
+) -> List[str]:
+    testModules = []
+    for folder in folders:
+        skip = False
+        for to_skip in Folder_To_Skip_TestModules:
+            if regex.match(to_skip, folder):
+                skip = True
+                break
+        if skip == True:
+            continue
+        if folder in Folder_To_TestModules_Dict.keys():
+            modules = Folder_To_TestModules_Dict[folder]
+            testModules.extend(modules)
+        else:
+            # if there is files changed in other folders, always run the base tests
+            testModules.extend(Base_Test_Modules)
+    return list(dict.fromkeys(testModules))
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--event-name",
+        help="",
+        type=str,
+        default="",
+    )
+    parser.add_argument(
+        "--files",
+        help="",
+        type=str,
+        default="",
+    )
+    return parser.parse_args(sys.argv[1:])
+
+
+def main() -> None:
+    options = parse_args()
+    assert options.event_name != "", "Must provide the --event-name str"
+
+    # if it is the Pull Request:
+    # for the pull request, it will run the full tests of the test module if the related files changed
+    # else only run the l0 testcases
+    if options.event_name == "pull_request":
+        assert (
+            options.files != ""
+        ), "Must provide the --files str, if the --event-name is pull_request"
+        folders = filter_files_to_folders(options.files)
+        test_modules = generate_test_modules(folders)
+    else:
+        # for the non pull_request, always run the full tests of all the test modules
+        test_modules = Full_Test_Modules
+
+    print(test_modules)
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/build-test-linux.yml

@@ -54,9 +54,38 @@ jobs:
       smoke-test-script: ${{ matrix.smoke-test-script }}
       trigger-event: ${{ github.event_name }}
 
+  check:
+    needs: build
+    name: Check files
+    outputs:
+      test_modules: ${{ steps.check_files.outputs.test_modules }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout torch-tensorrt repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.repository }}
+          ref: ${{ inputs.ref }}
+          fetch-depth: 0
+      - name: Check modified files
+        id: check_files
+        run: |
+          set -euxo pipefail
+          if [ ${{ github.event_name }} == 'pull_request' ]; then
+            base=${{ github.event.pull_request.base.sha }}
+            head=${{ github.event.pull_request.head.sha }}
+            python -m pip install regex
+            files=$(git diff --name-only "${base}" "${head}")
+            test_modules="$(python .github/scripts/generate_test_module.py --event-name 'pull_request' --files '${files}')"
+          else
+            test_modules="$(python .github/scripts/generate_test_module.py --event-name ${{ github.event_name }})"
+          fi
+          echo "test_modules=${test_modules}" >> "${GITHUB_OUTPUT}"
+
   tests-py-torchscript-fe:
     name: Test torchscript frontend [Python]
-    needs: [generate-matrix, build]
+    needs: [generate-matrix, build, check]
+    if: ${{ contains(needs.check.outputs.test_modules, 'ts_frontend') }}
     strategy:
       fail-fast: false
       matrix:
@@ -91,7 +120,8 @@ jobs:
 
   tests-py-dynamo-converters:
     name: Test dynamo converters [Python]
-    needs: [generate-matrix, build]
+    needs: [generate-matrix, build, check]
+    if: ${{ contains(needs.check.outputs.test_modules, 'dynamo_converter') }}
     strategy:
       fail-fast: false
       matrix:
@@ -120,6 +150,7 @@ jobs:
   tests-py-dynamo-fe:
     name: Test dynamo frontend [Python]
     needs: [generate-matrix, build]
+    if: ${{ contains(needs.check.outputs.test_modules, 'dynamo_frontend') }}
     strategy:
       fail-fast: false
       matrix:
@@ -139,6 +170,8 @@ jobs:
       build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
       pre-script: ${{ matrix.pre-script }}
       script: |
+        set -x
+        echo "needs.check.outputs.test_modules: ${{ needs.check.outputs.test_modules }}"
         export USE_HOST_DEPS=1
         pushd .
         cd tests/py/dynamo
@@ -148,7 +181,8 @@ jobs:
 
   tests-py-dynamo-serde:
     name: Test dynamo export serde [Python]
-    needs: [generate-matrix, build]
+    needs: [generate-matrix, build, check]
+    if: ${{ contains(needs.check.outputs.test_modules, 'dynamo_serde') }}
     strategy:
       fail-fast: false
       matrix:
@@ -176,7 +210,8 @@ jobs:
 
   tests-py-torch-compile-be:
     name: Test torch compile backend [Python]
-    needs: [generate-matrix, build]
+    needs: [generate-matrix, build ,check]
+    if: ${{ contains(needs.check.outputs.test_modules, 'torch_compile_backend') }}
     strategy:
       fail-fast: false
       matrix:
@@ -206,7 +241,8 @@ jobs:
 
   tests-py-dynamo-core:
     name: Test dynamo core [Python]
-    needs: [generate-matrix, build]
+    needs: [generate-matrix, build, check]
+    if: ${{ contains(needs.check.outputs.test_modules, 'dynamo_core') }}
     strategy:
       fail-fast: false
       matrix:
@@ -236,7 +272,8 @@ jobs:
 
   tests-py-core:
     name: Test core [Python]
-    needs: [generate-matrix, build]
+    needs: [generate-matrix, build, check]
+    if: ${{ contains(needs.check.outputs.test_modules, 'py_core') }}
     strategy:
       fail-fast: false
       matrix:
@@ -256,12 +293,127 @@ jobs:
       build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
       pre-script: ${{ matrix.pre-script }}
       script: |
+        set -x
         export USE_HOST_DEPS=1
         pushd .
         cd tests/py/core
         python -m pytest -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml .
         popd
 
+  tests-cpp-core-conversion:
+    name: Test core conversion[CPP]
+    needs: [generate-matrix, build, check]
+    if: ${{ contains(needs.check.outputs.test_modules, 'cpp_core_conversion') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - repository: pytorch/tensorrt
+            package-name: torch_tensorrt
+            pre-script: packaging/pre_build_script.sh
+            post-script: packaging/post_build_script.sh
+            smoke-test-script: packaging/smoke_test_script.sh
+    uses: ./.github/workflows/linux-test.yml
+    with:
+      job-name: tests-cpp-core-conversion
+      repository: "pytorch/tensorrt"
+      ref: ""
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
+      pre-script: ${{ matrix.pre-script }}
+      script: |
+        set -x
+        export USE_HOST_DEPS=1
+        python -m pip show torch_tensorrt
+        location=$(python -m pip show torch_tensorrt | grep Location | sed "s/Location: //g")
+        ls -lart $location | grep torch_tensorrt
+        bazel test //tests/core/conversion:conversion_tests --compilation_mode=opt \
+        --test_output=summary --config use_precompiled_torchtrt --config pre_cxx11_abi --sandbox_debug --verbose_failures --cxxopt='-std=c++17'
+
+  tests-cpp-core-lowering:
+    name: Test core lowering[CPP]
+    needs: [generate-matrix, build, check]
+    if: ${{ contains(needs.check.outputs.test_modules, 'cpp_core_lowering') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - repository: pytorch/tensorrt
+            package-name: torch_tensorrt
+            pre-script: packaging/pre_build_script.sh
+            post-script: packaging/post_build_script.sh
+            smoke-test-script: packaging/smoke_test_script.sh
+    uses: ./.github/workflows/linux-test.yml
+    with:
+      job-name: tests-cpp-core-lowering
+      repository: "pytorch/tensorrt"
+      ref: ""
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
+      pre-script: ${{ matrix.pre-script }}
+      script: |
+        set -x
+        export USE_HOST_DEPS=1
+        bazel test //tests/core/lowering:lowering_tests --compilation_mode=opt \
+        --test_output=summary --config use_precompiled_torchtrt --config pre_cxx11_abi --sandbox_debug --verbose_failures --cxxopt='-std=c++17'
+
+  tests-cpp-core-partitioning:
+    name: Test core partitioning[CPP]
+    needs: [generate-matrix, build, check]
+    if: ${{ contains(needs.check.outputs.test_modules, 'cpp_core_partitioning') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - repository: pytorch/tensorrt
+            package-name: torch_tensorrt
+            pre-script: packaging/pre_build_script.sh
+            post-script: packaging/post_build_script.sh
+            smoke-test-script: packaging/smoke_test_script.sh
+    uses: ./.github/workflows/linux-test.yml
+    with:
+      job-name: tests-cpp-core-partitioning
+      repository: "pytorch/tensorrt"
+      ref: ""
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
+      pre-script: ${{ matrix.pre-script }}
+      script: |
+        set -x
+        export USE_HOST_DEPS=1
+        bazel test //tests/core/partitioning:partitioning_tests --compilation_mode=opt \
+        --test_output=summary --config use_precompiled_torchtrt --config pre_cxx11_abi --sandbox_debug --verbose_failures --cxxopt='-std=c++17'
+
+  tests-cpp-core-runtime:
+    name: Test core runtime[CPP]
+    needs: [generate-matrix, build, check]
+    if: ${{ contains(needs.check.outputs.test_modules, 'cpp_core_runtime') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - repository: pytorch/tensorrt
+            package-name: torch_tensorrt
+            pre-script: packaging/pre_build_script.sh
+            post-script: packaging/post_build_script.sh
+            smoke-test-script: packaging/smoke_test_script.sh
+    uses: ./.github/workflows/linux-test.yml
+    with:
+      job-name: tests-cpp-core-runtime
+      repository: "pytorch/tensorrt"
+      ref: ""
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
+      pre-script: ${{ matrix.pre-script }}
+      script: |
+        set -x
+        export USE_HOST_DEPS=1
+        bazel test //tests/core/runtime:runtime_tests --compilation_mode=opt \
+        --test_output=summary --config use_precompiled_torchtrt --config pre_cxx11_abi --sandbox_debug --verbose_failures --cxxopt='-std=c++17'
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }}
   cancel-in-progress: true