[MrBot] Add timed test suite validation to build_test_folder

build_functions/CMakeLists.txt

@@ -344,6 +344,39 @@ function(build_exe whatIsBuilding solution_folder custom_main)
     endif()
 endfunction()
 
+# Validates that *_int.c files in test_folder use TIMED_TEST_SUITE_INITIALIZE/CLEANUP.
+# Skip specific folders: cmake -Dtimed_test_suite_skip_regex="module_a_int|module_b_int"
+# Optional: NUM_EXPECTED_VIOLATIONS <n> - expected number of files with violations (default 0)
+function(validate_timed_test_suite test_folder)
+    set(options)
+    set(oneValueArgs NUM_EXPECTED_VIOLATIONS)
+    set(multiValueArgs)
+    cmake_parse_arguments(arg "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+    if(NOT DEFINED arg_NUM_EXPECTED_VIOLATIONS)
+        set(arg_NUM_EXPECTED_VIOLATIONS 0)
+    endif()
+
+    # Allow skipping via regex
+    if(DEFINED timed_test_suite_skip_regex AND NOT "${timed_test_suite_skip_regex}" STREQUAL "")
+        if("${test_folder}" MATCHES "${timed_test_suite_skip_regex}")
+            return()
+        endif()
+    endif()
+
+    set(script_path "${trsw_internal_dir}/scripts/validate_timed_test_suite.ps1")
+    if(EXISTS "${script_path}")
+        add_custom_target(${test_folder}_timed_suite_check ALL
+            COMMAND powershell.exe -ExecutionPolicy Bypass
+                -File "${script_path}"
+                -TestFolder "${CMAKE_CURRENT_SOURCE_DIR}/${test_folder}"
+                -NumExpectedViolations ${arg_NUM_EXPECTED_VIOLATIONS}
+            COMMENT "Validating timed test suite in ${test_folder}"
+            VERBATIM
+        )
+    endif()
+endfunction()
+
 #drop in replacement for add_subdirectory :)
 #test_folder is the name of a folder that exists on the drive. For example "clds_hash_table_ut". It is not a complete path.
 function(build_test_folder test_folder)
@@ -368,6 +401,11 @@ function(build_test_folder test_folder)
             set(building dll)
             add_subdirectory(${test_folder} ${test_folder}/${arg_BINARY_DIR}/${building})
         endif()
+
+        # Validate timed test suite for integration tests
+        if(WIN32 AND ("${test_folder}" MATCHES ".*int.*"))
+            validate_timed_test_suite(${test_folder})
+        endif()
     else()
         #message("test_folder is ${test_folder} NOT BUILDING")
     endif()
@@ -404,3 +442,7 @@ function(build_test_artifacts_with_custom_main whatIsBuilding solution_folder )
         MESSAGE(FATAL_ERROR "not an expected value: building=${building} test_folder=${test_folder} ARGN=${ARGN}")
     endif()
 endfunction()
+
+if(${run_unittests})
+    add_subdirectory(tests/validate_timed_test_suite)
+endif()

build_functions/scripts/validate_timed_test_suite.ps1

@@ -0,0 +1,134 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+<#
+.SYNOPSIS
+    Validates that integration test files use TIMED_TEST_SUITE_INITIALIZE/CLEANUP macros.
+
+.DESCRIPTION
+    This script checks integration test files (*_int.c) in a given test folder to ensure
+    they use the timed versions of TEST_SUITE_INITIALIZE and TEST_SUITE_CLEANUP macros.
+    The timed macros (TIMED_TEST_SUITE_INITIALIZE / TIMED_TEST_SUITE_CLEANUP) wrap the
+    vanilla versions with a process watchdog that crashes the process and produces a dump
+    on timeout, allowing dumps to be collected if the test hangs.
+
+.PARAMETER TestFolder
+    The path to the test folder to validate.
+
+.PARAMETER NumExpectedViolations
+    The number of files expected to have violations. When the actual violation count matches
+    this value, the script exits with code 0. Default is 0 (no violations expected).
+
+.EXAMPLE
+    .\validate_timed_test_suite.ps1 -TestFolder "C:\repo\tests\my_module_int"
+
+    Validates integration test files in the folder and reports files using non-timed macros.
+
+.NOTES
+    Returns exit code 0 if violation count matches NumExpectedViolations, 1 otherwise.
+#>
+
+param(
+    [Parameter(Mandatory=$true)]
+    [string]$TestFolder,
+
+    [Parameter(Mandatory=$false)]
+    [int]$NumExpectedViolations = 0
+)
+
+# Set error action preference
+$ErrorActionPreference = "Stop"
+
+Write-Host "========================================" -ForegroundColor Cyan
+Write-Host "Timed Test Suite Validation" -ForegroundColor Cyan
+Write-Host "========================================" -ForegroundColor Cyan
+Write-Host "Test Folder: $TestFolder" -ForegroundColor White
+if ($NumExpectedViolations -gt 0) {
+    Write-Host "Expected Violations: $NumExpectedViolations" -ForegroundColor White
+}
+Write-Host ""
+
+# Get all integration test files in the folder
+Write-Host "Searching for integration test files (*_int.c)..." -ForegroundColor White
+$allFiles = @(Get-ChildItem -Path $TestFolder -Recurse -Filter "*_int.c" -ErrorAction SilentlyContinue)
+
+Write-Host "Found $($allFiles.Count) integration test files to check" -ForegroundColor White
+Write-Host ""
+
+# Pattern to match non-timed TEST_SUITE_INITIALIZE/CLEANUP
+# Uses negative lookbehind to exclude lines that already have TIMED_ prefix
+$initPattern = '(?<!TIMED_)TEST_SUITE_INITIALIZE\s*\('
+$cleanupPattern = '(?<!TIMED_)TEST_SUITE_CLEANUP\s*\('
+
+$totalFiles = 0
+$filesWithViolations = @()
+
+foreach ($file in $allFiles) {
+    $totalFiles++
+
+    # Read file content as lines
+    try {
+        $lines = Get-Content -Path $file.FullName -ErrorAction Stop
+    }
+    catch {
+        Write-Host "  [WARN] Cannot read file: $($file.FullName)" -ForegroundColor Yellow
+        continue
+    }
+
+    $matchingLines = @()
+    $lineNumber = 0
+    foreach ($line in $lines) {
+        $lineNumber++
+        if ($line -match $initPattern -or $line -match $cleanupPattern) {
+            $matchingLines += [PSCustomObject]@{
+                LineNumber = $lineNumber
+                Content = $line.Trim()
+            }
+        }
+    }
+
+    if ($matchingLines.Count -gt 0) {
+        Write-Host "  [FAIL] $($file.FullName)" -ForegroundColor Red
+        Write-Host "         Contains $($matchingLines.Count) non-timed TEST_SUITE macro(s)" -ForegroundColor Yellow
+
+        foreach ($match in $matchingLines) {
+            Write-Host "         Line $($match.LineNumber): $($match.Content)" -ForegroundColor Yellow
+        }
+
+        $filesWithViolations += [PSCustomObject]@{
+            FilePath = $file.FullName
+            MatchCount = $matchingLines.Count
+            Matches = $matchingLines
+        }
+    }
+}
+
+Write-Host ""
+Write-Host "========================================" -ForegroundColor Cyan
+Write-Host "Validation Summary" -ForegroundColor Cyan
+Write-Host "========================================" -ForegroundColor Cyan
+Write-Host "Total integration test files checked: $totalFiles" -ForegroundColor White
+
+if ($filesWithViolations.Count -gt 0) {
+    Write-Host "Files with non-timed macros: $($filesWithViolations.Count)" -ForegroundColor Red
+    Write-Host ""
+    Write-Host "The following files use TEST_SUITE_INITIALIZE/CLEANUP instead of timed versions:" -ForegroundColor Yellow
+
+    foreach ($item in $filesWithViolations) {
+        Write-Host "  - $($item.FilePath) ($($item.MatchCount) macro(s))" -ForegroundColor White
+    }
+
+    Write-Host ""
+    Write-Host "Integration tests should use TIMED_TEST_SUITE_INITIALIZE and" -ForegroundColor Cyan
+    Write-Host "TIMED_TEST_SUITE_CLEANUP from c_pal/timed_test_suite.h to allow" -ForegroundColor Cyan
+    Write-Host "dumps to be collected if the test hangs." -ForegroundColor Cyan
+    Write-Host ""
+}
+
+if ($filesWithViolations.Count -eq $NumExpectedViolations) {
+    Write-Host "[VALIDATION PASSED]" -ForegroundColor Green
+    exit 0
+} else {
+    Write-Host "[VALIDATION FAILED] Expected $NumExpectedViolations violation(s), found $($filesWithViolations.Count)" -ForegroundColor Red
+    exit 1
+}

build_functions/tests/validate_timed_test_suite/CMakeLists.txt

@@ -0,0 +1,32 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#
+# Timed Test Suite Validation Tests
+#
+# Tests the validate_timed_test_suite CMake function and underlying script.
+#
+
+# Test 1: Clean files pass validation (0 expected violations)
+validate_timed_test_suite(no_violations)
+
+# Test 2: Detection of violations (3 files with violations expected)
+validate_timed_test_suite(has_violations NUM_EXPECTED_VIOLATIONS 3)
+
+# Test 3: Skip regex causes check to be skipped
+set(timed_test_suite_skip_regex "skipped_folder_int")
+validate_timed_test_suite(skipped_folder_int)
+if(TARGET skipped_folder_int_timed_suite_check)
+    message(FATAL_ERROR "skipped_folder_int should have been skipped by regex but target was created")
+endif()
+unset(timed_test_suite_skip_regex)
+
+# Master target for all timed test suite validation tests
+add_custom_target(test_validate_timed_test_suite
+    COMMENT "Running all timed test suite validation tests"
+)
+
+add_dependencies(test_validate_timed_test_suite
+    no_violations_timed_suite_check
+    has_violations_timed_suite_check
+)

build_functions/tests/validate_timed_test_suite/has_violations/has_include_int.c

@@ -0,0 +1,16 @@
+#include "testrunnerswitcher.h"
+#include "c_pal/gballoc_hl.h"
+#include "c_pal/timed_test_suite.h"
+
+BEGIN_TEST_SUITE(has_include_int)
+TEST_SUITE_INITIALIZE(suite_init)
+{
+}
+TEST_SUITE_CLEANUP(suite_cleanup)
+{
+}
+TEST_FUNCTION(test1)
+{
+    ASSERT_IS_TRUE(1);
+}
+END_TEST_SUITE(has_include_int)

build_functions/tests/validate_timed_test_suite/has_violations/multi_arg_int.c

@@ -0,0 +1,15 @@
+#include "testrunnerswitcher.h"
+#include "c_pal/gballoc_hl.h"
+
+BEGIN_TEST_SUITE(multi_arg_int)
+TEST_SUITE_INITIALIZE(suite_init, setup_gballoc)
+{
+}
+TEST_SUITE_CLEANUP(suite_cleanup, cleanup_gballoc)
+{
+}
+TEST_FUNCTION(test1)
+{
+    ASSERT_IS_TRUE(1);
+}
+END_TEST_SUITE(multi_arg_int)

build_functions/tests/validate_timed_test_suite/has_violations/sample_module_int.c

@@ -0,0 +1,15 @@
+#include "testrunnerswitcher.h"
+#include "c_pal/gballoc_hl.h"
+
+BEGIN_TEST_SUITE(sample_module_int)
+TEST_SUITE_INITIALIZE(suite_init)
+{
+}
+TEST_SUITE_CLEANUP(suite_cleanup)
+{
+}
+TEST_FUNCTION(test1)
+{
+    ASSERT_IS_TRUE(1);
+}
+END_TEST_SUITE(sample_module_int)

build_functions/tests/validate_timed_test_suite/no_violations/no_init_int.c

@@ -0,0 +1,9 @@
+#include "testrunnerswitcher.h"
+#include "c_pal/gballoc_hl.h"
+
+BEGIN_TEST_SUITE(no_init_int)
+TEST_FUNCTION(test1)
+{
+    ASSERT_IS_TRUE(1);
+}
+END_TEST_SUITE(no_init_int)

build_functions/tests/validate_timed_test_suite/no_violations/timed_module_int.c

@@ -0,0 +1,16 @@
+#include "testrunnerswitcher.h"
+#include "c_pal/gballoc_hl.h"
+#include "c_pal/timed_test_suite.h"
+
+BEGIN_TEST_SUITE(timed_module_int)
+TIMED_TEST_SUITE_INITIALIZE(suite_init, TIMED_TEST_DEFAULT_TIMEOUT_MS)
+{
+}
+TIMED_TEST_SUITE_CLEANUP(suite_cleanup)
+{
+}
+TEST_FUNCTION(test1)
+{
+    ASSERT_IS_TRUE(1);
+}
+END_TEST_SUITE(timed_module_int)