diff --git a/setup.cfg b/setup.cfg index 9d8b74a..fab77f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,6 +43,8 @@ install_requires = [options.extras_require] dev = + # freezegun 1.2.2 fixed pytest timing interference. + freezegun>=1.2.2 requests-mock[fixture] [options.entry_points] diff --git a/tests/common.py b/tests/common.py index 51094f6..99ad327 100644 --- a/tests/common.py +++ b/tests/common.py @@ -8,6 +8,8 @@ import os import re import subprocess +import time +from datetime import datetime from enum import Enum from pathlib import Path from typing import (TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, @@ -308,6 +310,10 @@ def assert_regex(regex: str, string: str) -> None: assert re.match(regex, string) is not None, f'`{string}` does not match regex {regex}' +def _ts_from_rfc3339(ts: str) -> float: + return datetime.fromisoformat(ts).timestamp() + + @patch.multiple('time', sleep=mock.DEFAULT) @requests_mock.Mocker(case_sensitive=True, kw='requests_mocker') def run_test_case( @@ -404,6 +410,8 @@ def run_test_case( ) + list(extra_args) ) + start_time = time.time() + __pytest_current_test = os.environ['PYTEST_CURRENT_TEST'] if monkeypatch is not None: with monkeypatch.context() as mp: @@ -414,6 +422,10 @@ def run_test_case( else: result = pytester.runpytest(*pytest_args) + end_time = time.time() + + assert end_time >= start_time + # pytester clears PYTEST_CURRENT_TEST for some reason. os.environ['PYTEST_CURRENT_TEST'] = __pytest_current_test @@ -649,7 +661,13 @@ def run_test_case( ) assert_regex(TIMESTAMP_REGEX, upload_body['start_time']) + assert _ts_from_rfc3339(upload_body['start_time']) >= start_time + assert_regex(TIMESTAMP_REGEX, upload_body['end_time']) + assert _ts_from_rfc3339(upload_body['end_time']) <= end_time + + assert _ts_from_rfc3339(upload_body['end_time']) >= ( + _ts_from_rfc3339(upload_body['start_time'])) actual_test_runs = { (test_run_record['filename'], tuple(test_run_record['name'])): [ @@ -659,6 +677,20 @@ def run_test_case( } assert actual_test_runs == expected_uploaded_test_runs + for test_run_record in upload_body['test_runs']: + for attempt in test_run_record['attempts']: + assert_regex(TIMESTAMP_REGEX, attempt['start_time']) + # Check for timestamp interference (e.g., from freezetime). + assert _ts_from_rfc3339(attempt['start_time']) >= start_time + + assert_regex(TIMESTAMP_REGEX, attempt['end_time']) + assert _ts_from_rfc3339(attempt['end_time']) <= end_time + + assert _ts_from_rfc3339(attempt['end_time']) >= ( + _ts_from_rfc3339(attempt['start_time'])) + + assert isinstance(attempt['duration_ms'], int) + # Make sure there aren't any duplicate test keys. assert len(upload_body['test_runs']) == len(actual_test_runs) diff --git a/tests/test_unflakable.py b/tests/test_unflakable.py index e27386f..8e8981d 100644 --- a/tests/test_unflakable.py +++ b/tests/test_unflakable.py @@ -2255,3 +2255,57 @@ def test_pass(): extra_args=XDIST_ARGS if xdist else [], failed_upload_requests=1, ) + + +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, + TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) +def test_freeze_time( + pytester: pytest.Pytester, + subprocess_mock: GitMock, + verbose: bool, + xdist: bool, +) -> None: + pytester.makepyfile(test_input=""" + from freezegun import freeze_time + import unittest + + @freeze_time("2023-02-04") + class FrozenTimeTests(unittest.TestCase): + @freeze_time("2024-01-01") + def test_pass(self): + pass + + @freeze_time("2024-01-02") + def test_pass2(self): + pass + + @freeze_time("2024-01-03") + def test_pass3(self): + pass + """) + + subprocess_mock.update(branch=None, commit=None) + + run_test_case( + pytester, + manifest={'quarantined_tests': []}, + expected_test_file_outcomes=[ + ('test_input.py', [ + (('FrozenTimeTests', 'test_pass'), [_TestAttemptOutcome.PASSED]), + (('FrozenTimeTests', 'test_pass2'), [_TestAttemptOutcome.PASSED]), + (('FrozenTimeTests', 'test_pass3'), [_TestAttemptOutcome.PASSED]), + ]) + ], + expected_test_result_counts=_TestResultCounts(num_passed=3), + expected_uploaded_test_runs={ + ('test_input.py', ('FrozenTimeTests', 'test_pass')): ['pass'], + ('test_input.py', ('FrozenTimeTests', 'test_pass2')): ['pass'], + ('test_input.py', ('FrozenTimeTests', 'test_pass3')): ['pass'], + }, + expected_exit_code=ExitCode.OK, + expected_branch=None, + expected_commit=None, + expect_xdist=xdist, + extra_args=XDIST_ARGS if xdist else [], + verbose=verbose, + )