From b11bc9f4d93355a3b411ffc3451b709c2b02b483 Mon Sep 17 00:00:00 2001 From: connoraird Date: Wed, 12 Nov 2025 16:59:51 +0000 Subject: [PATCH 1/2] Use mean instead on min as it should be more reliable and make less strict --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 3063fe9..6d45c67 100644 --- a/noxfile.py +++ b/noxfile.py @@ -84,5 +84,5 @@ def regression_tests(session: nox.Session) -> None: session.run( "pytest", "--benchmark-compare=0001", - "--benchmark-compare-fail=min:5%", + "--benchmark-compare-fail=mean:15%", ) From 9f5b86033ceaeaea8c4b69ae4436698d7c37f3a5 Mon Sep 17 00:00:00 2001 From: connoraird Date: Thu, 13 Nov 2025 10:39:58 +0000 Subject: [PATCH 2/2] Use time.process_time timer which ignores sleeps and IO and make failure threshold a little more sensitive --- noxfile.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 6d45c67..35681ad 100644 --- a/noxfile.py +++ b/noxfile.py @@ -77,12 +77,17 @@ def regression_tests(session: nox.Session) -> None: print(f"Generating before benchmark for comparison from revision {before_revision}") session.install(f"git+{GLASS_REPO_URL}@{before_revision}") - session.run("pytest", "--benchmark-autosave") + session.run( + "pytest", + "--benchmark-autosave", + "--benchmark-timer=time.process_time", + ) print(f"Comparing before benchmark to revisiob {after_revision}") session.install(f"git+{GLASS_REPO_URL}@{after_revision}") session.run( "pytest", "--benchmark-compare=0001", - "--benchmark-compare-fail=mean:15%", + "--benchmark-compare-fail=mean:10%", + "--benchmark-timer=time.process_time", )