From 67715f4028208a1c5dcc0e4ca6c33f3b0d21b312 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Tue, 19 Nov 2024 14:12:14 +0100 Subject: [PATCH] Add better test IDs --- .../python_tests/test_cache_optimizations.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/python_tests/test_cache_optimizations.py b/tests/python_tests/test_cache_optimizations.py index 18be429549..aedf7b1143 100644 --- a/tests/python_tests/test_cache_optimizations.py +++ b/tests/python_tests/test_cache_optimizations.py @@ -58,6 +58,7 @@ def converted_model(tmp_path_factory): @dataclass class CacheOptTestStruct: + test_id: str prompt_file: str max_new_tokens: int num_kv_blocks: int @@ -70,6 +71,31 @@ class CacheOptTestStruct: SHORT_CACHE_EVICTION_CONFIG = CacheEvictionConfig(start_size=32, recent_size=32, max_cache_size=96, aggregation_mode=AggregationMode.NORM_SUM) +def print_text_results(evaluator): + metric_of_interest = "similarity" + worst_examples = evaluator.worst_examples( + top_k=5, metric=metric_of_interest) + for i, e in enumerate(worst_examples): + ref_text = "" + actual_text = "" + diff = "" + for l1, l2 in zip( + e["source_model"].splitlines(), e["optimized_model"].splitlines() + ): + if l1 == "" and l2 == "": + continue + ref_text += l1 + "\n" + actual_text += l2 + "\n" + diff += diff_strings(l1, l2) + "\n" + + print( + "--------------------------------------------------------------------------------------" + ) + print("## Reference text %d:\n%s", i + 1, ref_text) + print("## Actual text %d:\n%s", i + 1, actual_text) + print("## Diff %d: ", i + 1) + print(diff) + @pytest.mark.precommit @pytest.mark.skipif(sys.platform in ("win32", "darwin"), reason="doesn't work on win due to optimum-intel export bug, segfault on mac") @pytest.mark.parametrize("test_struct", [