diff --git a/src/pytest_flakefighters/plugin.py b/src/pytest_flakefighters/plugin.py index f671bb6..410d728 100644 --- a/src/pytest_flakefighters/plugin.py +++ b/src/pytest_flakefighters/plugin.py @@ -4,6 +4,7 @@ from datetime import datetime from enum import Enum +from re import escape from typing import Union from xml.etree import ElementTree as ET @@ -99,7 +100,7 @@ def pytest_runtest_call(self, item: pytest.Item): item.start = datetime.now().timestamp() self.cov.start() # Lines cannot appear as covered on our tests because the coverage measurement is leaking into the self.cov - self.cov.switch_context(item.nodeid) # pragma: no cover + self.cov.switch_context(escape(item.nodeid)) # pragma: no cover yield # pragma: no cover self.cov.stop() # pragma: no cover item.stop = datetime.now().timestamp() @@ -166,7 +167,7 @@ def pytest_runtest_protocol(self, item: pytest.Item, nextitem: pytest.Item) -> b skipped = True if report.when == "call": line_coverage = self.cov.get_data() - line_coverage.set_query_contexts(["collection", item.nodeid]) + line_coverage.set_query_contexts(["collection", escape(item.nodeid)]) captured_output = dict(report.sections) test_execution = TestExecution( # pylint: disable=E1123 outcome=report.outcome, diff --git a/tests/conftest.py b/tests/conftest.py index f080fb4..c0bbf0c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,6 +31,24 @@ def fixture_flaky_triangle_repo(tmpdir_factory): return repo +@pytest.fixture(scope="function", name="gatorgrade_repo") +def fixture_gatorgrade_repo(tmpdir_factory): + """ + Fixture for a repo containing the gatorgrade test that broke the plugin. + """ + repo_root = tmpdir_factory.mktemp("gatorgrade_repo") + repo = git.Repo.init(repo_root, initial_branch="main") + + shutil.copy(os.path.join(CURRENT_DIR, "resources", "gatorgrade.py"), os.path.join(repo_root, "gatorgrade.py")) + repo.index.add(["gatorgrade.py"]) + repo.index.commit("Initial commit of test file.") + os.chdir(repo_root) + os.mkdir("test_assignment") + with open(os.path.join("test_assignment", "result.txt"), "w", encoding="utf8") as f: + f.write("✓ Complete all TODOs\n✓ Use an if statement\n✓ Complete all TODOs\nPassed 3/3 (100%) of checks") + return repo + + @pytest.fixture(scope="function", name="deflaker_repo") def fixture_deflaker_repo(tmpdir_factory): """ diff --git a/tests/resources/gatorgrade.py b/tests/resources/gatorgrade.py new file mode 100644 index 0000000..e8101ad --- /dev/null +++ b/tests/resources/gatorgrade.py @@ -0,0 +1,28 @@ +import os + +import pytest + + +@pytest.mark.parametrize( + "assignment_path,expected_output_and_freqs", + [ + ( + "test_assignment", + [ + ("Complete all TODOs", 2), + ("Use an if statement", 1), + ("✓", 3), + ("✕", 0), + ("Passed 3/3 (100%) of checks", 1), + ], + ) + ], +) +def test_full_integration_creates_valid_output(assignment_path, expected_output_and_freqs): + """Simplified version of + https://github.com/GatorEducator/gatorgrade/blob/91cb86d5383675c5bc3c95363bc29b45108b2e29/tests/test_main.py#L70 + which initially broke the plugin due to the test IDs contaning [] characters from the parameterisation.""" + with open(os.path.join(assignment_path, "result.txt"), encoding="utf8") as f: + result = f.read() + for output, freq in expected_output_and_freqs: + assert result.count(output) == freq diff --git a/tests/test_end_2_end.py b/tests/test_end_2_end.py index 17873ba..9dd7574 100644 --- a/tests/test_end_2_end.py +++ b/tests/test_end_2_end.py @@ -317,3 +317,11 @@ def test_display_test_level_verdicts(pytester, deflaker_repo): result.assert_outcomes(failed=1) result.stdout.fnmatch_lines(["FAILED app.py::test_app - assert False"]) result.stdout.fnmatch_lines([" CoverageIndependence: genuine"]) + + +def test_gatorgrade_parameterised(pytester, gatorgrade_repo): + """ + Test that flakefighters can run OK on parameterised tests. + """ + result = pytester.runpytest(os.path.join(gatorgrade_repo.working_dir, "gatorgrade.py")) + result.assert_outcomes(passed=1)