diff --git a/tests/python_test_cases/BUILD b/tests/python_test_cases/BUILD index b2950bc2..5d6b04f5 100644 --- a/tests/python_test_cases/BUILD +++ b/tests/python_test_cases/BUILD @@ -40,7 +40,7 @@ score_virtualenv( # Tests targets score_py_pytest( name = "cit_cpp", - srcs = glob(["tests/**/*.py"]), + srcs = glob(["tests/**/*.py"]) + ["test_properties.py"], args = [ "-m cpp", "--cpp-target-path=$(rootpath //tests/cpp_test_scenarios)", @@ -58,7 +58,7 @@ score_py_pytest( score_py_pytest( name = "cit_rust", - srcs = glob(["tests/**/*.py"]), + srcs = glob(["tests/**/*.py"]) + ["test_properties.py"], args = [ "-m rust", "--rust-target-path=$(rootpath //tests/rust_test_scenarios)", diff --git a/tests/python_test_cases/pytest.ini b/tests/python_test_cases/pytest.ini index 46f32015..17c10969 100644 --- a/tests/python_test_cases/pytest.ini +++ b/tests/python_test_cases/pytest.ini @@ -13,14 +13,17 @@ [pytest] addopts = -v testpaths = tests +pythonpath = + . + tests markers = cpp rust - PartiallyVerifies - FullyVerifies - Description - TestType - DerivationTechnique + test_properties(dict): Add custom properties to test XML output ; Additional environment variables env = D:RUST_BACKTRACE = 1 +junit_family = xunit1 +filterwarnings = + ignore:record_property is incompatible with junit_family:pytest.PytestWarning + ignore:record_xml_attribute is an experimental feature:pytest.PytestExperimentalApiWarning diff --git a/tests/python_test_cases/test_properties.py b/tests/python_test_cases/test_properties.py new file mode 100644 index 00000000..cbd4da96 --- /dev/null +++ b/tests/python_test_cases/test_properties.py @@ -0,0 +1,22 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +try: + from attribute_plugin import add_test_properties # type: ignore[import-untyped] +except ImportError: + # Define no-op decorator if attribute_plugin is not available (outside bazel) + # Keeps IDE debugging functionality + def add_test_properties(*args, **kwargs): + def decorator(func): + return func # No-op decorator + + return decorator diff --git a/tests/python_test_cases/tests/test_cit_default_values.py b/tests/python_test_cases/tests/test_cit_default_values.py index 9547181a..4398e8a3 100644 --- a/tests/python_test_cases/tests/test_cit_default_values.py +++ b/tests/python_test_cases/tests/test_cit_default_values.py @@ -18,6 +18,7 @@ import pytest from testing_utils import LogContainer, ScenarioResult +from test_properties import add_test_properties from .common import CommonScenario, ResultCode, temp_dir_common @@ -89,22 +90,20 @@ def temp_dir( ) -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + partially_verifies=[ "comp_req__persistency__value_default_v2", "comp_req__persistency__default_value_cfg_v2", "comp_req__persistency__default_value_types_v2", "comp_req__persistency__default_value_query_v2", - ] + ], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Verifies default value loading, querying, and override behavior for KVS instances with and without defaults." -) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") @pytest.mark.parametrize("defaults", ["optional", "required", "without"], scope="class") class TestDefaultValues(DefaultValuesScenario): + """Verifies default value loading, querying, and override behavior for KVS instances with and without defaults.""" + KEY = "test_number" VALUE = 111.1 @@ -174,21 +173,19 @@ def test_valid( assert logs[1].current_value == "Ok(F64(432.1))" -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + partially_verifies=[ "comp_req__persistency__value_default_v2", "comp_req__persistency__default_value_cfg_v2", "comp_req__persistency__default_value_types_v2", - ] + ], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Tests removal of values in KVS with defaults enabled, ensuring keys revert to their default values." -) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") @pytest.mark.parametrize("defaults", ["optional", "required", "without"], scope="class") class TestRemoveKey(DefaultValuesScenario): + """Tests removal of values in KVS with defaults enabled, ensuring keys revert to their default values.""" + KEY = "test_number" VALUE = 111.1 @@ -265,21 +262,19 @@ def test_valid( assert logs[2].current_value == "Err(KeyNotFound)" -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + partially_verifies=[ "comp_req__persistency__value_default_v2", "comp_req__persistency__default_value_cfg_v2", "comp_req__persistency__default_value_types_v2", - ] + ], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Verifies that KVS fails to open when the defaults file contains invalid JSON." -) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") @pytest.mark.parametrize("defaults", ["optional", "required"], scope="class") class TestMalformedDefaultsFile(DefaultValuesScenario): + """Verifies that KVS fails to open when the defaults file contains invalid JSON.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.default_values.default_values" @@ -336,21 +331,19 @@ def test_invalid( assert re.findall(pattern, results.stderr) is not None -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + partially_verifies=[ "comp_req__persistency__value_default_v2", "comp_req__persistency__default_value_cfg_v2", "comp_req__persistency__default_value_types_v2", - ] -) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Verifies that KVS fails to open when the defaults file is missing." + ], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") @pytest.mark.parametrize("defaults", ["required"], scope="class") class TestMissingDefaultsFile(DefaultValuesScenario): + """Verifies that KVS fails to open when the defaults file is missing.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.default_values.default_values" @@ -375,21 +368,20 @@ def test_invalid(self, results: ScenarioResult) -> None: assert re.findall(pattern, results.stderr) is not None -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + fully_verifies=["comp_req__persistency__value_reset_v2"], + partially_verifies=[ "comp_req__persistency__value_default_v2", "comp_req__persistency__default_value_cfg_v2", "comp_req__persistency__default_value_types_v2", - ] -) -@pytest.mark.FullyVerifies(["comp_req__persistency__value_reset_v2"]) -@pytest.mark.Description( - "Checks that resetting KVS restores all keys to their default values." + ], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") @pytest.mark.parametrize("defaults", ["optional", "required"], scope="class") class TestResetAllKeys(DefaultValuesScenario): + """Checks that resetting KVS restores all keys to their default values.""" + NUM_VALUES = 5 @pytest.fixture(scope="class") @@ -446,20 +438,18 @@ def test_valid( assert logs[2].current_value == 432.1 * i -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + partially_verifies=[ "comp_req__persistency__value_default_v2", "comp_req__persistency__default_value_cfg_v2", - ] -) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Checks that resetting single key restores it to its default value." + ], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") @pytest.mark.parametrize("defaults", ["optional", "required"], scope="class") class TestResetSingleKey(DefaultValuesScenario): + """Checks that resetting single key restores it to its default value.""" + NUM_VALUES = 5 RESET_INDEX = 2 @@ -531,20 +521,19 @@ def test_valid( assert logs[2].current_value == 123.4 * i -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + fully_verifies=["comp_req__persistency__default_val_chksum_v2"], + partially_verifies=[ "comp_req__persistency__value_default_v2", "comp_req__persistency__default_value_cfg_v2", - ] -) -@pytest.mark.FullyVerifies(["comp_req__persistency__default_val_chksum_v2"]) -@pytest.mark.Description( - "Ensures that a checksum file is created when opening KVS with defaults." + ], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") @pytest.mark.parametrize("defaults", ["optional", "required"], scope="class") class TestChecksumOnProvidedDefaults(DefaultValuesScenario): + """Ensures that a checksum file is created when opening KVS with defaults.""" + KEY = "test_number" VALUE = 111.1 diff --git a/tests/python_test_cases/tests/test_cit_multiple_kvs.py b/tests/python_test_cases/tests/test_cit_multiple_kvs.py index d4e15d67..db6a3d42 100644 --- a/tests/python_test_cases/tests/test_cit_multiple_kvs.py +++ b/tests/python_test_cases/tests/test_cit_multiple_kvs.py @@ -15,25 +15,24 @@ import pytest from testing_utils import LogContainer, ScenarioResult +from test_properties import add_test_properties from .common import CommonScenario, ResultCode pytestmark = pytest.mark.parametrize("version", ["rust", "cpp"], scope="class") -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + partially_verifies=[ "comp_req__persistency__multi_instance_v2", "comp_req__persistency__concurrency_v2", - ] + ], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Verifies that multiple KVS instances with different IDs store and retrieve independent values without interference." -) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") class TestMultipleInstanceIds(CommonScenario): + """Verifies that multiple KVS instances with different IDs store and retrieve independent values without interference.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.multiple_kvs.multiple_instance_ids" @@ -64,19 +63,17 @@ def test_ok(self, results: ScenarioResult, logs_info_level: LogContainer): assert round(log2.value, 1) == 222.2 -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + partially_verifies=[ "comp_req__persistency__multi_instance_v2", "comp_req__persistency__concurrency_v2", - ] -) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Checks that multiple KVS instances with the same ID and key maintain consistent values across instances." + ], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") class TestSameInstanceIdSameValue(CommonScenario): + """Checks that multiple KVS instances with the same ID and key maintain consistent values across instances.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.multiple_kvs.same_instance_id_same_value" @@ -102,19 +99,17 @@ def test_ok(self, results: ScenarioResult, logs_info_level: LogContainer): assert log1.value == log2.value -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + partially_verifies=[ "comp_req__persistency__multi_instance_v2", "comp_req__persistency__concurrency_v2", - ] -) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Verifies that changes in one KVS instance with a shared ID and key are reflected in another instance, demonstrating interference." + ], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") class TestSameInstanceIdDifferentValue(CommonScenario): + """Verifies that changes in one KVS instance with a shared ID and key are reflected in another instance, demonstrating interference.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.multiple_kvs.same_instance_id_diff_value" diff --git a/tests/python_test_cases/tests/test_cit_persistency.py b/tests/python_test_cases/tests/test_cit_persistency.py index e608885c..de966a0b 100644 --- a/tests/python_test_cases/tests/test_cit_persistency.py +++ b/tests/python_test_cases/tests/test_cit_persistency.py @@ -15,20 +15,21 @@ import pytest from testing_utils import LogContainer, ScenarioResult +from test_properties import add_test_properties from .common import CommonScenario, ResultCode pytestmark = pytest.mark.parametrize("version", ["rust"], scope="class") -@pytest.mark.PartiallyVerifies([]) -@pytest.mark.FullyVerifies(["comp_req__persistency__persist_data_com_v2"]) -@pytest.mark.Description( - "Verifies that disabling flush on exit but manually flushing ensures data is persisted correctly." +@add_test_properties( + fully_verifies=["comp_req__persistency__persist_data_com_v2"], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") class TestExplicitFlush(CommonScenario): + """Verifies that disabling flush on exit but manually flushing ensures data is persisted correctly.""" + NUM_VALUES = 5 @pytest.fixture(scope="class") diff --git a/tests/python_test_cases/tests/test_cit_snapshots.py b/tests/python_test_cases/tests/test_cit_snapshots.py index d8b5077d..aa78ea02 100644 --- a/tests/python_test_cases/tests/test_cit_snapshots.py +++ b/tests/python_test_cases/tests/test_cit_snapshots.py @@ -15,6 +15,7 @@ import pytest from .common import CommonScenario, ResultCode, temp_dir_common from testing_utils import ScenarioResult, LogContainer +from test_properties import add_test_properties pytestmark = pytest.mark.parametrize("version", ["rust", "cpp"], scope="class") @@ -39,15 +40,15 @@ def temp_dir( ) -@pytest.mark.PartiallyVerifies(["comp_req__persistency__snapshot_creation_v2"]) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Verifies that a snapshot is only created after the first flush, and not before." +@add_test_properties( + partially_verifies=["comp_req__persistency__snapshot_creation_v2"], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") @pytest.mark.parametrize("snapshot_max_count", [0, 1, 3, 10], scope="class") class TestSnapshotCountFirstFlush(MaxSnapshotsScenario): + """Verifies that a snapshot is only created after the first flush, and not before.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.snapshots.count" @@ -87,14 +88,14 @@ def test_ok( assert logs[-1].snapshot_count == min(count, snapshot_max_count) -@pytest.mark.PartiallyVerifies(["comp_req__persistency__snapshot_creation_v2"]) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Checks that the snapshot count increases with each flush, up to the maximum allowed count." +@add_test_properties( + partially_verifies=["comp_req__persistency__snapshot_creation_v2"], + test_type="requirements-based", + derivation_technique="requirements-based", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("requirements-based") class TestSnapshotCountFull(TestSnapshotCountFirstFlush): + """ "Checks that the snapshot count increases with each flush, up to the maximum allowed count.""" + @pytest.fixture(scope="class") def test_config(self, temp_dir: Path, snapshot_max_count: int) -> dict[str, Any]: return { @@ -107,15 +108,15 @@ def test_config(self, temp_dir: Path, snapshot_max_count: int) -> dict[str, Any] } -@pytest.mark.PartiallyVerifies(["comp_req__persistency__snapshot_max_num_v2"]) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Verifies that the maximum number of snapshots is a constant value." +@add_test_properties( + partially_verifies=["comp_req__persistency__snapshot_max_num_v2"], + test_type="requirements-based", + derivation_technique="inspection", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("inspection") @pytest.mark.parametrize("snapshot_max_count", [0, 1, 3, 10], scope="class") class TestSnapshotMaxCount(MaxSnapshotsScenario): + """Verifies that the maximum number of snapshots is a constant value.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.snapshots.max_count" @@ -147,20 +148,19 @@ def test_ok( ) -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + fully_verifies=["comp_req__persistency__snapshot_restore_v2"], + partially_verifies=[ "comp_req__persistency__snapshot_creation_v2", "comp_req__persistency__snapshot_rotate_v2", - ] + ], + test_type="requirements-based", + derivation_technique="control-flow-analysis", ) -@pytest.mark.FullyVerifies(["comp_req__persistency__snapshot_restore_v2"]) -@pytest.mark.Description( - "Verifies restoring to a previous snapshot returns the expected value." -) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("control-flow-analysis") @pytest.mark.parametrize("snapshot_max_count", [3, 10], scope="class") class TestSnapshotRestorePrevious(MaxSnapshotsScenario): + """Verifies restoring to a previous snapshot returns the expected value.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.snapshots.restore" @@ -193,14 +193,14 @@ def test_ok( assert value_log.value == 1 -@pytest.mark.PartiallyVerifies(["comp_req__persistency__snapshot_creation_v2"]) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Checks that restoring the current snapshot ID fails with InvalidSnapshotId error." +@add_test_properties( + partially_verifies=["comp_req__persistency__snapshot_creation_v2"], + test_type="requirements-based", + derivation_technique="fault-injection", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("fault-injection") class TestSnapshotRestoreCurrent(CommonScenario): + """Checks that restoring the current snapshot ID fails with InvalidSnapshotId error.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.snapshots.restore" @@ -233,19 +233,17 @@ def test_error( assert result_log.result == "Err(InvalidSnapshotId)" -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + partially_verifies=[ "comp_req__persistency__snapshot_creation_v2", "comp_req__persistency__snapshot_restore_v2", - ] -) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Checks that restoring a non-existing snapshot fails with InvalidSnapshotId error." + ], + test_type="requirements-based", + derivation_technique="fault-injection", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("fault-injection") class TestSnapshotRestoreNonexistent(CommonScenario): + """Checks that restoring a non-existing snapshot fails with InvalidSnapshotId error.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.snapshots.restore" @@ -275,14 +273,14 @@ def test_error( assert result_log.result == "Err(InvalidSnapshotId)" -@pytest.mark.PartiallyVerifies(["comp_req__persistency__snapshot_creation_v2"]) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Verifies that the KVS and hash filenames for an existing snapshot is generated correctly." +@add_test_properties( + partially_verifies=["comp_req__persistency__snapshot_creation_v2"], + test_type="requirements-based", + derivation_technique="interface-test", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("interface-test") class TestSnapshotPathsExist(CommonScenario): + """Verifies that the KVS and hash filenames for an existing snapshot is generated correctly.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.snapshots.paths" @@ -311,14 +309,14 @@ def test_ok( assert Path(paths_log.hash_path).exists() -@pytest.mark.PartiallyVerifies(["comp_req__persistency__snapshot_creation_v2"]) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Checks that requesting the KVS and hash filenames for a non-existing snapshot returns FileNotFound error." +@add_test_properties( + partially_verifies=["comp_req__persistency__snapshot_creation_v2"], + test_type="requirements-based", + derivation_technique="fault-injection", ) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("fault-injection") class TestSnapshotPathsNonexistent(CommonScenario): + """Checks that requesting the KVS and hash filenames for a non-existing snapshot returns FileNotFound error.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.snapshots.paths" diff --git a/tests/python_test_cases/tests/test_cit_supported_datatypes.py b/tests/python_test_cases/tests/test_cit_supported_datatypes.py index f6f948f0..3bda6126 100644 --- a/tests/python_test_cases/tests/test_cit_supported_datatypes.py +++ b/tests/python_test_cases/tests/test_cit_supported_datatypes.py @@ -16,25 +16,24 @@ import pytest from testing_utils import LogContainer, ScenarioResult +from test_properties import add_test_properties from .common import CommonScenario, ResultCode pytestmark = pytest.mark.parametrize("version", ["rust"], scope="class") -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + partially_verifies=[ "comp_req__persistency__key_encoding_v2", "comp_req__persistency__value_data_types_v2", - ] + ], + test_type="requirements-based", + derivation_technique="interface-test", ) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Verifies that KVS supports UTF-8 string keys for storing and retrieving values." -) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("interface-test") class TestSupportedDatatypesKeys(CommonScenario): + """Verifies that KVS supports UTF-8 string keys for storing and retrieving values.""" + @pytest.fixture(scope="class") def scenario_name(self) -> str: return "cit.supported_datatypes.keys" @@ -54,19 +53,17 @@ def test_ok(self, results: ScenarioResult, logs_info_level: LogContainer) -> Non assert len(act_keys.symmetric_difference(exp_keys)) == 0 -@pytest.mark.PartiallyVerifies( - [ +@add_test_properties( + partially_verifies=[ "comp_req__persistency__key_encoding_v2", "comp_req__persistency__value_data_types_v2", - ] + ], + test_type="requirements-based", + derivation_technique="interface-test", ) -@pytest.mark.FullyVerifies([]) -@pytest.mark.Description( - "Verifies that KVS supports UTF-8 string keys for storing and retrieving values." -) -@pytest.mark.TestType("requirements-based") -@pytest.mark.DerivationTechnique("interface-test") class TestSupportedDatatypesValues(CommonScenario): + """Verifies that KVS supports UTF-8 string keys for storing and retrieving values.""" + @abstractmethod def exp_key(self) -> str: pass