From 68ebe1678af275cbb2facd623bc9e34f1101e10a Mon Sep 17 00:00:00 2001 From: Guy Shemesh Date: Sun, 21 Dec 2025 12:10:08 +0200 Subject: [PATCH 1/5] Adding pre-test steps to handle Bug for Local/Remote fault tests, connected to optical module with the supported CMIS Signed-off-by: Guy Shemesh --- tests/common/mellanox_data.py | 92 +++++++++++++++++++ tests/conftest.py | 52 ++++++++++- tests/layer1/test_port_error.py | 154 +++++++++++++++++++++++++------- 3 files changed, 264 insertions(+), 34 deletions(-) diff --git a/tests/common/mellanox_data.py b/tests/common/mellanox_data.py index 9dc67436b60..fcbc1ebdc8f 100644 --- a/tests/common/mellanox_data.py +++ b/tests/common/mellanox_data.py @@ -1,4 +1,7 @@ import functools +import pytest +import logging +logger = logging.getLogger(__name__) SPC1_HWSKUS = ["ACS-MSN2700", "Mellanox-SN2700", "Mellanox-SN2700-D48C8", "ACS-MSN2740", "ACS-MSN2100", "ACS-MSN2410", @@ -88,6 +91,9 @@ } } }, + "x86_64-nvidia_sn5600_simx-r0": { + "chip_type": "spectrum4" + }, "x86_64-nvidia_sn5640-r0": { "chip_type": "spectrum5", "reboot": { @@ -265,6 +271,9 @@ } } }, + "x86_64-mlnx_msn2700_simx-r0": { + "chip_type": "spectrum1" + }, "x86_64-mlnx_msn2700a1-r0": { "chip_type": "spectrum1", "reboot": { @@ -1251,3 +1260,86 @@ def get_hardware_version(duthost, platform): def get_hw_management_version(duthost): full_version = duthost.shell('dpkg-query --showformat=\'${Version}\' --show hw-management')['stdout'] return full_version[len('1.mlnx.'):] + + +def is_pinewave_module(port_info): + """ Check if the given port info indicates an pinewave module and handle known issues """ + vendor_name = port_info.get('Vendor Name', '').upper() + vendor_pn = port_info.get('Vendor PN', '').upper() + manufacturer = port_info.get('manufacturer', '').upper() + + return ( + ('PINEWAVE' in vendor_name and 'T-OH8CNT-NMT' in vendor_pn) or + ('PINEWAVE' in manufacturer) + ) + + +def is_unsupported_module(port_info, port_number): + if is_pinewave_module(port_info): + logger.info(f"Port {port_number} has an unsupported module, skipping it and continue to check other ports") + return True + return False + + +def skip_on_unsupported_module(): + pytest.skip("All ports are with unsupported modules, skipping the test due to Github issue #21878") + + +def is_cmis_version_supported(cmis_version, min_required_version=5.0, failed_api_ports=None, port_name=None): + """ + Check if a CMIS version supports a specific feature by comparing it to a minimum required version + @param: cmis_version: CMIS version string (e.g., "5.0", "4.0", etc.) + @param: min_required_version: Minimum required CMIS version (default: 5.0) + @param: failed_api_ports: List to append failed ports to (optional) + @param: port_name: Port name to append to failed list if version check fails (optional) + @return: bool: True if CMIS version is supported, False otherwise + """ + try: + cmis_version_float = float(cmis_version) + return cmis_version_float >= min_required_version + except (ValueError, TypeError): + if failed_api_ports is not None and port_name is not None: + failed_api_ports.append(port_name) + return False + + +def get_supported_available_optical_interfaces(eeprom_infos, parsed_presence, + min_cmis_version=5.0, return_failed_api_ports=False): + """ + Filter available optical interfaces based on presence, EEPROM detection, media type, and CMIS version support + @param: eeprom_infos: Dictionary containing EEPROM information for each port + @param: parsed_presence: Dictionary containing presence status for each port + @param: min_cmis_version: Minimum required CMIS version (default: 5.0) + @param: return_failed_api_ports: If True, return both available_optical_interfaces and failed_api_ports. + If False, return only available_optical_interfaces (default: False) + @return: list or tuple: If return_failed_api_ports=False, returns list of available optical interface names. + If return_failed_api_ports=True, returns (available_optical_interfaces, failed_api_ports) + """ + available_optical_interfaces = [] + failed_api_ports = [] + + for port_name, eeprom_info in eeprom_infos.items(): + if parsed_presence.get(port_name) != "Present": + continue + if "SFP EEPROM detected" not in eeprom_info[port_name]: + continue + media_technology = eeprom_info.get("Media Interface Technology", "N/A").upper() + if "COPPER" in media_technology: + continue + if "N/A" in media_technology: + failed_api_ports.append(port_name) + continue + cmis_version = eeprom_info.get("CMIS Revision", "N/A") + if "N/A" in cmis_version: + failed_api_ports.append(port_name) + continue + elif not is_cmis_version_supported(cmis_version, min_cmis_version, failed_api_ports, port_name): + logging.info(f"Port {port_name} skipped: CMIS not supported on this port.") + continue + + available_optical_interfaces.append(port_name) + + if return_failed_api_ports: + return available_optical_interfaces, failed_api_ports + else: + return available_optical_interfaces diff --git a/tests/conftest.py b/tests/conftest.py index 76aec33d0a1..7d4b3b11c95 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,6 +37,8 @@ from tests.common.fixtures.ptfhost_utils import ptf_portmap_file # noqa: F401 from tests.common.fixtures.ptfhost_utils import ptf_test_port_map_active_active # noqa: F401 from tests.common.fixtures.ptfhost_utils import run_icmp_responder_session # noqa: F401 +from tests.common.fixtures.grpc_fixtures import ptf_grpc, ptf_gnoi, ptf_grpc_custom, \ + setup_gnoi_tls_server, ptf_gnmi # noqa: F401 from tests.common.dualtor.dual_tor_utils import disable_timed_oscillation_active_standby # noqa: F401 from tests.common.dualtor.dual_tor_utils import config_active_active_dualtor from tests.common.dualtor.dual_tor_common import active_active_ports # noqa: F401 @@ -60,7 +62,7 @@ from tests.common.utilities import str2bool from tests.common.utilities import safe_filename from tests.common.utilities import get_duts_from_host_pattern -from tests.common.utilities import get_upstream_neigh_type, file_exists_on_dut +from tests.common.utilities import get_upstream_neigh_type, get_downstream_neigh_type, file_exists_on_dut from tests.common.helpers.dut_utils import is_supervisor_node, is_frontend_node, create_duthost_console, creds_on_dut, \ is_enabled_nat_for_dpu, get_dpu_names_and_ssh_ports, enable_nat_for_dpus, is_macsec_capable_node from tests.common.cache import FactsCache @@ -84,6 +86,8 @@ from ptf import testutils from ptf.mask import Mask +from tests.common.telemetry.fixtures import db_reporter, ts_reporter # noqa: F401 + logger = logging.getLogger(__name__) cache = FactsCache() @@ -178,6 +182,8 @@ def pytest_addoption(parser): ############################ parser.addoption("--skip_sanity", action="store_true", default=False, help="Skip sanity check") + parser.addoption("--skip_pre_sanity", action="store_true", default=False, + help="Skip pre-test sanity check") parser.addoption("--allow_recover", action="store_true", default=False, help="Allow recovery attempt in sanity check in case of failure") parser.addoption("--check_items", action="store", default=False, @@ -301,6 +307,12 @@ def pytest_addoption(parser): parser.addoption("--container_test", action="store", default="", help="This flag indicates that the test is being run by the container test.") + ################################# + # Port error test options # + ################################# + parser.addoption("--collected-ports-num", action="store", default=5, type=int, + help="Number of ports to collect for testing (default: 5)") + ################################# # YANG validation options # ################################# @@ -347,6 +359,14 @@ def enhance_inventory(request, tbinfo): logger.error("Failed to set enhanced 'ansible_inventory' to request.config.option") +def pytest_cmdline_main(config): + + # Filter out unnecessary logs generated by calling the ptfadapter plugin + dataplane_logger = logging.getLogger("dataplane") + if dataplane_logger: + dataplane_logger.setLevel(logging.ERROR) + + def pytest_collection(session): """Workaround to reduce messy plugin logs generated during collection only @@ -2278,6 +2298,24 @@ def enum_upstream_dut_hostname(duthosts, tbinfo): format(tbinfo["topo"]["type"], upstream_nbr_type)) +@pytest.fixture(scope='module') +def enum_downstream_dut_hostname(duthosts, tbinfo): + s = get_downstream_neigh_type(tbinfo, is_upper=True).split(',') + downstream_nbr_type = [item.strip() for item in s if item.strip()] + if downstream_nbr_type is None: + downstream_nbr_type = "T1" + + for a_dut in duthosts.frontend_nodes: + minigraph_facts = a_dut.get_extended_minigraph_facts(tbinfo) + minigraph_neighbors = minigraph_facts['minigraph_neighbors'] + for key, value in minigraph_neighbors.items(): + if any(downstream_type in value['name'] for downstream_type in downstream_nbr_type): + return a_dut.hostname + + pytest.fail("Did not find a dut in duthosts that for topo type {} that has downstream nbr type {}". + format(tbinfo["topo"]["type"], downstream_nbr_type)) + + @pytest.fixture(scope="module") def duthost_console(duthosts, enum_supervisor_dut_hostname, localhost, conn_graph_facts, creds): # noqa: F811 duthost = duthosts[enum_supervisor_dut_hostname] @@ -2985,13 +3023,19 @@ def _remove_entry(table_name, key_name, config): @pytest.fixture(scope="module", autouse=True) -def temporarily_disable_route_check(request, duthosts): +def temporarily_disable_route_check(request, tbinfo, duthosts): check_flag = False for m in request.node.iter_markers(): if m.name == "disable_route_check": check_flag = True break + allowed_topologies = {"t2", "ut2", "lt2"} + topo_name = tbinfo['topo']['name'] + if check_flag and topo_name not in allowed_topologies: + logger.info("Topology {} is not allowed for temporarily_disable_route_check fixture".format(topo_name)) + check_flag = False + def wait_for_route_check_to_pass(dut): def run_route_check(): @@ -3014,7 +3058,7 @@ def run_route_check(): with SafeThreadPoolExecutor(max_workers=8) as executor: for duthost in duthosts.frontend_nodes: - executor.submit(stop_route_checker_on_duthost, duthost) + executor.submit(stop_route_checker_on_duthost, duthost, wait_for_status=True) yield @@ -3024,7 +3068,7 @@ def run_route_check(): finally: with SafeThreadPoolExecutor(max_workers=8) as executor: for duthost in duthosts.frontend_nodes: - executor.submit(start_route_checker_on_duthost, duthost) + executor.submit(start_route_checker_on_duthost, duthost, wait_for_status=True) else: logger.info("Skipping temporarily_disable_route_check fixture") yield diff --git a/tests/layer1/test_port_error.py b/tests/layer1/test_port_error.py index 2801127146b..deb92bb3cba 100644 --- a/tests/layer1/test_port_error.py +++ b/tests/layer1/test_port_error.py @@ -2,9 +2,14 @@ import pytest import random import time +import os +import re from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import skip_release +from tests.common.platform.transceiver_utils import parse_sfp_eeprom_infos +from tests.common.mellanox_data import get_supported_available_optical_interfaces +from tests.common.utilities import wait_until pytestmark = [ pytest.mark.disable_loganalyzer, # disable automatic loganalyzer @@ -15,9 +20,25 @@ cmd_sfp_presence = "sudo sfpshow presence" +@pytest.fixture(scope="session") +def collected_ports_num(request): + """ + Fixture to get the number of ports to collect from command line argument + """ + return request.config.getoption("--collected-ports-num") + + class TestMACFault(object): - @pytest.fixture(autouse=True) - def is_supported_platform(self, duthost, tbinfo): + @pytest.fixture(scope="class", autouse=True) + def is_supported_nvidia_platform_with_sw_control_disabled(self, duthost): + return 'nvidia' in duthost.facts['platform'].lower() and not self.is_sw_control_feature_enabled(duthost) + + @pytest.fixture(scope="class", autouse=True) + def is_supported_nvidia_platform_with_sw_control_enabled(self, duthost): + return 'nvidia' in duthost.facts['platform'].lower() and self.is_sw_control_feature_enabled(duthost) + + @pytest.fixture(scope="class", autouse=True) + def is_supported_platform(self, duthost, tbinfo, is_supported_nvidia_platform_with_sw_control_disabled): if 'ptp' not in tbinfo['topo']['name']: pytest.skip("Skipping test: Not applicable for non-PTP topology") @@ -26,6 +47,9 @@ def is_supported_platform(self, duthost, tbinfo): else: pytest.skip("DUT has platform {}, test is not supported".format(duthost.facts['platform'])) + if is_supported_nvidia_platform_with_sw_control_disabled: + pytest.skip("SW control feature is not enabled on Nvidia platform") + @staticmethod def get_mac_fault_count(dut, interface, fault_type): output = dut.show_and_parse("show int errors {}".format(interface)) @@ -44,30 +68,85 @@ def get_mac_fault_count(dut, interface, fault_type): def get_interface_status(dut, interface): return dut.show_and_parse("show interfaces status {}".format(interface))[0].get("oper", "unknown") - @pytest.fixture - def select_random_interfaces(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + @pytest.fixture(scope="class", autouse=True) + def reboot_dut(self, duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname): + from tests.common.reboot import reboot + reboot(duthosts[enum_rand_one_per_hwsku_frontend_hostname], + localhost, safe_reboot=True, check_intf_up_ports=True) + + @pytest.fixture(scope="class") + def get_dut_and_supported_available_optical_interfaces(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, + is_supported_nvidia_platform_with_sw_control_enabled): dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - interfaces = list(dut.show_and_parse("show interfaces status")) sfp_presence = dut.command(cmd_sfp_presence) parsed_presence = {line.split()[0]: line.split()[1] for line in sfp_presence["stdout_lines"][2:]} + supported_available_optical_interfaces = [] + failed_api_ports = [] + + if is_supported_nvidia_platform_with_sw_control_enabled: + + eeprom_infos = dut.shell("sudo sfputil show eeprom -d")['stdout'] + eeprom_infos = parse_sfp_eeprom_infos(eeprom_infos) + + supported_available_optical_interfaces, failed_api_ports = ( + get_supported_available_optical_interfaces( + eeprom_infos, parsed_presence, return_failed_api_ports=True + ) + ) + pytest_assert(supported_available_optical_interfaces, + "No interfaces with SFP detected. Cannot proceed with tests.") + logging.info("Available Optical interfaces for tests: {}".format(supported_available_optical_interfaces)) + else: + interfaces = list(dut.show_and_parse("show interfaces status")) + supported_available_optical_interfaces = [ + intf["interface"] for intf in interfaces + if parsed_presence.get(intf["interface"]) == "Present" + ] + + pytest_assert(supported_available_optical_interfaces, + "No interfaces with SFP detected. Cannot proceed with tests.") + + return dut, supported_available_optical_interfaces, failed_api_ports + + def is_sw_control_feature_enabled(self, duthost): + """ + Check if SW control feature is enabled. + """ + try: + platform_name = duthost.facts['platform'] + hwsku = duthost.facts.get('hwsku', '') + sai_profile_path = os.path.join('/usr/share/sonic/device', platform_name, hwsku, 'sai.profile') + cmd = duthost.shell('cat {}'.format(sai_profile_path), module_ignore_errors=True) + if cmd['rc'] == 0 and 'SAI_INDEPENDENT_MODULE_MODE' in cmd['stdout']: + sc_enabled = re.search(r"SAI_INDEPENDENT_MODULE_MODE=(\d?)", cmd['stdout']) + if sc_enabled and sc_enabled.group(1) == '1': + return True + except Exception as e: + logging.error("Error checking SW control feature on Nvidia platform: {}".format(e)) + return False + + def shutdown_and_startup_interfaces(self, dut, interface): + dut.command("sudo config interface shutdown {}".format(interface)) + pytest_assert(wait_until(30, 2, 0, lambda: self.get_interface_status(dut, interface) == "down"), + "Interface {} did not go down after shutdown".format(interface)) + + dut.command("sudo config interface startup {}".format(interface)) + pytest_assert(wait_until(30, 2, 0, lambda: self.get_interface_status(dut, interface) == "up"), + "Interface {} did not come up after startup".format(interface)) + + def test_mac_local_fault_increment(self, get_dut_and_supported_available_optical_interfaces, + collected_ports_num): + dut, supported_available_optical_interfaces, failed_api_ports = ( + get_dut_and_supported_available_optical_interfaces() + ) + selected_interfaces = random.sample(supported_available_optical_interfaces, + min(collected_ports_num, len(supported_available_optical_interfaces))) + logging.info("Selected interfaces for tests: {}".format(selected_interfaces)) + + for interface in selected_interfaces: + self.shutdown_and_startup_interfaces(dut, interface) - available_interfaces = [ - intf["interface"] for intf in interfaces - if parsed_presence.get(intf["interface"]) == "Present" - ] - - pytest_assert(available_interfaces, "No interfaces with SFP detected. Cannot proceed with tests.") - - # Select 5 random interfaces (or fewer if not enough available) - selected_interfaces = random.sample(available_interfaces, min(5, len(available_interfaces))) - - return dut, selected_interfaces - - def test_mac_local_fault_increment(self, select_random_interfaces): - dut, interfaces = select_random_interfaces - - for interface in interfaces: pytest_assert(self.get_interface_status(dut, interface) == "up", "Interface {} was not up before disabling/enabling rx-output using sfputil".format(interface)) @@ -77,24 +156,35 @@ def test_mac_local_fault_increment(self, select_random_interfaces): dut.shell("sudo sfputil debug rx-output {} disable".format(interface)) time.sleep(5) pytest_assert(self.get_interface_status(dut, interface) == "down", - "Interface {} did not go down after disabling rx-output using sfputil".format(interface)) + "Interface {iface} did not go down after 'sudo sfputil debug rx-output {iface} disable'" + .format(iface=interface)) dut.shell("sudo sfputil debug rx-output {} enable".format(interface)) time.sleep(20) pytest_assert(self.get_interface_status(dut, interface) == "up", - "Interface {} did not come up after enabling rx-output using sfputil".format(interface)) + "Interface {iface} did not come up after 'sudo sfputil debug rx-output {iface} enable'" + .format(iface=interface)) local_fault_after = self.get_mac_fault_count(dut, interface, "mac local fault") logging.info("MAC local fault count after disabling/enabling rx-output using sfputil {}: {}".format( interface, local_fault_after)) pytest_assert(local_fault_after > local_fault_before, - "MAC local fault count did not increment after disabling/enabling tx-output on the device") + "MAC local fault count did not increment after disabling/enabling rx-output on the device") - def test_mac_remote_fault_increment(self, select_random_interfaces): - dut, interfaces = select_random_interfaces + pytest_assert(len(failed_api_ports) == 0, "Interfaces with failed API ports: {}".format(failed_api_ports)) + + def test_mac_remote_fault_increment(self, get_dut_and_supported_available_optical_interfaces, collected_ports_num): + dut, supported_available_optical_interfaces, failed_api_ports = ( + get_dut_and_supported_available_optical_interfaces() + ) + selected_interfaces = random.sample(supported_available_optical_interfaces, + min(collected_ports_num, len(supported_available_optical_interfaces))) + logging.info("Selected interfaces for tests: {}".format(selected_interfaces)) + + for interface in selected_interfaces: + self.shutdown_and_startup_interfaces(dut, interface) - for interface in interfaces: pytest_assert(self.get_interface_status(dut, interface) == "up", "Interface {} was not up before disabling/enabling tx-output using sfputil".format(interface)) @@ -104,17 +194,21 @@ def test_mac_remote_fault_increment(self, select_random_interfaces): dut.shell("sudo sfputil debug tx-output {} disable".format(interface)) time.sleep(5) pytest_assert(self.get_interface_status(dut, interface) == "down", - "Interface {} did not go down after disabling rx-output using sfputil".format(interface)) + "Interface {iface} did not go down after 'sudo sfputil debug tx-output {iface} disable'" + .format(iface=interface)) dut.shell("sudo sfputil debug tx-output {} enable".format(interface)) time.sleep(20) pytest_assert(self.get_interface_status(dut, interface) == "up", - "Interface {} did not come up after disabling tx-output using sfputil".format(interface)) + "Interface {iface} did not come up after 'sudo sfputil debug tx-output {iface} enable'" + .format(iface=interface)) remote_fault_after = self.get_mac_fault_count(dut, interface, "mac remote fault") - logging.info("MAC remote fault count after disabling/enabling rx-output using sfputil {}: {}".format( + logging.info("MAC remote fault count after disabling/enabling tx-output using sfputil {}: {}".format( interface, remote_fault_after)) pytest_assert(remote_fault_after > remote_fault_before, "MAC remote fault count did not increment after disabling/enabling tx-output on the device") + + pytest_assert(len(failed_api_ports) == 0, "Interfaces with failed API ports: {}".format(failed_api_ports)) From 25ba787f160e93c836bbc51e4fb3656e0528d3b8 Mon Sep 17 00:00:00 2001 From: gshemesh2 Date: Sun, 21 Dec 2025 13:28:55 +0200 Subject: [PATCH 2/5] Update test_port_error.py Signed-off-by: Guy Shemesh --- tests/layer1/test_port_error.py | 3820 +++++++++++++++++++++++++++++-- 1 file changed, 3661 insertions(+), 159 deletions(-) diff --git a/tests/layer1/test_port_error.py b/tests/layer1/test_port_error.py index deb92bb3cba..51455a61de8 100644 --- a/tests/layer1/test_port_error.py +++ b/tests/layer1/test_port_error.py @@ -1,214 +1,3716 @@ +from functools import lru_cache +import enum +import os +import json import logging -import pytest import random -import time -import os import re +import sys + +import pytest +import yaml +import copy +import time +import subprocess +import threading +import pathlib +import importlib +import inspect + +from datetime import datetime +from ipaddress import ip_interface, IPv4Interface +from tests.common.multi_servers_utils import MultiServersUtils +from tests.common.fixtures.conn_graph_facts import conn_graph_facts # noqa: F401 +from tests.common.devices.local import Localhost +from tests.common.devices.ptf import PTFHost +from tests.common.devices.eos import EosHost +from tests.common.devices.sonic import SonicHost +from tests.common.devices.fanout import FanoutHost +from tests.common.devices.k8s import K8sMasterHost +from tests.common.devices.k8s import K8sMasterCluster +from tests.common.devices.duthosts import DutHosts +from tests.common.devices.vmhost import VMHost +from tests.common.devices.base import NeighborDevice +from tests.common.devices.cisco import CiscoHost +from tests.common.fixtures.duthost_utils import backup_and_restore_config_db_session, \ + stop_route_checker_on_duthost, start_route_checker_on_duthost # noqa: F401 +from tests.common.fixtures.ptfhost_utils import ptf_portmap_file # noqa: F401 +from tests.common.fixtures.ptfhost_utils import ptf_test_port_map_active_active # noqa: F401 +from tests.common.fixtures.ptfhost_utils import run_icmp_responder_session # noqa: F401 +from tests.common.dualtor.dual_tor_utils import disable_timed_oscillation_active_standby # noqa: F401 +from tests.common.dualtor.dual_tor_utils import config_active_active_dualtor +from tests.common.dualtor.dual_tor_common import active_active_ports # noqa: F401 +from tests.common.dualtor import mux_simulator_control # noqa: F401 + +from tests.common.helpers.constants import ( + ASIC_PARAM_TYPE_ALL, ASIC_PARAM_TYPE_FRONTEND, DEFAULT_ASIC_ID, NAMESPACE_PREFIX, + ASICS_PRESENT, DUT_CHECK_NAMESPACE +) +from tests.common.helpers.custom_msg_utils import add_custom_msg +from tests.common.helpers.dut_ports import encode_dut_port_name +from tests.common.helpers.dut_utils import encode_dut_and_container_name +from tests.common.helpers.parallel_utils import ParallelCoordinator, ParallelStatus, ParallelRunContext +from tests.common.helpers.pfcwd_helper import TrafficPorts, select_test_ports, set_pfc_timers +from tests.common.system_utils import docker +from tests.common.testbed import TestbedInfo +from tests.common.utilities import get_inventory_files, wait_until +from tests.common.utilities import get_host_vars +from tests.common.utilities import get_host_visible_vars +from tests.common.utilities import get_test_server_host +from tests.common.utilities import str2bool +from tests.common.utilities import safe_filename +from tests.common.utilities import get_duts_from_host_pattern +from tests.common.utilities import get_upstream_neigh_type, file_exists_on_dut +from tests.common.helpers.dut_utils import is_supervisor_node, is_frontend_node, create_duthost_console, creds_on_dut, \ + is_enabled_nat_for_dpu, get_dpu_names_and_ssh_ports, enable_nat_for_dpus, is_macsec_capable_node +from tests.common.cache import FactsCache +from tests.common.config_reload import config_reload +from tests.common.helpers.assertions import pytest_assert as pt_assert +from tests.common.helpers.inventory_utils import trim_inventory +from tests.common.utilities import InterruptableThread +from tests.common.plugins.ptfadapter.dummy_testutils import DummyTestUtils +from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor + +import tests.common.gnmi_setup as gnmi_setup + +try: + from tests.common.macsec import MacsecPluginT2, MacsecPluginT0 +except ImportError as e: + logging.error(e) + +from tests.common.platform.args.advanced_reboot_args import add_advanced_reboot_args +from tests.common.platform.args.cont_warm_reboot_args import add_cont_warm_reboot_args +from tests.common.platform.args.normal_reboot_args import add_normal_reboot_args +from ptf import testutils +from ptf.mask import Mask + + +logger = logging.getLogger(__name__) +cache = FactsCache() + +DUTHOSTS_FIXTURE_FAILED_RC = 15 +CUSTOM_MSG_PREFIX = "sonic_custom_msg" +GOLDEN_CONFIG_DB_PATH = "/etc/sonic/golden_config_db.json" +GOLDEN_CONFIG_DB_PATH_ORI = "/etc/sonic/golden_config_db.json.origin.backup" + +pytest_plugins = ('tests.common.plugins.ptfadapter', + 'tests.common.plugins.ansible_fixtures', + 'tests.common.plugins.dut_monitor', + 'tests.common.plugins.loganalyzer', + 'tests.common.plugins.pdu_controller', + 'tests.common.plugins.sanity_check', + 'tests.common.plugins.custom_markers', + 'tests.common.plugins.test_completeness', + 'tests.common.plugins.log_section_start', + 'tests.common.plugins.custom_fixtures', + 'tests.common.dualtor', + 'tests.decap', + 'tests.platform_tests.api', + 'tests.common.plugins.allure_server', + 'tests.common.plugins.conditional_mark', + 'tests.common.plugins.random_seed', + 'tests.common.plugins.memory_utilization', + 'tests.common.fixtures.duthost_utils') + + +def pytest_addoption(parser): + parser.addoption("--testbed", action="store", default=None, help="testbed name") + parser.addoption("--testbed_file", action="store", default=None, help="testbed file name") + parser.addoption("--uhd_config", action="store", help="Enable UHD config mode") + parser.addoption("--save_uhd_config", action="store_true", help="Save UHD config mode") + parser.addoption("--npu_dpu_startup", action="store_true", help="Startup NPU and DPUs and install configurations") + parser.addoption("--l47_trafficgen", action="store_true", help="Enable L47 trafficgen config") + parser.addoption("--save_l47_trafficgen", action="store_true", help="Save L47 trafficgen config") + + # test_vrf options + parser.addoption("--vrf_capacity", action="store", default=None, type=int, help="vrf capacity of dut (4-1000)") + parser.addoption("--vrf_test_count", action="store", default=None, type=int, + help="number of vrf to be tested (1-997)") + + # qos_sai options + parser.addoption("--ptf_portmap", action="store", default=None, type=str, + help="PTF port index to DUT port alias map") + parser.addoption("--qos_swap_syncd", action="store", type=str2bool, default=True, + help="Swap syncd container with syncd-rpc container") + + # Kubernetes master options + parser.addoption("--kube_master", action="store", default=None, type=str, + help="Name of k8s master group used in k8s inventory, format: k8s_vms{msetnumber}_{servernumber}") + + # neighbor device type + parser.addoption("--neighbor_type", action="store", default="eos", type=str, choices=["eos", "sonic", "cisco"], + help="Neighbor devices type") + + # ceos neighbor lacp multiplier + parser.addoption("--ceos_neighbor_lacp_multiplier", action="store", default=3, type=int, + help="LACP multiplier for ceos neighbors") + + # FWUtil options + parser.addoption('--fw-pkg', action='store', help='Firmware package file') + + ##################################### + # dash, vxlan, route shared options # + ##################################### + parser.addoption("--skip_cleanup", action="store_true", help="Skip config cleanup after test (tests: dash, vxlan)") + parser.addoption("--num_routes", action="store", default=None, type=int, + help="Number of routes (tests: route, vxlan)") + + ############################ + # pfc_asym options # + ############################ + parser.addoption("--server_ports_num", action="store", default=20, type=int, help="Number of server ports to use") + parser.addoption("--fanout_inventory", action="store", default="lab", help="Inventory with defined fanout hosts") + + ############################ + # test_techsupport options # + ############################ + parser.addoption("--loop_num", action="store", default=2, type=int, + help="Change default loop range for show techsupport command") + parser.addoption("--loop_delay", action="store", default=2, type=int, + help="Change default loops delay") + parser.addoption("--logs_since", action="store", type=int, + help="number of minutes for show techsupport command") + parser.addoption("--collect_techsupport", action="store", default=True, type=str2bool, + help="Enable/Disable tech support collection. Default is enabled (True)") + + ############################ + # sanity_check options # + ############################ + parser.addoption("--skip_sanity", action="store_true", default=False, + help="Skip sanity check") + parser.addoption("--allow_recover", action="store_true", default=False, + help="Allow recovery attempt in sanity check in case of failure") + parser.addoption("--check_items", action="store", default=False, + help="Change (add|remove) check items in the check list") + parser.addoption("--post_check", action="store_true", default=False, + help="Perform post test sanity check if sanity check is enabled") + parser.addoption("--post_check_items", action="store", default=False, + help="Change (add|remove) post test check items based on pre test check items") + parser.addoption("--recover_method", action="store", default="adaptive", + help="Set method to use for recover if sanity failed") + + ######################## + # pre-test options # + ######################## + parser.addoption("--deep_clean", action="store_true", default=False, + help="Deep clean DUT before tests (remove old logs, cores, dumps)") + parser.addoption("--py_saithrift_url", action="store", default=None, type=str, + help="Specify the url of the saithrift package to be installed on the ptf " + "(should be http:///path/python-saithrift_0.9.4_amd64.deb") + + ######################### + # post-test options # + ######################### + parser.addoption("--posttest_show_tech_since", action="store", default="yesterday", + help="collect show techsupport since . should be a string which can " + "be parsed by bash command 'date --d '. Default value is yesterday. " + "To collect all time spans, please use '@0' as the value.") + + ############################ + # keysight ixanvl options # + ############################ + parser.addoption("--testnum", action="store", default=None, type=str) + parser.addoption("--enable-snappi-dynamic-ports", action="store_true", default=False, + help="Force to use dynamic port allocation for snappi port selections") + + ################################## + # advance-reboot,upgrade options # + ################################## + add_advanced_reboot_args(parser) + add_cont_warm_reboot_args(parser) + add_normal_reboot_args(parser) + + ############################ + # loop_times options # + ############################ + parser.addoption("--loop_times", metavar="LOOP_TIMES", action="store", default=1, type=int, + help="Define the loop times of the test") + ############################ + # collect logs option # + ############################ + parser.addoption("--collect_db_data", action="store_true", default=False, help="Collect db info if test failed") + + ############################ + # macsec options # + ############################ + parser.addoption("--enable_macsec", action="store_true", default=False, + help="Enable macsec on some links of testbed") + parser.addoption("--macsec_profile", action="store", default="all", + type=str, help="profile name list in macsec/profile.json") + + ############################ + # QoS options # + ############################ + parser.addoption("--public_docker_registry", action="store_true", default=False, + help="To use public docker registry for syncd swap, by default is disabled (False)") + + ############################## + # ansible inventory option # + ############################## + parser.addoption("--trim_inv", action="store_true", default=False, help="Trim inventory files") + + ############################## + # gnmi connection options # + ############################## + # The gNMI target port number to connect to the DUT gNMI server. + parser.addoption("--gnmi_port", action="store", default="8080", type=str, + help="gNMI target port number") + parser.addoption("--gnmi_insecure", action="store_true", default=True, + help="Use insecure connection to gNMI target") + parser.addoption("--disable_sai_validation", action="store_true", default=True, + help="Disable SAI validation") + ############################ + # Parallel run options # + ############################ + parser.addoption("--target_hostname", action="store", default=None, type=str, + help="Target hostname to run the test in parallel") + parser.addoption("--parallel_state_file", action="store", default=None, type=str, + help="File to store the state of the parallel run") + parser.addoption("--is_parallel_leader", action="store_true", default=False, help="Is the parallel leader") + parser.addoption("--parallel_followers", action="store", default=0, type=int, help="Number of parallel followers") + parser.addoption("--parallel_mode", action="store", default=None, type=str, + help="Parallel mode to run the test. Either FULL_PARALLEL or RP_FIRST if parallel run enabled") + + ############################ + # SmartSwitch options # + ############################ + parser.addoption("--dpu-pattern", action="store", default="all", help="dpu host name") + + ################################## + # Container Upgrade options # + ################################## + parser.addoption("--containers", action="store", default=None, type=str, + help="Container bundle to test on each iteration") + parser.addoption("--os_versions", action="store", default=None, type=str, + help="OS Versions to install, one per iteration") + parser.addoption("--image_url_template", action="store", default=None, type=str, + help="Template url to use to download image") + parser.addoption("--parameters_file", action="store", default=None, type=str, + help="File that containers parameters for each container") + parser.addoption("--testcase_file", action="store", default=None, type=str, + help="File that contains testcases to execute per iteration") + + ################################# + # Stress test options # + ################################# + parser.addoption("--run-stress-tests", action="store_true", default=False, help="Run only tests stress tests") + + ################################# + # Container upgrade test options # + ################################# + parser.addoption("--container_test", action="store", default="", + help="This flag indicates that the test is being run by the container test.") + + ################################# + # YANG validation options # + ################################# + parser.addoption("--skip_yang", action="store_true", default=False, + help="Skip YANG validation") + ################################# + # Port error test options # + ################################# + parser.addoption("--collected-ports-num", action="store", default=5, type=int, + help="Number of ports to collect for testing (default: 5)") + + +def pytest_configure(config): + if config.getoption("enable_macsec"): + topo = config.getoption("topology") + if topo is not None and "t2" in topo: + config.pluginmanager.register(MacsecPluginT2()) + else: + config.pluginmanager.register(MacsecPluginT0()) + + +@pytest.fixture(scope="session", autouse=True) +def enhance_inventory(request, tbinfo): + """ + This fixture is to enhance the capability of parsing the value of pytest cli argument '--inventory'. + The pytest-ansible plugin always assumes that the value of cli argument '--inventory' is a single + inventory file. With this enhancement, we can pass in multiple inventory files using the cli argument + '--inventory'. The multiple inventory files can be separated by comma ','. + + For example: + pytest --inventory "inventory1, inventory2" + pytest --inventory inventory1,inventory2 + + This fixture is automatically applied, you don't need to declare it in your test script. + """ + inv_opt = request.config.getoption("ansible_inventory") + if isinstance(inv_opt, list): + return + inv_files = [inv_file.strip() for inv_file in inv_opt.split(",")] + + if request.config.getoption("trim_inv"): + target_hostname = get_target_hostname(request) + trim_inventory(inv_files, tbinfo, target_hostname) + + try: + logger.info(f"Inventory file: {inv_files}") + setattr(request.config.option, "ansible_inventory", inv_files) + except AttributeError: + logger.error("Failed to set enhanced 'ansible_inventory' to request.config.option") + + +def pytest_collection(session): + """Workaround to reduce messy plugin logs generated during collection only + + Args: + session (ojb): Pytest session object + """ + if session.config.option.collectonly: + root_logger = logging.getLogger() + root_logger.setLevel(logging.WARNING) + + +def get_target_hostname(request): + return request.config.getoption("--target_hostname") + + +def get_parallel_state_file(request): + return request.config.getoption("--parallel_state_file") + + +def is_parallel_run(request): + return get_target_hostname(request) is not None + -from tests.common.helpers.assertions import pytest_assert -from tests.common.utilities import skip_release -from tests.common.platform.transceiver_utils import parse_sfp_eeprom_infos -from tests.common.mellanox_data import get_supported_available_optical_interfaces -from tests.common.utilities import wait_until +def is_parallel_leader(request): + return request.config.getoption("--is_parallel_leader") -pytestmark = [ - pytest.mark.disable_loganalyzer, # disable automatic loganalyzer - pytest.mark.topology('any') -] -SUPPORTED_PLATFORMS = ["arista_7060x6", "nvidia_sn5640", "nvidia_sn5600"] -cmd_sfp_presence = "sudo sfpshow presence" +def get_parallel_followers(request): + return request.config.getoption("--parallel_followers") + + +def get_parallel_mode(request): + return request.config.getoption("--parallel_mode") + + +def get_tbinfo(request): + """ + Helper function to create and return testbed information + """ + tbname = request.config.getoption("--testbed") + tbfile = request.config.getoption("--testbed_file") + if tbname is None or tbfile is None: + raise ValueError("testbed and testbed_file are required!") + + testbedinfo = cache.read(tbname, 'tbinfo') + if testbedinfo is cache.NOTEXIST: + testbedinfo = TestbedInfo(tbfile) + cache.write(tbname, 'tbinfo', testbedinfo) + + return tbname, testbedinfo.testbed_topo.get(tbname, {}) @pytest.fixture(scope="session") -def collected_ports_num(request): +def tbinfo(request): + """ + Create and return testbed information + """ + _, testbedinfo = get_tbinfo(request) + return testbedinfo + + +@pytest.fixture(scope="session") +def parallel_run_context(request): + return ParallelRunContext( + is_parallel_run(request), + get_target_hostname(request), + is_parallel_leader(request), + get_parallel_followers(request), + get_parallel_state_file(request), + get_parallel_mode(request), + ) + + +def get_specified_device_info(request, device_pattern): + """ + Get a list of device hostnames specified with the --host-pattern or --dpu-pattern CLI option """ - Fixture to get the number of ports to collect from command line argument + tbname, tbinfo = get_tbinfo(request) + testbed_duts = tbinfo['duts'] + + if is_parallel_run(request): + return [get_target_hostname(request)] + + host_pattern = request.config.getoption(device_pattern) + if host_pattern == 'all': + if device_pattern == '--dpu-pattern': + testbed_duts = [dut for dut in testbed_duts if 'dpu' in dut] + logger.info(f"dpu duts: {testbed_duts}") + return testbed_duts + else: + specified_duts = get_duts_from_host_pattern(host_pattern) + + if any([dut not in testbed_duts for dut in specified_duts]): + pytest.fail("One of the specified DUTs {} does not belong to the testbed {}".format(specified_duts, tbname)) + + if len(testbed_duts) != specified_duts: + duts = specified_duts + logger.debug("Different DUTs specified than in testbed file, using {}" + .format(str(duts))) + + return duts + + +def get_specified_duts(request): + """ + Get a list of DUT hostnames specified with the --host-pattern CLI option + or -d if using `run_tests.sh` """ - return request.config.getoption("--collected-ports-num") + return get_specified_device_info(request, "--host-pattern") -class TestMACFault(object): - @pytest.fixture(scope="class", autouse=True) - def is_supported_nvidia_platform_with_sw_control_disabled(self, duthost): - return 'nvidia' in duthost.facts['platform'].lower() and not self.is_sw_control_feature_enabled(duthost) +def get_specified_dpus(request): + """ + Get a list of DUT hostnames specified with the --dpu-pattern CLI option + """ + return get_specified_device_info(request, "--dpu-pattern") + + +def pytest_sessionstart(session): + # reset all the sonic_custom_msg keys from cache + # reset here because this fixture will always be very first fixture to be called + cache_dir = session.config.cache._cachedir + keys = [p.name for p in cache_dir.glob('**/*') if p.is_file() and p.name.startswith(CUSTOM_MSG_PREFIX)] + for key in keys: + logger.debug("reset existing key: {}".format(key)) + session.config.cache.set(key, None) - @pytest.fixture(scope="class", autouse=True) - def is_supported_nvidia_platform_with_sw_control_enabled(self, duthost): - return 'nvidia' in duthost.facts['platform'].lower() and self.is_sw_control_feature_enabled(duthost) + # Invoke the build-gnmi-stubs.sh script + script_path = os.path.join(os.path.dirname(__file__), "build-gnmi-stubs.sh") + base_dir = os.getcwd() # Use the current working directory as the base directory + logger.info(f"Invoking {script_path} with base directory: {base_dir}") - @pytest.fixture(scope="class", autouse=True) - def is_supported_platform(self, duthost, tbinfo, is_supported_nvidia_platform_with_sw_control_disabled): - if 'ptp' not in tbinfo['topo']['name']: - pytest.skip("Skipping test: Not applicable for non-PTP topology") + try: + result = subprocess.run( + [script_path, base_dir], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False # Do not raise an exception automatically on non-zero exit + ) + logger.info(f"Output of {script_path}:\n{result.stdout}") + # logger.error(f"Error output of {script_path}:\n{result.stderr}") - if any(platform in duthost.facts['platform'] for platform in SUPPORTED_PLATFORMS): - skip_release(duthost, ["201811", "201911", "202012", "202205", "202211", "202305", "202405"]) + if result.returncode != 0: + logger.error(f"{script_path} failed with exit code {result.returncode}") + session.exitstatus = 1 # Fail the pytest session else: - pytest.skip("DUT has platform {}, test is not supported".format(duthost.facts['platform'])) + # Add the generated directory to sys.path for module imports + generated_path = os.path.join(base_dir, "common", "sai_validation", "generated") + if generated_path not in sys.path: + sys.path.insert(0, generated_path) + logger.info(f"Added {generated_path} to sys.path") + except Exception as e: + logger.error(f"Exception occurred while invoking {script_path}: {e}") + session.exitstatus = 1 # Fail the pytest session + + +def pytest_sessionfinish(session, exitstatus): + if session.config.cache.get("duthosts_fixture_failed", None): + session.config.cache.set("duthosts_fixture_failed", None) + session.exitstatus = DUTHOSTS_FIXTURE_FAILED_RC + - if is_supported_nvidia_platform_with_sw_control_disabled: - pytest.skip("SW control feature is not enabled on Nvidia platform") +@pytest.fixture(name="duthosts", scope="session") +def fixture_duthosts(enhance_inventory, ansible_adhoc, tbinfo, request): + """ + @summary: fixture to get DUT hosts defined in testbed. + @param enhance_inventory: fixture to enhance the capability of parsing the value of pytest cli argument + @param ansible_adhoc: Fixture provided by the pytest-ansible package. + Source of the various device objects. It is + mandatory argument for the class constructors. + @param tbinfo: fixture provides information about testbed. + @param request: pytest request object + """ + try: + host = DutHosts(ansible_adhoc, tbinfo, request, get_specified_duts(request), + target_hostname=get_target_hostname(request), is_parallel_leader=is_parallel_leader(request)) + return host + except BaseException as e: + logger.error("Failed to initialize duthosts.") + request.config.cache.set("duthosts_fixture_failed", True) + pt_assert(False, "!!!!!!!!!!!!!!!! duthosts fixture failed !!!!!!!!!!!!!!!!" + "Exception: {}".format(repr(e))) + + +@pytest.fixture(scope="session") +def duthost(duthosts, request): + ''' + @summary: Shortcut fixture for getting DUT host. For a lengthy test case, test case module can + pass a request to disable sh time out mechanis on dut in order to avoid ssh timeout. + After test case completes, the fixture will restore ssh timeout. + @param duthosts: fixture to get DUT hosts + @param request: request parameters for duthost test fixture + ''' + dut_index = getattr(request.session, "dut_index", 0) + assert dut_index < len(duthosts), \ + "DUT index '{0}' is out of bound '{1}'".format(dut_index, + len(duthosts)) + + duthost = duthosts[dut_index] + + return duthost + + +@pytest.fixture(scope="session") +def enable_nat_for_dpuhosts(duthosts, ansible_adhoc, request): + """ + @summary: fixture to enable nat for dpuhost. + @param duthosts: fixture to get DUT hosts + @param ansible_adhoc: Fixture provided by the pytest-ansible package. + Source of the various device objects. It is + mandatory argument for the class constructors. + @param request: request parameters for duthost test fixture + """ + dpuhost_names = get_specified_dpus(request) + if dpuhost_names: + logging.info(f"dpuhost_names: {dpuhost_names}") + for duthost in duthosts: + if not is_enabled_nat_for_dpu(duthost, request): + dpu_name_ssh_port_dict = get_dpu_names_and_ssh_ports(duthost, dpuhost_names, ansible_adhoc) + enable_nat_for_dpus(duthost, dpu_name_ssh_port_dict, request) + + +@pytest.fixture(name="dpuhosts", scope="session") +def fixture_dpuhosts(enhance_inventory, ansible_adhoc, tbinfo, request, enable_nat_for_dpuhosts): + """ + @summary: fixture to get DPU hosts defined in testbed. + @param ansible_adhoc: Fixture provided by the pytest-ansible package. + Source of the various device objects. It is + mandatory argument for the class constructors. + @param tbinfo: fixture provides information about testbed. + """ + # Before calling dpuhosts, we must enable NAT on NPU. + # E.g. run sonic-dpu-mgmt-traffic.sh on NPU to enable NAT + # sonic-dpu-mgmt-traffic.sh inbound -e --dpus all --ports 5021,5022,5023,5024 + try: + host = DutHosts(ansible_adhoc, tbinfo, request, get_specified_dpus(request), + target_hostname=get_target_hostname(request), is_parallel_leader=is_parallel_leader(request)) + return host + except BaseException as e: + logger.error("Failed to initialize dpuhosts.") + request.config.cache.set("dpuhosts_fixture_failed", True) + pt_assert(False, "!!!!!!!!!!!!!!!! dpuhosts fixture failed !!!!!!!!!!!!!!!!" + "Exception: {}".format(repr(e))) - @staticmethod - def get_mac_fault_count(dut, interface, fault_type): - output = dut.show_and_parse("show int errors {}".format(interface)) - logging.info("Raw output for show int errors on {}: {}".format(interface, output)) - fault_count = 0 - for error_info in output: - if error_info['port errors'] == fault_type: - fault_count = int(error_info['count']) +@pytest.fixture(scope="session") +def dpuhost(dpuhosts, request): + ''' + @summary: Shortcut fixture for getting DPU host. For a lengthy test case, test case module can + pass a request to disable sh time out mechanis on dut in order to avoid ssh timeout. + After test case completes, the fixture will restore ssh timeout. + @param duthosts: fixture to get DPU hosts + @param request: request parameters for duphost test fixture + ''' + dpu_index = getattr(request.session, "dpu_index", 0) + assert dpu_index < len(dpuhosts), \ + "DPU index '{0}' is out of bound '{1}'".format(dpu_index, + len(dpuhosts)) + + duthost = dpuhosts[dpu_index] + + return duthost + + +@pytest.fixture(scope="session") +def mg_facts(duthost): + return duthost.minigraph_facts(host=duthost.hostname)['ansible_facts'] + + +@pytest.fixture(scope="session") +def macsec_duthost(duthosts, tbinfo): + # get the first macsec capable node + macsec_dut = None + if 't2' in tbinfo['topo']['name']: + # currently in the T2 topo only the uplink linecard will have + # macsec enabled + for duthost in duthosts: + if duthost.is_macsec_capable_node(): + macsec_dut = duthost break + else: + return duthosts[0] + return macsec_dut - logging.info("{} count on {}: {}".format(fault_type, interface, fault_count)) - return fault_count - @staticmethod - def get_interface_status(dut, interface): - return dut.show_and_parse("show interfaces status {}".format(interface))[0].get("oper", "unknown") +@pytest.fixture(scope="session") +def is_macsec_enabled_for_test(duthosts): + # If macsec is enabled, use the override option to get macsec profile from golden config + macsec_en = False + request = duthosts.request + if request: + macsec_en = request.config.getoption("--enable_macsec", default=False) + return macsec_en - @pytest.fixture(scope="class", autouse=True) - def reboot_dut(self, duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname): - from tests.common.reboot import reboot - reboot(duthosts[enum_rand_one_per_hwsku_frontend_hostname], - localhost, safe_reboot=True, check_intf_up_ports=True) - @pytest.fixture(scope="class") - def get_dut_and_supported_available_optical_interfaces(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - is_supported_nvidia_platform_with_sw_control_enabled): - dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] +# Make sure in same test module, always use same random DUT +rand_one_dut_hostname_var = None - sfp_presence = dut.command(cmd_sfp_presence) - parsed_presence = {line.split()[0]: line.split()[1] for line in sfp_presence["stdout_lines"][2:]} - supported_available_optical_interfaces = [] - failed_api_ports = [] - if is_supported_nvidia_platform_with_sw_control_enabled: +def set_rand_one_dut_hostname(request): + global rand_one_dut_hostname_var + if rand_one_dut_hostname_var is None: + dut_hostnames = generate_params_dut_hostname(request) + if len(dut_hostnames) > 1: + dut_hostnames = random.sample(dut_hostnames, 1) + rand_one_dut_hostname_var = dut_hostnames[0] + logger.info("Randomly select dut {} for testing".format(rand_one_dut_hostname_var)) - eeprom_infos = dut.shell("sudo sfputil show eeprom -d")['stdout'] - eeprom_infos = parse_sfp_eeprom_infos(eeprom_infos) - supported_available_optical_interfaces, failed_api_ports = ( - get_supported_available_optical_interfaces( - eeprom_infos, parsed_presence, return_failed_api_ports=True - ) +@pytest.fixture(scope="module") +def rand_one_dut_hostname(request): + """ + """ + global rand_one_dut_hostname_var + if rand_one_dut_hostname_var is None: + set_rand_one_dut_hostname(request) + return rand_one_dut_hostname_var + + +@pytest.fixture(scope="module") +def rand_selected_dut(duthosts, rand_one_dut_hostname): + """ + Return the randomly selected duthost + """ + return duthosts[rand_one_dut_hostname] + + +@pytest.fixture(scope="module") +def selected_rand_dut(request): + global rand_one_dut_hostname_var + if rand_one_dut_hostname_var is None: + set_rand_one_dut_hostname(request) + return rand_one_dut_hostname_var + + +@pytest.fixture(scope="module") +def rand_one_dut_front_end_hostname(request): + """ + """ + dut_hostnames = generate_params_frontend_hostname(request) + if len(dut_hostnames) > 1: + dut_hostnames = random.sample(dut_hostnames, 1) + logger.info("Randomly select dut {} for testing".format(dut_hostnames[0])) + return dut_hostnames[0] + + +@pytest.fixture(scope="module") +def rand_one_tgen_dut_hostname(request, tbinfo, rand_one_dut_front_end_hostname, rand_one_dut_hostname): + """ + Return the randomly selected duthost for TGEN test cases + """ + # For T2, we need to skip supervisor, only use linecards. + if 't2' in tbinfo['topo']['name']: + return rand_one_dut_front_end_hostname + return rand_one_dut_hostname + + +@pytest.fixture(scope="module") +def rand_selected_front_end_dut(duthosts, rand_one_dut_front_end_hostname): + """ + Return the randomly selected duthost + """ + return duthosts[rand_one_dut_front_end_hostname] + + +@pytest.fixture(scope="module") +def rand_unselected_dut(request, duthosts, rand_one_dut_hostname): + """ + Return the left duthost after random selection. + Return None for non dualtor testbed + """ + dut_hostnames = generate_params_dut_hostname(request) + if len(dut_hostnames) <= 1: + return None + idx = dut_hostnames.index(rand_one_dut_hostname) + return duthosts[dut_hostnames[1 - idx]] + + +@pytest.fixture(scope="module") +def selected_rand_one_per_hwsku_hostname(request): + """ + Return the selected hostnames for the given module. + This fixture will return the list of selected dut hostnames + when another fixture like enum_rand_one_per_hwsku_hostname + or enum_rand_one_per_hwsku_frontend_hostname is used. + """ + if request.module in _hosts_per_hwsku_per_module: + return _hosts_per_hwsku_per_module[request.module] + else: + return [] + + +@pytest.fixture(scope="module") +def rand_one_dut_portname_oper_up(request): + oper_up_ports = generate_port_lists(request, "oper_up_ports") + if len(oper_up_ports) > 1: + oper_up_ports = random.sample(oper_up_ports, 1) + return oper_up_ports[0] + + +@pytest.fixture(scope="module") +def rand_one_dut_lossless_prio(request): + lossless_prio_list = generate_priority_lists(request, 'lossless') + if len(lossless_prio_list) > 1: + lossless_prio_list = random.sample(lossless_prio_list, 1) + return lossless_prio_list[0] + + +@pytest.fixture(scope="module", autouse=True) +def reset_critical_services_list(duthosts): + """ + Resets the critical services list between test modules to ensure that it is + left in a known state after tests finish running. + """ + [a_dut.critical_services_tracking_list() for a_dut in duthosts] + + +@pytest.fixture(scope="session") +def localhost(ansible_adhoc): + return Localhost(ansible_adhoc) + + +@pytest.fixture(scope="session") +def ptfhost(ptfhosts): + if not ptfhosts: + return ptfhosts + return ptfhosts[0] # For backward compatibility, this is for single ptfhost testbed. + + +@pytest.fixture(scope="session") +def ptfhosts(enhance_inventory, ansible_adhoc, tbinfo, duthost, request): + _hosts = [] + if 'ptp' in tbinfo['topo']['name']: + return None + if tbinfo['topo']['name'].startswith("nut-"): + return None + if "ptf_image_name" in tbinfo and "docker-keysight-api-server" in tbinfo["ptf_image_name"]: + return None + if "ptf" in tbinfo: + _hosts.append(PTFHost(ansible_adhoc, tbinfo["ptf"], duthost, tbinfo, + macsec_enabled=request.config.option.enable_macsec)) + elif "servers" in tbinfo: + for server in tbinfo["servers"].values(): + if "ptf" in server and server["ptf"]: + _host = PTFHost(ansible_adhoc, server["ptf"], duthost, tbinfo, + macsec_enabled=request.config.option.enable_macsec) + _hosts.append(_host) + else: + # when no ptf defined in testbed.csv + # try to parse it from inventory + ptf_host = duthost.host.options["inventory_manager"].get_host(duthost.hostname).get_vars()["ptf_host"] + _hosts.apend(PTFHost(ansible_adhoc, ptf_host, duthost, tbinfo, + macsec_enabled=request.config.option.enable_macsec)) + return _hosts + + +@pytest.fixture(scope="module") +def k8smasters(enhance_inventory, ansible_adhoc, request): + """ + Shortcut fixture for getting Kubernetes master hosts + """ + k8s_master_ansible_group = request.config.getoption("--kube_master") + master_vms = {} + inv_files = request.config.getoption("ansible_inventory") + k8s_inv_file = None + for inv_file in inv_files: + if "k8s" in inv_file: + k8s_inv_file = inv_file + if not k8s_inv_file: + pytest.skip("k8s inventory not found, skipping tests") + with open('../ansible/{}'.format(k8s_inv_file), 'r') as kinv: + k8sinventory = yaml.safe_load(kinv) + for hostname, attributes in list(k8sinventory[k8s_master_ansible_group]['hosts'].items()): + if 'haproxy' in attributes: + is_haproxy = True + else: + is_haproxy = False + master_vms[hostname] = {'host': K8sMasterHost(ansible_adhoc, + hostname, + is_haproxy)} + return master_vms + + +@pytest.fixture(scope="module") +def k8scluster(k8smasters): + k8s_master_cluster = K8sMasterCluster(k8smasters) + return k8s_master_cluster + + +@pytest.fixture(scope="session") +def nbrhosts(enhance_inventory, ansible_adhoc, tbinfo, creds, request): + """ + Shortcut fixture for getting VM host + """ + logger.info("Fixture nbrhosts started") + devices = {} + if ('vm_base' in tbinfo and not tbinfo['vm_base'] and 'tgen' in tbinfo['topo']['name']) or \ + 'ptf' in tbinfo['topo']['name'] or \ + 'ixia' in tbinfo['topo']['name']: + logger.info("No VMs exist for this topology: {}".format(tbinfo['topo']['name'])) + return devices + + neighbor_type = request.config.getoption("--neighbor_type") + if 'VMs' not in tbinfo['topo']['properties']['topology']: + logger.info("No VMs exist for this topology: {}".format(tbinfo['topo']['properties']['topology'])) + return devices + + def initial_neighbor(neighbor_name, vm_name): + logger.info(f"nbrhosts started: {neighbor_name}_{vm_name}") + if neighbor_type == "eos": + device = NeighborDevice( + { + 'host': EosHost( + ansible_adhoc, + vm_name, + creds['eos_login'], + creds['eos_password'], + shell_user=creds['eos_root_user'] if 'eos_root_user' in creds else None, + shell_passwd=creds['eos_root_password'] if 'eos_root_password' in creds else None + ), + 'conf': tbinfo['topo']['properties']['configuration'][neighbor_name] + } ) - pytest_assert(supported_available_optical_interfaces, - "No interfaces with SFP detected. Cannot proceed with tests.") - logging.info("Available Optical interfaces for tests: {}".format(supported_available_optical_interfaces)) - else: - interfaces = list(dut.show_and_parse("show interfaces status")) - supported_available_optical_interfaces = [ - intf["interface"] for intf in interfaces - if parsed_presence.get(intf["interface"]) == "Present" - ] + elif neighbor_type == "sonic": + device = NeighborDevice( + { + 'host': SonicHost( + ansible_adhoc, + vm_name, + ssh_user=creds['sonic_login'] if 'sonic_login' in creds else None, + ssh_passwd=creds['sonic_password'] if 'sonic_password' in creds else None + ), + 'conf': tbinfo['topo']['properties']['configuration'][neighbor_name] + } + ) + elif neighbor_type == "cisco": + device = NeighborDevice( + { + 'host': CiscoHost( + ansible_adhoc, + vm_name, + creds['cisco_login'], + creds['cisco_password'], + ), + 'conf': tbinfo['topo']['properties']['configuration'][neighbor_name] + } + ) + else: + raise ValueError("Unknown neighbor type %s" % (neighbor_type,)) + devices[neighbor_name] = device + logger.info(f"nbrhosts finished: {neighbor_name}_{vm_name}") - pytest_assert(supported_available_optical_interfaces, - "No interfaces with SFP detected. Cannot proceed with tests.") + servers = [] + if 'servers' in tbinfo: + servers.extend(tbinfo['servers'].values()) + elif 'server' in tbinfo: + servers.append(tbinfo) + else: + logger.warning("Unknown testbed schema for setup nbrhosts") - return dut, supported_available_optical_interfaces, failed_api_ports + with SafeThreadPoolExecutor(max_workers=8) as executor: + for server in servers: + vm_base = int(server['vm_base'][2:]) + vm_name_fmt = 'VM%0{}d'.format(len(server['vm_base']) - 2) + vms = MultiServersUtils.get_vms_by_dut_interfaces( + tbinfo['topo']['properties']['topology']['VMs'], + server['dut_interfaces'] + ) if 'dut_interfaces' in server else tbinfo['topo']['properties']['topology']['VMs'] + for neighbor_name, neighbor in vms.items(): + vm_name = vm_name_fmt % (vm_base + neighbor['vm_offset']) + executor.submit(initial_neighbor, neighbor_name, vm_name) + + logger.info("Fixture nbrhosts finished") + return devices - def is_sw_control_feature_enabled(self, duthost): - """ - Check if SW control feature is enabled. - """ - try: - platform_name = duthost.facts['platform'] - hwsku = duthost.facts.get('hwsku', '') - sai_profile_path = os.path.join('/usr/share/sonic/device', platform_name, hwsku, 'sai.profile') - cmd = duthost.shell('cat {}'.format(sai_profile_path), module_ignore_errors=True) - if cmd['rc'] == 0 and 'SAI_INDEPENDENT_MODULE_MODE' in cmd['stdout']: - sc_enabled = re.search(r"SAI_INDEPENDENT_MODULE_MODE=(\d?)", cmd['stdout']) - if sc_enabled and sc_enabled.group(1) == '1': - return True - except Exception as e: - logging.error("Error checking SW control feature on Nvidia platform: {}".format(e)) - return False - def shutdown_and_startup_interfaces(self, dut, interface): - dut.command("sudo config interface shutdown {}".format(interface)) - pytest_assert(wait_until(30, 2, 0, lambda: self.get_interface_status(dut, interface) == "down"), - "Interface {} did not go down after shutdown".format(interface)) +@pytest.fixture(scope="module") +def fanouthosts(enhance_inventory, ansible_adhoc, tbinfo, conn_graph_facts, creds, duthosts): # noqa: F811 + """ + Shortcut fixture for getting Fanout hosts + """ - dut.command("sudo config interface startup {}".format(interface)) - pytest_assert(wait_until(30, 2, 0, lambda: self.get_interface_status(dut, interface) == "up"), - "Interface {} did not come up after startup".format(interface)) + dev_conn = conn_graph_facts.get('device_conn', {}) + fanout_hosts = {} - def test_mac_local_fault_increment(self, get_dut_and_supported_available_optical_interfaces, - collected_ports_num): - dut, supported_available_optical_interfaces, failed_api_ports = ( - get_dut_and_supported_available_optical_interfaces() - ) - selected_interfaces = random.sample(supported_available_optical_interfaces, - min(collected_ports_num, len(supported_available_optical_interfaces))) - logging.info("Selected interfaces for tests: {}".format(selected_interfaces)) + if tbinfo['topo']['name'].startswith('nut-'): + # Nut topology has no fanout + return fanout_hosts + + # WA for virtual testbed which has no fanout + for dut_host, value in list(dev_conn.items()): + duthost = duthosts[dut_host] + if duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0': + continue # skip for kvm platform which has no fanout + mg_facts = duthost.minigraph_facts(host=duthost.hostname)['ansible_facts'] + for dut_port in list(value.keys()): + fanout_rec = value[dut_port] + fanout_host = str(fanout_rec['peerdevice']) + fanout_port = str(fanout_rec['peerport']) - for interface in selected_interfaces: - self.shutdown_and_startup_interfaces(dut, interface) + if fanout_host in list(fanout_hosts.keys()): + fanout = fanout_hosts[fanout_host] + else: + host_vars = ansible_adhoc().options[ + 'inventory_manager'].get_host(fanout_host).vars + os_type = host_vars.get('os', 'eos') + if 'fanout_tacacs_user' in creds: + fanout_user = creds['fanout_tacacs_user'] + fanout_password = creds['fanout_tacacs_password'] + elif 'fanout_tacacs_{}_user'.format(os_type) in creds: + fanout_user = creds['fanout_tacacs_{}_user'.format(os_type)] + fanout_password = creds['fanout_tacacs_{}_password'.format(os_type)] + elif os_type == 'sonic': + fanout_user = creds.get('fanout_sonic_user', None) + fanout_password = creds.get('fanout_sonic_password', None) + elif os_type == 'eos': + fanout_user = creds.get('fanout_network_user', None) + fanout_password = creds.get('fanout_network_password', None) + elif os_type == 'onyx': + fanout_user = creds.get('fanout_mlnx_user', None) + fanout_password = creds.get('fanout_mlnx_password', None) + elif os_type == 'ixia': + # Skip for ixia device which has no fanout + continue + else: + # when os is mellanox, not supported + pytest.fail("os other than sonic and eos not supported") - pytest_assert(self.get_interface_status(dut, interface) == "up", - "Interface {} was not up before disabling/enabling rx-output using sfputil".format(interface)) + eos_shell_user = None + eos_shell_password = None + if os_type == "eos": + admin_user = creds['fanout_admin_user'] + admin_password = creds['fanout_admin_password'] + eos_shell_user = creds.get('fanout_shell_user', admin_user) + eos_shell_password = creds.get('fanout_shell_password', admin_password) - local_fault_before = self.get_mac_fault_count(dut, interface, "mac local fault") - logging.info("Initial MAC local fault count on {}: {}".format(interface, local_fault_before)) + fanout = FanoutHost(ansible_adhoc, + os_type, + fanout_host, + 'FanoutLeaf', + fanout_user, + fanout_password, + eos_shell_user=eos_shell_user, + eos_shell_passwd=eos_shell_password) + fanout.dut_hostnames = [dut_host] + fanout_hosts[fanout_host] = fanout - dut.shell("sudo sfputil debug rx-output {} disable".format(interface)) - time.sleep(5) - pytest_assert(self.get_interface_status(dut, interface) == "down", - "Interface {iface} did not go down after 'sudo sfputil debug rx-output {iface} disable'" - .format(iface=interface)) + if fanout.os == 'sonic': + ifs_status = fanout.host.get_interfaces_status() + for key, interface_info in list(ifs_status.items()): + fanout.fanout_port_alias_to_name[interface_info['alias']] = interface_info['interface'] + logging.info("fanout {} fanout_port_alias_to_name {}" + .format(fanout_host, fanout.fanout_port_alias_to_name)) - dut.shell("sudo sfputil debug rx-output {} enable".format(interface)) - time.sleep(20) - pytest_assert(self.get_interface_status(dut, interface) == "up", - "Interface {iface} did not come up after 'sudo sfputil debug rx-output {iface} enable'" - .format(iface=interface)) + fanout.add_port_map(encode_dut_port_name(dut_host, dut_port), fanout_port) - local_fault_after = self.get_mac_fault_count(dut, interface, "mac local fault") - logging.info("MAC local fault count after disabling/enabling rx-output using sfputil {}: {}".format( - interface, local_fault_after)) + # Add port name to fanout port mapping port if dut_port is alias. + if dut_port in mg_facts['minigraph_port_alias_to_name_map']: + mapped_port = mg_facts['minigraph_port_alias_to_name_map'][dut_port] + # only add the mapped port which isn't in device_conn ports to avoid overwriting port map wrongly, + # it happens when an interface has the same name with another alias, for example: + # Interface Alias + # -------------------- + # Ethernet108 Ethernet32 + # Ethernet32 Ethernet13/1 + if mapped_port not in list(value.keys()): + fanout.add_port_map(encode_dut_port_name(dut_host, mapped_port), fanout_port) - pytest_assert(local_fault_after > local_fault_before, - "MAC local fault count did not increment after disabling/enabling rx-output on the device") + if dut_host not in fanout.dut_hostnames: + fanout.dut_hostnames.append(dut_host) - pytest_assert(len(failed_api_ports) == 0, "Interfaces with failed API ports: {}".format(failed_api_ports)) + return fanout_hosts - def test_mac_remote_fault_increment(self, get_dut_and_supported_available_optical_interfaces, collected_ports_num): - dut, supported_available_optical_interfaces, failed_api_ports = ( - get_dut_and_supported_available_optical_interfaces() + +@pytest.fixture(scope="session") +def vmhost(vmhosts): + if not vmhosts: + return vmhosts + return vmhosts[0] # For backward compatibility, this is for single vmhost testbed. + + +@pytest.fixture(scope="session") +def vmhosts(enhance_inventory, ansible_adhoc, request, tbinfo): + hosts = [] + inv_files = get_inventory_files(request) + if 'ptp' in tbinfo['topo']['name']: + return None + elif "servers" in tbinfo: + for server in tbinfo["servers"].keys(): + vmhost = get_test_server_host(inv_files, server) + hosts.append(VMHost(ansible_adhoc, vmhost.name)) + elif "server" in tbinfo: + server = tbinfo["server"] + vmhost = get_test_server_host(inv_files, server) + hosts.append(VMHost(ansible_adhoc, vmhost.name)) + else: + logger.info("No VM host exist for this topology: {}".format(tbinfo['topo']['name'])) + return hosts + + +@pytest.fixture(scope='session') +def eos(): + """ read and yield eos configuration """ + with open('eos/eos.yml') as stream: + eos = yaml.safe_load(stream) + return eos + + +@pytest.fixture(scope='session') +def sonic(): + """ read and yield sonic configuration """ + with open('sonic/sonic.yml') as stream: + eos = yaml.safe_load(stream) + return eos + + +@pytest.fixture(scope='session') +def pdu(): + """ read and yield pdu configuration """ + with open('../ansible/group_vars/pdu/pdu.yml') as stream: + pdu = yaml.safe_load(stream) + return pdu + + +@pytest.fixture(scope="session") +def creds(duthost): + return creds_on_dut(duthost) + + +@pytest.fixture(scope="session") +def topo_bgp_routes(localhost, ptfhosts, tbinfo): + bgp_routes = {} + topo_name = tbinfo['topo']['name'] + servers_dut_interfaces = None + if 'servers' in tbinfo: + servers_dut_interfaces = {value['ptf_ip'].split("/")[0]: value['dut_interfaces'] + for value in tbinfo['servers'].values()} + for ptfhost in ptfhosts: + ptf_ip = ptfhost.mgmt_ip + res = localhost.announce_routes( + topo_name=topo_name, + ptf_ip=ptf_ip, + action='generate', + path="../ansible/", + log_path="logs", + dut_interfaces=servers_dut_interfaces.get(ptf_ip) if servers_dut_interfaces else None, ) - selected_interfaces = random.sample(supported_available_optical_interfaces, - min(collected_ports_num, len(supported_available_optical_interfaces))) - logging.info("Selected interfaces for tests: {}".format(selected_interfaces)) + if 'topo_routes' not in res: + logger.warning("No routes generated.") + else: + for host in res['topo_routes'].keys(): + if host in bgp_routes: + pytest.fail("Duplicate vm name={} on multiple servers".format(host)) + bgp_routes[host] = res['topo_routes'][host] + return bgp_routes + + +@pytest.fixture(scope='module') +def creds_all_duts(duthosts): + creds_all_duts = dict() + for duthost in duthosts.nodes: + creds_all_duts[duthost.hostname] = creds_on_dut(duthost) + return creds_all_duts + + +def update_custom_msg(custom_msg, key, value): + if custom_msg is None: + custom_msg = {} + chunks = key.split('.') + if chunks[0] == CUSTOM_MSG_PREFIX: + chunks = chunks[1:] + if len(chunks) == 1: + custom_msg.update({chunks[0]: value}) + return custom_msg + if chunks[0] not in custom_msg: + custom_msg[chunks[0]] = {} + custom_msg[chunks[0]] = update_custom_msg(custom_msg[chunks[0]], '.'.join(chunks[1:]), value) + return custom_msg + + +def log_custom_msg(item): + # temp log output to track module name + logger.debug("[log_custom_msg] item: {}".format(item)) + + cache_dir = item.session.config.cache._cachedir + keys = [p.name for p in cache_dir.glob('**/*') if p.is_file() and p.name.startswith(CUSTOM_MSG_PREFIX)] + + custom_msg = {} + for key in keys: + value = item.session.config.cache.get(key, None) + if value is not None: + custom_msg = update_custom_msg(custom_msg, key, value) + + if custom_msg: + logger.debug("append custom_msg: {}".format(custom_msg)) + item.user_properties.append(('CustomMsg', json.dumps(custom_msg))) + + +# This function is a pytest hook implementation that is called to create a test report. +# By placing the call to log_custom_msg in the 'teardown' phase, we ensure that it is executed +# at the end of each test, after all other fixture teardowns. This guarantees that any custom +# messages are logged at the latest possible stage in the test lifecycle. +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + + if call.when == 'setup': + item.user_properties.append(('start', str(datetime.fromtimestamp(call.start)))) + elif call.when == 'teardown': + if item.nodeid == item.session.items[-1].nodeid: + log_custom_msg(item) + item.user_properties.append(('end', str(datetime.fromtimestamp(call.stop)))) + + # Filter out unnecessary logs captured on "stdout" and "stderr" + item._report_sections = list([report for report in item._report_sections if report[1] not in ("stdout", "stderr")]) + + # execute all other hooks to obtain the report object + outcome = yield + rep = outcome.get_result() + + # set a report attribute for each phase of a call, which can + # be "setup", "call", "teardown" + + setattr(item, "rep_" + rep.when, rep) + + +# This function is a pytest hook implementation that is called in runtest call stage. +# We are using this hook to set ptf.testutils to DummyTestUtils if the test is marked with "skip_traffic_test", +# DummyTestUtils would always return True for all verify function in ptf.testutils. +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_call(item): + # See tests/common/plugins/conditional_mark/tests_mark_conditions_skip_traffic_test.yaml + if "skip_traffic_test" in item.keywords: + logger.info("Got skip_traffic_test marker, will skip traffic test") + with DummyTestUtils(): + logger.info("Set ptf.testutils to DummyTestUtils to skip traffic test") + yield + logger.info("Reset ptf.testutils") + else: + yield + + +def collect_techsupport_on_dut(request, a_dut): + # request.node is an "item" because we use the default + # "function" scope + testname = request.node.name + if request.config.getoption("--collect_techsupport") and request.node.rep_call.failed: + res = a_dut.shell("generate_dump -s \"-2 hours\"") + fname = res['stdout_lines'][-1] + a_dut.fetch(src=fname, dest="logs/{}".format(testname)) + + logging.info("########### Collected tech support for test {} ###########".format(testname)) + + +@pytest.fixture +def collect_techsupport(request, duthosts, enum_dut_hostname): + yield + # request.node is an "item" because we use the default + # "function" scope + duthost = duthosts[enum_dut_hostname] + collect_techsupport_on_dut(request, duthost) + + +@pytest.fixture +def collect_techsupport_all_duts(request, duthosts): + yield + [collect_techsupport_on_dut(request, a_dut) for a_dut in duthosts] + + +@pytest.fixture +def collect_techsupport_all_nbrs(request, nbrhosts): + yield + if request.config.getoption("neighbor_type") == "sonic": + [collect_techsupport_on_dut(request, nbrhosts[nbrhost]['host']) for nbrhost in nbrhosts] + + +@pytest.fixture(scope="session", autouse=True) +def tag_test_report(request, pytestconfig, tbinfo, duthost, record_testsuite_property): + if not request.config.getoption("--junit-xml"): + return + + # Test run information + record_testsuite_property("topology", tbinfo["topo"]["name"]) + record_testsuite_property("testbed", tbinfo["conf-name"]) + record_testsuite_property("timestamp", datetime.utcnow()) + + # Device information + record_testsuite_property("host", duthost.hostname) + record_testsuite_property("asic", duthost.facts["asic_type"]) + record_testsuite_property("platform", duthost.facts["platform"]) + record_testsuite_property("hwsku", duthost.facts["hwsku"]) + record_testsuite_property("os_version", duthost.os_version) + + +@pytest.fixture(scope="module", autouse=True) +def clear_neigh_entries(duthosts, tbinfo): + """ + This is a stop bleeding change for dualtor testbed. Because dualtor duts will + learn the same set of arp entries during tests. But currently the test only + cleans up on the dut under test. So the other dut will accumulate arp entries + until kernel start to barf. + Adding this fixture to flush out IPv4/IPv6 static ARP entries after each test + moduel is done. + """ + + yield + + if 'dualtor' in tbinfo['topo']['name']: + for dut in duthosts: + dut.command("sudo ip neigh flush nud permanent") + + +@pytest.fixture(scope="module") +def patch_lldpctl(): + def patch_lldpctl(localhost, duthost): + output = localhost.shell('ansible --version') + if 'ansible 2.8.12' in output['stdout']: + """ + Work around a known lldp module bug in ansible version 2.8.12: + When neighbor sent more than one unknown tlv. Ansible will throw + exception. + This function applies the patch before test. + """ + duthost.shell( + 'sudo sed -i -e \'s/lldp lldpctl "$@"$/lldp lldpctl "$@" | grep -v "unknown-tlvs"/\' /usr/bin/lldpctl' + ) + + return patch_lldpctl + + +@pytest.fixture(scope="module") +def unpatch_lldpctl(): + def unpatch_lldpctl(localhost, duthost): + output = localhost.shell('ansible --version') + if 'ansible 2.8.12' in output['stdout']: + """ + Work around a known lldp module bug in ansible version 2.8.12: + When neighbor sent more than one unknown tlv. Ansible will throw + exception. + This function removes the patch after the test is done. + """ + duthost.shell( + 'sudo sed -i -e \'s/lldp lldpctl "$@"$/lldp lldpctl "$@" | grep -v "unknown-tlvs"/\' /usr/bin/lldpctl' + ) + + return unpatch_lldpctl + + +@pytest.fixture(scope="module") +def disable_container_autorestart(): + def disable_container_autorestart(duthost, testcase="", feature_list=None): + ''' + @summary: Disable autorestart of the features present in feature_list. + + @param duthosts: Instance of DutHost + @param testcase: testcase name used to save pretest autorestart state. Later to be used for restoration. + @feature_list: List of features to disable autorestart. If None, autorestart of all the features will be + disabled. + ''' + command_output = duthost.shell("show feature autorestart", module_ignore_errors=True) + if command_output['rc'] != 0: + logging.info("Feature autorestart utility not supported. Error: {}".format(command_output['stderr'])) + logging.info("Skipping disable_container_autorestart") + return + container_autorestart_states = duthost.get_container_autorestart_states() + state_file_name = "/tmp/autorestart_state_{}_{}.json".format(duthost.hostname, testcase) + # Dump autorestart state to file + with open(state_file_name, "w") as f: + json.dump(container_autorestart_states, f) + # Disable autorestart for all containers + logging.info("Disable container autorestart") + cmd_disable = "config feature autorestart {} disabled" + cmds_disable = [] + for name, state in list(container_autorestart_states.items()): + if state == "enabled" and (feature_list is None or name in feature_list): + cmds_disable.append(cmd_disable.format(name)) + # Write into config_db + cmds_disable.append("config save -y") + duthost.shell_cmds(cmds=cmds_disable) + + return disable_container_autorestart + + +@pytest.fixture(scope="module") +def enable_container_autorestart(): + def enable_container_autorestart(duthost, testcase="", feature_list=None): + ''' + @summary: Enable autorestart of the features present in feature_list. + + @param duthosts: Instance of DutHost + @param testcase: testcase name used to find corresponding file to restore autorestart state. + @feature_list: List of features to enable autorestart. If None, autorestart of all the features will + be disabled. + ''' + state_file_name = "/tmp/autorestart_state_{}_{}.json".format(duthost.hostname, testcase) + if not os.path.exists(state_file_name): + return + stored_autorestart_states = {} + with open(state_file_name, "r") as f: + stored_autorestart_states = json.load(f) + container_autorestart_states = duthost.get_container_autorestart_states() + # Recover autorestart states + logging.info("Recover container autorestart") + cmd_enable = "config feature autorestart {} enabled" + cmds_enable = [] + for name, state in list(container_autorestart_states.items()): + if state == "disabled" and (feature_list is None or name in feature_list) \ + and name in stored_autorestart_states \ + and stored_autorestart_states[name] == "enabled": + cmds_enable.append(cmd_enable.format(name)) + # Write into config_db + cmds_enable.append("config save -y") + duthost.shell_cmds(cmds=cmds_enable) + os.remove(state_file_name) + + return enable_container_autorestart + + +@pytest.fixture(scope='module') +def swapSyncd(request, duthosts, enum_rand_one_per_hwsku_frontend_hostname, creds, tbinfo, lower_tor_host): + """ + Swap syncd on DUT host + + Args: + request (Fixture): pytest request object + duthost (AnsibleHost): Device Under Test (DUT) + + Returns: + None + """ + if 'dualtor' in tbinfo['topo']['name']: + duthost = lower_tor_host + else: + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + swapSyncd = request.config.getoption("--qos_swap_syncd") + public_docker_reg = request.config.getoption("--public_docker_registry") + try: + if swapSyncd: + if public_docker_reg: + new_creds = copy.deepcopy(creds) + new_creds['docker_registry_host'] = new_creds['public_docker_registry_host'] + new_creds['docker_registry_username'] = '' + new_creds['docker_registry_password'] = '' + else: + new_creds = creds + docker.swap_syncd(duthost, new_creds) + + yield + finally: + if swapSyncd: + docker.restore_default_syncd(duthost, new_creds) + + +def get_host_data(request, dut): + ''' + This function parses multple inventory files and returns the dut information present in the inventory + ''' + inv_files = get_inventory_files(request) + return get_host_vars(inv_files, dut) + + +def generate_params_frontend_hostname(request, macsec_only=False): + frontend_duts = [] + tbname, tbinfo = get_tbinfo(request) + duts = get_specified_duts(request) + inv_files = get_inventory_files(request) + host_type = "frontend" + + if macsec_only: + host_type = "macsec" + if 't2' in tbinfo['topo']['name'] and request.config.option.enable_macsec: + # currently in the T2 topo only the uplink linecard will have macsec enabled + # Please add "macsec_card = True" param to inventory the inventory file + # under Line Card with macsec capability. + for dut in duts: + if is_frontend_node(inv_files, dut) and is_macsec_capable_node(inv_files, dut): + frontend_duts.append(dut) + if not frontend_duts: + logging.info("no macsec card found") + else: + frontend_duts.append(duts[0]) + else: + for dut in duts: + if is_frontend_node(inv_files, dut): + frontend_duts.append(dut) + + assert len(frontend_duts) > 0, \ + "Test selected require at-least one {} node, " \ + "none of the DUTs '{}' in testbed '{}' are a {} node".format(host_type, duts, tbname, host_type) + return frontend_duts + + +def generate_params_hostname_rand_per_hwsku(request, frontend_only=False, macsec_only=False): + hosts = get_specified_duts(request) + if frontend_only: + hosts = generate_params_frontend_hostname(request, macsec_only=macsec_only) + + hosts_per_hwsku = get_hosts_per_hwsku(request, hosts) + return hosts_per_hwsku + + +def get_hosts_per_hwsku(request, hosts): + inv_files = get_inventory_files(request) + # Create a list of hosts per hwsku + host_hwskus = {} + for a_host in hosts: + host_vars = get_host_visible_vars(inv_files, a_host) + a_host_hwsku = None + if 'hwsku' in host_vars: + a_host_hwsku = host_vars['hwsku'] + else: + # Lets try 'sonic_hwsku' as well + if 'sonic_hwsku' in host_vars: + a_host_hwsku = host_vars['sonic_hwsku'] + if a_host_hwsku: + if a_host_hwsku not in host_hwskus: + host_hwskus[a_host_hwsku] = [a_host] + else: + host_hwskus[a_host_hwsku].append(a_host) + else: + pytest.fail("Test selected require a node per hwsku, but 'hwsku' for '{}' not defined in the inventory" + .format(a_host)) + + hosts_per_hwsku = [] + for hosts in list(host_hwskus.values()): + if len(hosts) == 1: + hosts_per_hwsku.append(hosts[0]) + else: + hosts_per_hwsku.extend(random.sample(hosts, 1)) + + return hosts_per_hwsku + + +def generate_params_supervisor_hostname(request): + duts = get_specified_duts(request) + if len(duts) == 1: + # We have a single node - dealing with pizza box, return it + return [duts[0]] + inv_files = get_inventory_files(request) + for dut in duts: + # Expecting only a single supervisor node + if is_supervisor_node(inv_files, dut): + return [dut] + # If there are no supervisor cards in a multi-dut tesbed, we are dealing with all pizza box in the testbed, + # pick the first DUT + return [duts[0]] + + +def generate_param_asic_index(request, dut_hostnames, param_type, random_asic=False): + _, tbinfo = get_tbinfo(request) + inv_files = get_inventory_files(request) + logging.info("generating {} asic indicies for DUT [{}] in ".format(param_type, dut_hostnames)) + + asic_index_params = [] + for dut in dut_hostnames: + inv_data = get_host_visible_vars(inv_files, dut) + # if the params are not present treat the device as a single asic device + dut_asic_params = [DEFAULT_ASIC_ID] + if inv_data: + if param_type == ASIC_PARAM_TYPE_ALL and ASIC_PARAM_TYPE_ALL in inv_data: + if int(inv_data[ASIC_PARAM_TYPE_ALL]) == 1: + dut_asic_params = [DEFAULT_ASIC_ID] + else: + if ASICS_PRESENT in inv_data: + dut_asic_params = inv_data[ASICS_PRESENT] + else: + dut_asic_params = list(range(int(inv_data[ASIC_PARAM_TYPE_ALL]))) + elif param_type == ASIC_PARAM_TYPE_FRONTEND and ASIC_PARAM_TYPE_FRONTEND in inv_data: + dut_asic_params = inv_data[ASIC_PARAM_TYPE_FRONTEND] + logging.info("dut name {} asics params = {}".format(dut, dut_asic_params)) + + if random_asic: + asic_index_params.append(random.sample(dut_asic_params, 1)) + else: + asic_index_params.append(dut_asic_params) + return asic_index_params + + +def generate_params_dut_index(request): + tbname, _ = get_tbinfo(request) + num_duts = len(get_specified_duts(request)) + logging.info("Using {} duts from testbed '{}'".format(num_duts, tbname)) + + return list(range(num_duts)) + + +def generate_params_dut_hostname(request): + tbname, _ = get_tbinfo(request) + duts = get_specified_duts(request) + logging.info("Using DUTs {} in testbed '{}'".format(str(duts), tbname)) + + return duts + + +def get_completeness_level_metadata(request): + completeness_level = request.config.getoption("--completeness_level") + # if completeness_level is not set or an unknown completeness_level is set + # return "thorough" to run all test set + if not completeness_level or completeness_level not in ["debug", "basic", "confident", "thorough"]: + return "debug" + return completeness_level + + +def get_testbed_metadata(request): + """ + Get the metadata for the testbed name. Return None if tbname is + not provided, or metadata file not found or metadata does not + contain tbname + """ + tbname = request.config.getoption("--testbed") + if not tbname: + return None + + folder = 'metadata' + filepath = os.path.join(folder, tbname + '.json') + metadata = None + + try: + with open(filepath, 'r') as yf: + metadata = json.load(yf) + except IOError: + return None + + return metadata.get(tbname) + + +def get_snappi_testbed_metadata(request): + """ + Get the metadata for the testbed name. Return None if tbname is + not provided, or metadata file not found or metadata does not + contain tbname + """ + tbname = request.config.getoption("--testbed") + if not tbname: + return None + + folder = 'metadata/snappi_tests' + filepath = os.path.join(folder, tbname + '.json') + metadata = None + + try: + with open(filepath, 'r') as yf: + metadata = json.load(yf) + except IOError: + return None + + return metadata.get(tbname) + + +def generate_port_lists(request, port_scope, with_completeness_level=False): + empty = [encode_dut_port_name('unknown', 'unknown')] + if 'ports' in port_scope: + scope = 'Ethernet' + elif 'pcs' in port_scope: + scope = 'PortChannel' + else: + return empty + + if 'all' in port_scope: + state = None + elif 'oper_up' in port_scope: + state = 'oper_state' + elif 'admin_up' in port_scope: + state = 'admin_state' + else: + return empty + + dut_ports = get_testbed_metadata(request) + + if dut_ports is None: + return empty + + dut_port_map = {} + for dut, val in list(dut_ports.items()): + dut_port_pairs = [] + if 'intf_status' not in val: + continue + for intf, status in list(val['intf_status'].items()): + if scope in intf and (not state or status[state] == 'up'): + dut_port_pairs.append(encode_dut_port_name(dut, intf)) + dut_port_map[dut] = dut_port_pairs + logger.info("Generate dut_port_map: {}".format(dut_port_map)) + + if with_completeness_level: + completeness_level = get_completeness_level_metadata(request) + # if completeness_level in ["debug", "basic", "confident"], + # only select several ports on every DUT to save test time + + def trim_dut_port_lists(dut_port_list, target_len): + if len(dut_port_list) <= target_len: + return dut_port_list + # for diversity, fetch the ports from both the start and the end of the original list + pos_1 = target_len // 2 + pos_2 = target_len - pos_1 + return dut_ports[:pos_1] + dut_ports[-pos_2:] + + if completeness_level in ["debug"]: + for dut, dut_ports in list(dut_port_map.items()): + dut_port_map[dut] = trim_dut_port_lists(dut_ports, 1) + elif completeness_level in ["basic", "confident"]: + for dut, dut_ports in list(dut_port_map.items()): + dut_port_map[dut] = trim_dut_port_lists(dut_ports, 4) + + ret = sum(list(dut_port_map.values()), []) + logger.info("Generate port_list: {}".format(ret)) + return ret if ret else empty + + +def generate_dut_feature_container_list(request): + """ + Generate list of containers given the list of features. + List of features and container names are both obtained from + metadata file + """ + empty = [encode_dut_and_container_name("unknown", "unknown")] + + meta = get_testbed_metadata(request) + + if meta is None: + return empty + + container_list = [] + + for dut, val in list(meta.items()): + if "features" not in val: + continue + for feature in list(val["features"].keys()): + if "disabled" in val["features"][feature]: + continue + + dut_info = meta[dut] + + if "asic_services" in dut_info and dut_info["asic_services"].get(feature) is not None: + for service in dut_info["asic_services"].get(feature): + container_list.append(encode_dut_and_container_name(dut, service)) + else: + container_list.append(encode_dut_and_container_name(dut, feature)) + + return container_list + + +def generate_dut_feature_list(request, duts_selected, asics_selected): + """ + Generate a list of features. + The list of features willl be obtained from + metadata file. + This list will be features that can be stopped + or restarted. + """ + meta = get_testbed_metadata(request) + tuple_list = [] + + if meta is None: + return tuple_list + + skip_feature_list = ['database', 'database-chassis', 'gbsyncd'] + + for a_dut_index, a_dut in enumerate(duts_selected): + if len(asics_selected): + for a_asic in asics_selected[a_dut_index]: + # Create tuple of dut and asic index + if "features" in meta[a_dut]: + for a_feature in list(meta[a_dut]["features"].keys()): + if a_feature not in skip_feature_list: + tuple_list.append((a_dut, a_asic, a_feature)) + else: + tuple_list.append((a_dut, a_asic, None)) + else: + if "features" in meta[a_dut]: + for a_feature in list(meta[a_dut]["features"].keys()): + if a_feature not in skip_feature_list: + tuple_list.append((a_dut, None, a_feature)) + else: + tuple_list.append((a_dut, None, None)) + return tuple_list + + +def generate_dut_backend_asics(request, duts_selected): + dut_asic_list = [] + + metadata = get_testbed_metadata(request) + + if metadata is None: + return [[None]]*len(duts_selected) + + for dut in duts_selected: + mdata = metadata.get(dut) + if mdata is None: + dut_asic_list.append([None]) + dut_asic_list.append(mdata.get("backend_asics", [None])) + + return dut_asic_list + + +def generate_priority_lists(request, prio_scope, with_completeness_level=False, one_dut_only=False): + empty = [] + + tbname = request.config.getoption("--testbed") + if not tbname: + return empty + + folder = 'priority' + filepath = os.path.join(folder, tbname + '-' + prio_scope + '.json') + + try: + with open(filepath, 'r') as yf: + info = json.load(yf) + except IOError: + return empty + + if tbname not in info: + return empty + + dut_prio = info[tbname] + ret = [] + + for dut, priorities in list(dut_prio.items()): + for p in priorities: + ret.append('{}|{}'.format(dut, p)) + + if one_dut_only: + break + + if with_completeness_level: + completeness_level = get_completeness_level_metadata(request) + # if completeness_level in ["debug", "basic", "confident"], + # select a small subnet to save test time + # if completeness_level in ["debug"], only select one item + # if completeness_level in ["basic", "confident"], select 1 priority per DUT + + if completeness_level in ["debug"] and ret: + ret = random.sample(ret, 1) + elif completeness_level in ["basic", "confident"]: + ret = [] + for dut, priorities in list(dut_prio.items()): + if priorities: + p = random.choice(priorities) + ret.append('{}|{}'.format(dut, p)) + + if one_dut_only: + break + + return ret if ret else empty + + +def pfc_pause_delay_test_params(request): + empty = [] + + tbname = request.config.getoption("--testbed") + if not tbname: + return empty + + folder = 'pfc_headroom_test_params' + filepath = os.path.join(folder, tbname + '.json') + + try: + with open(filepath, 'r') as yf: + info = json.load(yf) + except IOError: + return empty + + if tbname not in info: + return empty + + dut_pfc_delay_params = info[tbname] + ret = [] + + for dut, pfc_pause_delay_params in list(dut_pfc_delay_params.items()): + for pfc_delay, headroom_result in list(pfc_pause_delay_params.items()): + ret.append('{}|{}|{}'.format(dut, pfc_delay, headroom_result)) + + return ret if ret else empty + + +_frontend_hosts_per_hwsku_per_module = {} +_hosts_per_hwsku_per_module = {} +_rand_one_asic_per_module = {} +_rand_one_frontend_asic_per_module = {} +_macsec_frontend_hosts_per_hwsku_per_module = {} +def pytest_generate_tests(metafunc): # noqa: E302 + # The topology always has atleast 1 dut + dut_fixture_name = None + duts_selected = None + global _frontend_hosts_per_hwsku_per_module, _hosts_per_hwsku_per_module + global _macsec_frontend_hosts_per_hwsku_per_module + global _rand_one_asic_per_module, _rand_one_frontend_asic_per_module + # Enumerators for duts are mutually exclusive + target_hostname = get_target_hostname(metafunc) + if target_hostname: + duts_selected = [target_hostname] + if "enum_dut_hostname" in metafunc.fixturenames: + dut_fixture_name = "enum_dut_hostname" + elif "enum_supervisor_dut_hostname" in metafunc.fixturenames: + dut_fixture_name = "enum_supervisor_dut_hostname" + elif "enum_frontend_dut_hostname" in metafunc.fixturenames: + dut_fixture_name = "enum_frontend_dut_hostname" + elif "enum_rand_one_per_hwsku_hostname" in metafunc.fixturenames: + if metafunc.module not in _hosts_per_hwsku_per_module: + _hosts_per_hwsku_per_module[metafunc.module] = duts_selected + + dut_fixture_name = "enum_rand_one_per_hwsku_hostname" + elif "enum_rand_one_per_hwsku_frontend_hostname" in metafunc.fixturenames: + if metafunc.module not in _frontend_hosts_per_hwsku_per_module: + _frontend_hosts_per_hwsku_per_module[metafunc.module] = duts_selected + + dut_fixture_name = "enum_rand_one_per_hwsku_frontend_hostname" + elif "enum_rand_one_per_hwsku_macsec_frontend_hostname" in metafunc.fixturenames: + if metafunc.module not in _macsec_frontend_hosts_per_hwsku_per_module: + _macsec_frontend_hosts_per_hwsku_per_module[metafunc.module] = duts_selected + dut_fixture_name = "enum_rand_one_per_hwsku_macsec_frontend_hostname" + else: + if "enum_dut_hostname" in metafunc.fixturenames: + duts_selected = generate_params_dut_hostname(metafunc) + dut_fixture_name = "enum_dut_hostname" + elif "enum_supervisor_dut_hostname" in metafunc.fixturenames: + duts_selected = generate_params_supervisor_hostname(metafunc) + dut_fixture_name = "enum_supervisor_dut_hostname" + elif "enum_frontend_dut_hostname" in metafunc.fixturenames: + duts_selected = generate_params_frontend_hostname(metafunc) + dut_fixture_name = "enum_frontend_dut_hostname" + elif "enum_rand_one_per_hwsku_hostname" in metafunc.fixturenames: + if metafunc.module not in _hosts_per_hwsku_per_module: + hosts_per_hwsku = generate_params_hostname_rand_per_hwsku(metafunc) + _hosts_per_hwsku_per_module[metafunc.module] = hosts_per_hwsku + duts_selected = _hosts_per_hwsku_per_module[metafunc.module] + dut_fixture_name = "enum_rand_one_per_hwsku_hostname" + elif "enum_rand_one_per_hwsku_frontend_hostname" in metafunc.fixturenames: + if metafunc.module not in _frontend_hosts_per_hwsku_per_module: + hosts_per_hwsku = generate_params_hostname_rand_per_hwsku(metafunc, frontend_only=True) + _frontend_hosts_per_hwsku_per_module[metafunc.module] = hosts_per_hwsku + duts_selected = _frontend_hosts_per_hwsku_per_module[metafunc.module] + dut_fixture_name = "enum_rand_one_per_hwsku_frontend_hostname" + elif "enum_rand_one_per_hwsku_macsec_frontend_hostname" in metafunc.fixturenames: + if metafunc.module not in _macsec_frontend_hosts_per_hwsku_per_module: + hosts_per_hwsku = generate_params_hostname_rand_per_hwsku( + metafunc, frontend_only=True, macsec_only=True + ) + _macsec_frontend_hosts_per_hwsku_per_module[metafunc.module] = hosts_per_hwsku + duts_selected = _macsec_frontend_hosts_per_hwsku_per_module[metafunc.module] + dut_fixture_name = "enum_rand_one_per_hwsku_macsec_frontend_hostname" + + asics_selected = None + asic_fixture_name = None + + tbname, tbinfo = get_tbinfo(metafunc) + if duts_selected is None: + duts_selected = [tbinfo["duts"][0]] + + possible_asic_enums = ["enum_asic_index", "enum_frontend_asic_index", "enum_backend_asic_index", + "enum_rand_one_asic_index", "enum_rand_one_frontend_asic_index"] + enums_asic_fixtures = set(metafunc.fixturenames).intersection(possible_asic_enums) + assert len(enums_asic_fixtures) < 2, \ + "The number of asic_enum fixtures should be 1 or zero, " \ + "the following fixtures conflict one with each other: {}".format(str(enums_asic_fixtures)) + + if "enum_asic_index" in metafunc.fixturenames: + asic_fixture_name = "enum_asic_index" + asics_selected = generate_param_asic_index(metafunc, duts_selected, ASIC_PARAM_TYPE_ALL) + elif "enum_frontend_asic_index" in metafunc.fixturenames: + asic_fixture_name = "enum_frontend_asic_index" + asics_selected = generate_param_asic_index(metafunc, duts_selected, ASIC_PARAM_TYPE_FRONTEND) + elif "enum_backend_asic_index" in metafunc.fixturenames: + asic_fixture_name = "enum_backend_asic_index" + asics_selected = generate_dut_backend_asics(metafunc, duts_selected) + elif "enum_rand_one_asic_index" in metafunc.fixturenames: + asic_fixture_name = "enum_rand_one_asic_index" + if metafunc.module not in _rand_one_asic_per_module: + asics_selected = generate_param_asic_index(metafunc, duts_selected, + ASIC_PARAM_TYPE_ALL, random_asic=True) + _rand_one_asic_per_module[metafunc.module] = asics_selected + asics_selected = _rand_one_asic_per_module[metafunc.module] + elif "enum_rand_one_frontend_asic_index" in metafunc.fixturenames: + asic_fixture_name = "enum_rand_one_frontend_asic_index" + if metafunc.module not in _rand_one_frontend_asic_per_module: + asics_selected = generate_param_asic_index(metafunc, duts_selected, + ASIC_PARAM_TYPE_FRONTEND, random_asic=True) + _rand_one_frontend_asic_per_module[metafunc.module] = asics_selected + asics_selected = _rand_one_frontend_asic_per_module[metafunc.module] + + # Create parameterization tuple of dut_fixture_name, asic_fixture_name and feature to parameterize + if dut_fixture_name and asic_fixture_name and ("enum_dut_feature" in metafunc.fixturenames): + tuple_list = generate_dut_feature_list(metafunc, duts_selected, asics_selected) + feature_fixture = "enum_dut_feature" + metafunc.parametrize(dut_fixture_name + "," + asic_fixture_name + "," + feature_fixture, + tuple_list, scope="module", indirect=True) + # Create parameterization tuple of dut_fixture_name and asic_fixture_name to parameterize + elif dut_fixture_name and asic_fixture_name: + # parameterize on both - create tuple for each + tuple_list = [] + for a_dut_index, a_dut in enumerate(duts_selected): + if len(asics_selected): + for a_asic in asics_selected[a_dut_index]: + # Create tuple of dut and asic index + tuple_list.append((a_dut, a_asic)) + else: + tuple_list.append((a_dut, None)) + metafunc.parametrize(dut_fixture_name + "," + asic_fixture_name, tuple_list, scope="module", indirect=True) + elif dut_fixture_name: + # parameterize only on DUT + metafunc.parametrize(dut_fixture_name, duts_selected, scope="module", indirect=True) + elif asic_fixture_name: + # We have no duts selected, so need asic list for the first DUT + if len(asics_selected): + metafunc.parametrize(asic_fixture_name, asics_selected[0], scope="module", indirect=True) + else: + metafunc.parametrize(asic_fixture_name, [None], scope="module", indirect=True) + + # When selected_dut used and select a dut for test, parameterize dut for enable TACACS on all UT + if dut_fixture_name and "selected_dut" in metafunc.fixturenames: + metafunc.parametrize("selected_dut", duts_selected, scope="module", indirect=True) + + if "enum_dut_portname" in metafunc.fixturenames: + metafunc.parametrize("enum_dut_portname", generate_port_lists(metafunc, "all_ports")) + + def format_portautoneg_test_id(param): + speeds = param['speeds'] if 'speeds' in param else [param['speed']] + return "{}|{}|{}".format(param['dutname'], param['port'], ','.join(speeds)) + + if "enum_dut_portname_module_fixture" in metafunc.fixturenames or \ + "enum_speed_per_dutport_fixture" in metafunc.fixturenames: + autoneg_tests_data = get_autoneg_tests_data() + if "enum_dut_portname_module_fixture" in metafunc.fixturenames: + metafunc.parametrize( + "enum_dut_portname_module_fixture", + autoneg_tests_data, + scope="module", + ids=format_portautoneg_test_id, + indirect=True + ) + + if "enum_speed_per_dutport_fixture" in metafunc.fixturenames: + metafunc.parametrize( + "enum_speed_per_dutport_fixture", + parametrise_per_supported_port_speed(autoneg_tests_data), + scope="module", + ids=format_portautoneg_test_id, + indirect=True + ) + + if "enum_dut_portname_oper_up" in metafunc.fixturenames: + metafunc.parametrize("enum_dut_portname_oper_up", generate_port_lists(metafunc, "oper_up_ports")) + if "enum_dut_portname_admin_up" in metafunc.fixturenames: + metafunc.parametrize("enum_dut_portname_admin_up", generate_port_lists(metafunc, "admin_up_ports")) + if "enum_dut_portchannel" in metafunc.fixturenames: + metafunc.parametrize("enum_dut_portchannel", generate_port_lists(metafunc, "all_pcs")) + if "enum_dut_portchannel_oper_up" in metafunc.fixturenames: + metafunc.parametrize("enum_dut_portchannel_oper_up", generate_port_lists(metafunc, "oper_up_pcs")) + if "enum_dut_portchannel_admin_up" in metafunc.fixturenames: + metafunc.parametrize("enum_dut_portchannel_admin_up", generate_port_lists(metafunc, "admin_up_pcs")) + if "enum_dut_portchannel_with_completeness_level" in metafunc.fixturenames: + metafunc.parametrize("enum_dut_portchannel_with_completeness_level", + generate_port_lists(metafunc, "all_pcs", with_completeness_level=True)) + if "enum_dut_feature_container" in metafunc.fixturenames: + metafunc.parametrize( + "enum_dut_feature_container", generate_dut_feature_container_list(metafunc) + ) + if 'enum_dut_all_prio' in metafunc.fixturenames: + metafunc.parametrize("enum_dut_all_prio", generate_priority_lists(metafunc, 'all')) + if 'enum_dut_lossless_prio' in metafunc.fixturenames: + metafunc.parametrize("enum_dut_lossless_prio", generate_priority_lists(metafunc, 'lossless')) + if 'enum_one_dut_lossless_prio' in metafunc.fixturenames: + metafunc.parametrize("enum_one_dut_lossless_prio", + generate_priority_lists(metafunc, 'lossless', one_dut_only=True)) + if 'enum_dut_lossless_prio_with_completeness_level' in metafunc.fixturenames: + metafunc.parametrize("enum_dut_lossless_prio_with_completeness_level", + generate_priority_lists(metafunc, 'lossless', with_completeness_level=True)) + if 'enum_one_dut_lossless_prio_with_completeness_level' in metafunc.fixturenames: + metafunc.parametrize("enum_one_dut_lossless_prio_with_completeness_level", + generate_priority_lists(metafunc, 'lossless', with_completeness_level=True, + one_dut_only=True)) + if 'enum_dut_lossy_prio' in metafunc.fixturenames: + metafunc.parametrize("enum_dut_lossy_prio", generate_priority_lists(metafunc, 'lossy')) + if 'enum_one_dut_lossy_prio' in metafunc.fixturenames: + metafunc.parametrize("enum_one_dut_lossy_prio", + generate_priority_lists(metafunc, 'lossy', one_dut_only=True)) + if 'enum_dut_lossy_prio_with_completeness_level' in metafunc.fixturenames: + metafunc.parametrize("enum_dut_lossy_prio_with_completeness_level", + generate_priority_lists(metafunc, 'lossy', with_completeness_level=True)) + if 'enum_one_dut_lossy_prio_with_completeness_level' in metafunc.fixturenames: + metafunc.parametrize("enum_one_dut_lossy_prio_with_completeness_level", + generate_priority_lists(metafunc, 'lossy', with_completeness_level=True, + one_dut_only=True)) + if 'enum_pfc_pause_delay_test_params' in metafunc.fixturenames: + metafunc.parametrize("enum_pfc_pause_delay_test_params", pfc_pause_delay_test_params(metafunc)) + + if 'topo_scenario' in metafunc.fixturenames: + if tbinfo['topo']['type'] == 'm0' and 'topo_scenario' in metafunc.fixturenames: + metafunc.parametrize('topo_scenario', ['m0_vlan_scenario', 'm0_l3_scenario'], scope='module') + else: + metafunc.parametrize('topo_scenario', ['default'], scope='module') + + if 'tgen_port_info' in metafunc.fixturenames: + metafunc.parametrize('tgen_port_info', generate_skeleton_port_info(metafunc), indirect=True) + + if 'vlan_name' in metafunc.fixturenames: + if tbinfo['topo']['type'] == 'm0' and 'topo_scenario' in metafunc.fixturenames: + if tbinfo['topo']['name'] == 'm0-2vlan': + metafunc.parametrize('vlan_name', ['Vlan1000', 'Vlan2000'], scope='module') + else: + metafunc.parametrize('vlan_name', ['Vlan1000'], scope='module') + # Non M0 topo + else: + try: + if tbinfo["topo"]["type"] in ["t0", "mx"]: + default_vlan_config = tbinfo["topo"]["properties"]["topology"][ + "DUT" + ]["vlan_configs"]["default_vlan_config"] + if default_vlan_config == "two_vlan_a": + logger.info("default_vlan_config is two_vlan_a") + vlan_list = list( + tbinfo["topo"]["properties"]["topology"]["DUT"][ + "vlan_configs" + ]["two_vlan_a"].keys() + ) + elif default_vlan_config == "one_vlan_a": + logger.info("default_vlan_config is one_vlan_a") + vlan_list = list( + tbinfo["topo"]["properties"]["topology"]["DUT"][ + "vlan_configs" + ]["one_vlan_a"].keys() + ) + else: + vlan_list = ["Vlan1000"] + logger.info("parametrize vlan_name: {}".format(vlan_list)) + metafunc.parametrize("vlan_name", vlan_list, scope="module") + else: + metafunc.parametrize("vlan_name", ["no_vlan"], scope="module") + except KeyError: + logger.error("topo {} keys are missing in the tbinfo={}".format(tbinfo['topo']['name'], tbinfo)) + if tbinfo['topo']['type'] in ['t0', 'mx']: + metafunc.parametrize('vlan_name', ['Vlan1000'], scope='module') + else: + metafunc.parametrize('vlan_name', ['no_vlan'], scope='module') + + +@lru_cache +def parse_override(testbed, field): + is_dynamic_only = "--enable-snappi-dynamic-ports" in sys.argv + + if is_dynamic_only and field != "pfcQueueGroupSize": + # Args "--enable-snappi-dynamic-ports" should not affect field `pfcQueueGroupSize` + return False, None + + override_file = "snappi_tests/variables.override.yml" + + with open(override_file, 'r') as f: + all_values = yaml.safe_load(f) + if testbed not in all_values or field not in all_values[testbed]: + return False, None + + return True, all_values[testbed][field] + + return False, None + + +def generate_skeleton_port_info(request): + """ + Return minimal port_info parameters to populate later in the format of -. i.e + + ["400.0-single_linecard_single_asic", "400.0-multiple_linecard_multiple_asic",...] + """ + is_override, override_data = parse_override( + request.config.getoption("--testbed"), + 'multidut_port_info' + ) + + if is_override: + return override_data + + dut_info = get_snappi_testbed_metadata(request) or [] + available_interfaces = {} + matrix = {} + for index, linecard in enumerate(dut_info): + interface_to_asic = {} + for asic in dut_info[linecard]["asic_to_interface"]: + for interface in dut_info[linecard]["asic_to_interface"][asic]: + interface_to_asic[interface] = asic + + available_interfaces[linecard] = [dut_info[linecard]['intf_status'][interface] + for interface in dut_info[linecard]['intf_status'] + if dut_info[linecard]['intf_status'][interface]["admin_state"] == "up"] + + for interface in available_interfaces[linecard]: + for key, value in dut_info[linecard]["asic_to_interface"].items(): + if interface['name'] in value: + interface['asic'] = key + + for interface in available_interfaces[linecard]: + speed = float(re.match(r"([\d.]+)", interface['speed']).group(0)) + asic = interface['asic'] + if (speed not in matrix): + matrix[speed] = {} + if (linecard not in matrix[speed]): + matrix[speed][linecard] = {} + if (asic not in matrix[speed][linecard]): + matrix[speed][linecard][asic] = 1 + else: + matrix[speed][linecard][asic] += 1 + + def build_params(speed, category): + return f"{speed}-{category}" + + flattened_list = set() + + for speed, linecards in matrix.items(): + if len(linecards) >= 2: + flattened_list.add(build_params(speed, 'multiple_linecard_multiple_asic')) + + for linecard, asic_list in linecards.items(): + if len(asic_list) >= 2: + flattened_list.add(build_params(speed, 'single_linecard_multiple_asic')) + + for asics, port_count in asic_list.items(): + if int(port_count) >= 2: + flattened_list.add(build_params(speed, 'single_linecard_single_asic')) + + return list(flattened_list) + + +def get_autoneg_tests_data(): + folder = 'metadata' + filepath = os.path.join(folder, 'autoneg-test-params.json') + if not os.path.exists(filepath): + logger.warning('Autoneg tests datafile is missing: {}. " \ + "Run test_pretest -k test_update_testbed_metadata to create it'.format(filepath)) + return [{'dutname': 'unknown', 'port': 'unknown', 'speeds': ['unknown']}] + data = {} + with open(filepath) as yf: + data = json.load(yf) + + return [ + {'dutname': dutname, 'port': dutport, 'speeds': portinfo['common_port_speeds']} + for dutname, ports in list(data.items()) + for dutport, portinfo in list(ports.items()) + ] + + +def parametrise_per_supported_port_speed(data): + return [ + {'dutname': conn_info['dutname'], 'port': conn_info['port'], 'speed': speed} + for conn_info in data for speed in conn_info['speeds'] + ] + + +# Override enum fixtures for duts and asics to ensure that parametrization happens once per module. +@pytest.fixture(scope="module") +def enum_dut_hostname(request): + return request.param + + +@pytest.fixture(scope="module") +def enum_supervisor_dut_hostname(request): + return request.param + + +@pytest.fixture(scope="module") +def enum_frontend_dut_hostname(request): + return request.param + + +@pytest.fixture(scope="module") +def selected_dut(request): + try: + logger.debug("selected_dut host: {}".format(request.param)) + return request.param + except AttributeError: + return None + + +@pytest.fixture(scope="module") +def enum_rand_one_per_hwsku_hostname(request): + return request.param + + +@pytest.fixture(scope="module") +def enum_rand_one_per_hwsku_frontend_hostname(request): + return request.param + + +@pytest.fixture(scope="module") +def enum_rand_one_per_hwsku_macsec_frontend_hostname(request): + return request.param + + +@pytest.fixture(scope="module") +def enum_asic_index(request): + return request.param + + +@pytest.fixture(scope="module") +def enum_frontend_asic_index(request): + return request.param + + +@pytest.fixture(scope="module") +def enum_backend_asic_index(request): + return request.param + + +@pytest.fixture(scope="module") +def enum_rand_one_asic_index(request): + return request.param + + +@pytest.fixture(scope="module") +def enum_dut_feature(request): + return request.param + + +@pytest.fixture(scope="module") +def enum_rand_one_frontend_asic_index(request): + return request.param + + +@pytest.fixture(scope='module') +def enum_upstream_dut_hostname(duthosts, tbinfo): + upstream_nbr_type = get_upstream_neigh_type(tbinfo, is_upper=True) + if upstream_nbr_type is None: + upstream_nbr_type = "T3" + + for a_dut in duthosts.frontend_nodes: + minigraph_facts = a_dut.get_extended_minigraph_facts(tbinfo) + minigraph_neighbors = minigraph_facts['minigraph_neighbors'] + for key, value in minigraph_neighbors.items(): + if upstream_nbr_type in value['name']: + return a_dut.hostname + + pytest.fail("Did not find a dut in duthosts that for topo type {} that has upstream nbr type {}". + format(tbinfo["topo"]["type"], upstream_nbr_type)) + + +@pytest.fixture(scope="module") +def duthost_console(duthosts, enum_supervisor_dut_hostname, localhost, conn_graph_facts, creds): # noqa: F811 + duthost = duthosts[enum_supervisor_dut_hostname] + host = create_duthost_console(duthost, localhost, conn_graph_facts, creds) + + yield host + host.disconnect() + + +@pytest.fixture(scope='session') +def cleanup_cache_for_session(request): + """ + This fixture allows developers to cleanup the cached data for all DUTs in the testbed before test. + Use cases: + - Running tests where some 'facts' about the DUT that get cached are changed. + - Running tests/regression without running test_pretest which has a test to clean up cache (PR#2978) + - Test case development phase to work out testbed information changes. + + This fixture is not automatically applied, if you want to use it, you have to add a call to it in your tests. + """ + tbname, tbinfo = get_tbinfo(request) + inv_files = get_inventory_files(request) + cache.cleanup(zone=tbname) + for a_dut in tbinfo['duts']: + cache.cleanup(zone=a_dut) + inv_data = get_host_visible_vars(inv_files, a_dut) + if 'num_asics' in inv_data and inv_data['num_asics'] > 1: + for asic_id in range(0, inv_data['num_asics']): + cache.cleanup(zone="{}-asic{}".format(a_dut, asic_id)) + + +def get_l2_info(dut): + """ + Helper function for l2 mode fixture + """ + config_facts = dut.get_running_config_facts() + mgmt_intf_table = config_facts['MGMT_INTERFACE'] + metadata_table = config_facts['DEVICE_METADATA']['localhost'] + mgmt_ip = None + for ip in list(mgmt_intf_table['eth0'].keys()): + if type(ip_interface(ip)) is IPv4Interface: + mgmt_ip = ip + mgmt_gw = mgmt_intf_table['eth0'][mgmt_ip]['gwaddr'] + hwsku = metadata_table['hwsku'] + + return mgmt_ip, mgmt_gw, hwsku + + +@pytest.fixture(scope='session') +def enable_l2_mode(duthosts, tbinfo, backup_and_restore_config_db_session): # noqa: F811 + """ + Configures L2 switch mode according to + https://github.com/sonic-net/SONiC/wiki/L2-Switch-mode + + Currently not compatible with version 201811 + + This fixture does not auto-cleanup after itself + A manual config reload is required to restore regular state + """ + base_config_db_cmd = 'echo \'{}\' | config reload /dev/stdin -y' + l2_preset_cmd = 'sonic-cfggen --preset l2 -p -H -k {} -a \'{}\' | config load /dev/stdin -y' + is_dualtor = 'dualtor' in tbinfo['topo']['name'] + + for dut in duthosts: + logger.info("Setting L2 mode on {}".format(dut)) + cmds = [] + mgmt_ip, mgmt_gw, hwsku = get_l2_info(dut) + # step 1 + base_config_db = { + "MGMT_INTERFACE": { + "eth0|{}".format(mgmt_ip): { + "gwaddr": "{}".format(mgmt_gw) + } + }, + "DEVICE_METADATA": { + "localhost": { + "hostname": "sonic" + } + } + } + + if is_dualtor: + base_config_db["DEVICE_METADATA"]["localhost"]["subtype"] = "DualToR" + cmds.append(base_config_db_cmd.format(json.dumps(base_config_db))) + + # step 2 + cmds.append('sonic-cfggen -H --write-to-db') + + # step 3 is optional and skipped here + # step 4 + if is_dualtor: + mg_facts = dut.get_extended_minigraph_facts(tbinfo) + all_ports = list(mg_facts['minigraph_ports'].keys()) + downlinks = [] + for vlan_info in list(mg_facts['minigraph_vlans'].values()): + downlinks.extend(vlan_info['members']) + uplinks = [intf for intf in all_ports if intf not in downlinks] + extra_args = { + 'is_dualtor': 'true', + 'uplinks': uplinks, + 'downlinks': downlinks + } + else: + extra_args = {} + cmds.append(l2_preset_cmd.format(hwsku, json.dumps(extra_args))) + + # extra step needed to render the feature table correctly + if is_dualtor: + cmds.append('while [ $(show feature config mux | awk \'{print $2}\' | tail -n 1) != "enabled" ]; ' + 'do sleep 1; done') + + # step 5 + cmds.append('config save -y') + + # step 6 + cmds.append('config reload -y') + + logger.debug("Commands to be run:\n{}".format(cmds)) + + dut.shell_cmds(cmds=cmds) + + +@pytest.fixture(scope='session') +def duts_running_config_facts(duthosts): + """Return running config facts for all multi-ASIC DUT hosts + + Args: + duthosts (DutHosts): Instance of DutHosts for interacting with DUT hosts. + + Returns: + dict: { + : [ + (asic0_idx, {asic0_cfg_facts}), + (asic1_idx, {asic1_cfg_facts}) + ] + } + """ + cfg_facts = {} + for duthost in duthosts: + cfg_facts[duthost.hostname] = [] + for asic in duthost.asics: + if asic.is_it_backend(): + continue + asic_cfg_facts = asic.config_facts(source='running')['ansible_facts'] + cfg_facts[duthost.hostname].append((asic.asic_index, asic_cfg_facts)) + return cfg_facts + + +@pytest.fixture(scope='class') +def dut_test_params_qos(duthosts, tbinfo, ptfhost, get_src_dst_asic_and_duts, lower_tor_host, creds, + mux_server_url, mux_status_from_nic_simulator, duts_running_config_facts, duts_minigraph_facts): + if 'dualtor' in tbinfo['topo']['name']: + all_duts = [lower_tor_host] + else: + all_duts = get_src_dst_asic_and_duts['all_duts'] + + src_asic = get_src_dst_asic_and_duts['src_asic'] + dst_asic = get_src_dst_asic_and_duts['dst_asic'] + + src_dut = get_src_dst_asic_and_duts['src_dut'] + src_dut_ip = src_dut.host.options['inventory_manager'].get_host(src_dut.hostname).vars['ansible_host'] + src_server = "{}:{}".format(src_dut_ip, src_asic.get_rpc_port_ssh_tunnel()) + + duthost = all_duts[0] + mgFacts = duthost.get_extended_minigraph_facts(tbinfo) + topo = tbinfo["topo"]["name"] + + rtn_dict = { + "topo": topo, + "hwsku": mgFacts["minigraph_hwsku"], + "basicParams": { + "router_mac": duthost.facts["router_mac"], + "src_server": src_server, + "port_map_file": ptf_test_port_map_active_active( + ptfhost, tbinfo, duthosts, mux_server_url, + duts_running_config_facts, duts_minigraph_facts, + mux_status_from_nic_simulator()), + "sonic_asic_type": duthost.facts['asic_type'], + "sonic_version": duthost.os_version, + "src_dut_index": get_src_dst_asic_and_duts['src_dut_index'], + "src_asic_index": get_src_dst_asic_and_duts['src_asic_index'], + "dst_dut_index": get_src_dst_asic_and_duts['dst_dut_index'], + "dst_asic_index": get_src_dst_asic_and_duts['dst_asic_index'], + "dut_username": creds['sonicadmin_user'], + "dut_password": creds['sonicadmin_password'] + }, + + } + + # Add dst server info if src and dst asic are different + if src_asic != dst_asic: + dst_dut = get_src_dst_asic_and_duts['dst_dut'] + dst_dut_ip = dst_dut.host.options['inventory_manager'].get_host(dst_dut.hostname).vars['ansible_host'] + rtn_dict["basicParams"]["dst_server"] = "{}:{}".format(dst_dut_ip, dst_asic.get_rpc_port_ssh_tunnel()) + + if 'platform_asic' in duthost.facts: + rtn_dict['basicParams']["platform_asic"] = duthost.facts['platform_asic'] + + yield rtn_dict + + +@pytest.fixture(scope='class') +def dut_test_params(duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinfo, + ptf_portmap_file, lower_tor_host, creds): # noqa: F811 + """ + Prepares DUT host test params + + Args: + duthost (AnsibleHost): Device Under Test (DUT) + tbinfo (Fixture, dict): Map containing testbed information + ptfPortMapFile (Fxiture, str): filename residing + on PTF host and contains port maps information + + Returns: + dut_test_params (dict): DUT host test params + """ + if 'dualtor' in tbinfo['topo']['name']: + duthost = lower_tor_host + else: + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + mgFacts = duthost.get_extended_minigraph_facts(tbinfo) + topo = tbinfo["topo"]["name"] + + rtn_dict = { + "topo": topo, + "hwsku": mgFacts["minigraph_hwsku"], + "basicParams": { + "router_mac": duthost.facts["router_mac"], + "server": duthost.host.options['inventory_manager'].get_host( + duthost.hostname + ).vars['ansible_host'], + "port_map_file": ptf_portmap_file, + "sonic_asic_type": duthost.facts['asic_type'], + "sonic_version": duthost.os_version, + "dut_username": creds['sonicadmin_user'], + "dut_password": creds['sonicadmin_password'] + } + } + if 'platform_asic' in duthost.facts: + rtn_dict['basicParams']["platform_asic"] = duthost.facts['platform_asic'] + + yield rtn_dict + + +@pytest.fixture(scope='module') +def duts_minigraph_facts(duthosts, tbinfo): + """Return minigraph facts for all DUT hosts + + Args: + duthosts (DutHosts): Instance of DutHosts for interacting with DUT hosts. + tbinfo (object): Instance of TestbedInfo. + + Returns: + dict: { + : [ + (asic0_idx, {asic0_mg_facts}), + (asic1_idx, {asic1_mg_facts}) + ] + } + """ + mg_facts = {} + for duthost in duthosts: + mg_facts[duthost.hostname] = [] + for asic in duthost.asics: + if asic.is_it_backend(): + continue + asic_mg_facts = asic.get_extended_minigraph_facts(tbinfo) + mg_facts[duthost.hostname].append((asic.asic_index, asic_mg_facts)) + + return mg_facts + + +@pytest.fixture(scope="module", autouse=True) +def get_reboot_cause(duthost): + uptime_start = duthost.get_up_time() + yield + uptime_end = duthost.get_up_time() + if not uptime_end == uptime_start: + if "201811" in duthost.os_version or "201911" in duthost.os_version: + duthost.show_and_parse("show reboot-cause") + else: + duthost.show_and_parse("show reboot-cause history") + + +def collect_db_dump_on_duts(request, duthosts): + '''When test failed, this fixture will dump all the DBs on DUT and collect them to local + ''' + if hasattr(request.node, 'rep_call') and request.node.rep_call.failed: + dut_file_path = "/tmp/db_dump" + local_file_path = "./logs/db_dump" + + # Remove characters that can't be used in filename + nodename = safe_filename(request.node.nodeid) + db_dump_path = os.path.join(dut_file_path, nodename) + db_dump_tarfile = os.path.join(dut_file_path, "{}.tar.gz".format(nodename)) + + # We don't need to collect all DBs, db_names specify the DBs we want to collect + db_names = ["APPL_DB", "ASIC_DB", "COUNTERS_DB", "CONFIG_DB", "STATE_DB"] + raw_db_config = duthosts[0].shell("cat /var/run/redis/sonic-db/database_config.json")["stdout"] + db_config = json.loads(raw_db_config).get("DATABASES", {}) + db_ids = set() + for db_name in db_names: + # Skip STATE_DB dump on release 201911. + # JINJA2_CACHE can't be dumped by "redis-dump", and it is stored in STATE_DB on 201911 release. + # Please refer to issue: https://github.com/sonic-net/sonic-buildimage/issues/5587. + # The issue has been fixed in https://github.com/sonic-net/sonic-buildimage/pull/5646. + # However, the fix is not included in 201911 release. So we have to skip STATE_DB on release 201911 + # to avoid raising exception when dumping the STATE_DB. + if db_name == "STATE_DB" and duthosts[0].sonic_release in ['201911']: + continue + + if db_name in db_config: + db_ids.add(db_config[db_name].get("id", 0)) + + namespace_list = duthosts[0].get_asic_namespace_list() if duthosts[0].is_multi_asic else [] + if namespace_list: + for namespace in namespace_list: + # Collect DB dump + dump_dest_path = os.path.join(db_dump_path, namespace) + dump_cmds = ["mkdir -p {}".format(dump_dest_path)] + for db_id in db_ids: + dump_cmd = "ip netns exec {} redis-dump -d {} -y -o {}/{}" \ + .format(namespace, db_id, dump_dest_path, db_id) + dump_cmds.append(dump_cmd) + duthosts.shell_cmds(cmds=dump_cmds) + else: + # Collect DB dump + dump_dest_path = db_dump_path + dump_cmds = ["mkdir -p {}".format(dump_dest_path)] + for db_id in db_ids: + dump_cmd = "redis-dump -d {} -y -o {}/{}".format(db_id, dump_dest_path, db_id) + dump_cmds.append(dump_cmd) + duthosts.shell_cmds(cmds=dump_cmds) + + # compress dump file and fetch to docker + duthosts.shell("tar -czf {} -C {} {}".format(db_dump_tarfile, dut_file_path, nodename)) + duthosts.fetch(src=db_dump_tarfile, dest=local_file_path) + + # remove dump file from dut + duthosts.shell("rm -fr {} {}".format(db_dump_tarfile, db_dump_path)) + + +@pytest.fixture(autouse=True) +def collect_db_dump(request, duthosts): + """This autoused fixture is to generate DB dumps on DUT and collect them to local for later troubleshooting when + a test case failed. + """ + yield + if request.config.getoption("--collect_db_data"): + collect_db_dump_on_duts(request, duthosts) + + +def restore_config_db_and_config_reload(duts_data, duthosts, request): + # First copy the pre_running_config to the config_db.json files + for duthost in duthosts: + logger.info("dut reload called on {}".format(duthost.hostname)) + duthost.copy(content=json.dumps(duts_data[duthost.hostname]["pre_running_config"][None], indent=4), + dest='/etc/sonic/config_db.json', verbose=False) + + if duthost.is_multi_asic: + for asic_index in range(0, duthost.facts.get('num_asic')): + asic_ns = "asic{}".format(asic_index) + asic_cfg_file = "/tmp/{}_config_db{}.json".format(duthost.hostname, asic_index) + with open(asic_cfg_file, "w") as outfile: + outfile.write(json.dumps(duts_data[duthost.hostname]['pre_running_config'][asic_ns], indent=4)) + duthost.copy(src=asic_cfg_file, dest='/etc/sonic/config_db{}.json'.format(asic_index), verbose=False) + os.remove(asic_cfg_file) + + wait_for_bgp = False if request.config.getoption("skip_sanity") else True + + # Second execute config reload on all duthosts + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost in duthosts: + executor.submit(config_reload, duthost, wait_before_force_reload=300, safe_reload=True, + check_intf_up_ports=True, wait_for_bgp=wait_for_bgp) + + +def compare_running_config(pre_running_config, cur_running_config): + if type(pre_running_config) != type(cur_running_config): + return False + if pre_running_config == cur_running_config: + return True + else: + if type(pre_running_config) is dict: + if set(pre_running_config.keys()) != set(cur_running_config.keys()): + return False + for key in pre_running_config.keys(): + if not compare_running_config(pre_running_config[key], cur_running_config[key]): + return False + return True + # We only have string in list in running config now, so we can ignore the order of the list. + elif type(pre_running_config) is list: + if set(pre_running_config) != set(cur_running_config): + return False + else: + return True + else: + return False + + +@pytest.fixture(scope="module", autouse=True) +def core_dump_and_config_check(duthosts, tbinfo, parallel_run_context, request, + # make sure the tear down of sanity_check happened after core_dump_and_config_check + sanity_check): + ''' + Check if there are new core dump files and if the running config is modified after the test case running. + If so, we will reload the running config after test case running. + ''' + + par_ctx = parallel_run_context + parallel_coordinator = ParallelCoordinator(par_ctx) if par_ctx.is_par_run else None + if par_ctx.is_par_run and not par_ctx.is_par_leader: + logger.info( + "Fixture core_dump_and_config_check setup for non-leader nodes in parallel run is skipped. " + "Please refer to the leader node log for core dump and config check status." + ) + + parallel_coordinator.wait_and_ack_status_for_followers( + ParallelStatus.SETUP_COMPLETED, + par_ctx.is_par_leader, + par_ctx.target_hostname, + ) + + parallel_coordinator.wait_for_all_followers_ack(ParallelStatus.SETUP_COMPLETED) + + yield {} + + parallel_coordinator.mark_and_wait_for_status( + ParallelStatus.TESTS_COMPLETED, + par_ctx.target_hostname, + par_ctx.is_par_leader, + ) + + logger.info( + "Fixture core_dump_and_config_check teardown for non-leader nodes in parallel run is skipped. " + "Please refer to the leader node log for core dump and config check status." + ) + else: + check_flag = True + if hasattr(request.config.option, 'enable_macsec') and request.config.option.enable_macsec: + check_flag = False + if hasattr(request.config.option, 'markexpr') and request.config.option.markexpr: + if "bsl" in request.config.option.markexpr: + check_flag = False + for m in request.node.iter_markers(): + if m.name == "skip_check_dut_health": + check_flag = False + + module_name = request.node.name + + duts_data = {} + + if check_flag: + + def collect_before_test(dut): + logger.info("Dumping Disk and Memory Space information before test on {}".format(dut.hostname)) + dut.shell("free -h") + dut.shell("df -h") + + logger.info("Collecting core dumps before test on {}".format(dut.hostname)) + duts_data[dut.hostname] = {} + + if "20191130" in dut.os_version: + pre_existing_core_dumps = dut.shell('ls /var/core/ | grep -v python || true')['stdout'].split() + else: + pre_existing_core_dumps = dut.shell('ls /var/core/')['stdout'].split() + duts_data[dut.hostname]["pre_core_dumps"] = pre_existing_core_dumps + + logger.info("Collecting running config before test on {}".format(dut.hostname)) + duts_data[dut.hostname]["pre_running_config"] = {} + if not dut.stat(path="/etc/sonic/running_golden_config.json")['stat']['exists']: + logger.info("Collecting running golden config before test on {}".format(dut.hostname)) + dut.shell("sonic-cfggen -d --print-data > /etc/sonic/running_golden_config.json") + duts_data[dut.hostname]["pre_running_config"][None] = \ + json.loads(dut.shell("cat /etc/sonic/running_golden_config.json", verbose=False)['stdout']) + + if dut.is_multi_asic: + for asic_index in range(0, dut.facts.get('num_asic')): + asic_ns = "asic{}".format(asic_index) + if not dut.stat( + path="/etc/sonic/running_golden_config{}.json".format(asic_index))['stat']['exists']: + dut.shell( + "sonic-cfggen -n {} -d --print-data > /etc/sonic/running_golden_config{}.json".format( + asic_ns, + asic_index, + ) + ) + duts_data[dut.hostname]['pre_running_config'][asic_ns] = \ + json.loads(dut.shell("cat /etc/sonic/running_golden_config{}.json".format(asic_index), + verbose=False)['stdout']) + + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost in duthosts: + executor.submit(collect_before_test, duthost) + + if par_ctx.is_par_run and par_ctx.is_par_leader: + parallel_coordinator.set_new_status( + ParallelStatus.SETUP_COMPLETED, + par_ctx.is_par_leader, + par_ctx.target_hostname, + ) + + parallel_coordinator.wait_for_all_followers_ack(ParallelStatus.SETUP_COMPLETED) + + yield duts_data + + if par_ctx.is_par_run and par_ctx.is_par_leader: + parallel_coordinator.mark_and_wait_for_status( + ParallelStatus.TESTS_COMPLETED, + par_ctx.target_hostname, + par_ctx.is_par_leader, + ) + + parallel_coordinator.set_new_status( + ParallelStatus.TEARDOWN_STARTED, + par_ctx.is_par_leader, + par_ctx.target_hostname, + ) + + inconsistent_config = {} + pre_only_config = {} + cur_only_config = {} + new_core_dumps = {} + + core_dump_check_failed = False + config_db_check_failed = False + + check_result = {} + + if check_flag: + + def collect_after_test(dut): + inconsistent_config[dut.hostname] = {} + pre_only_config[dut.hostname] = {} + cur_only_config[dut.hostname] = {} + new_core_dumps[dut.hostname] = [] + + logger.info("Dumping Disk and Memory Space information after test on {}".format(dut.hostname)) + dut.shell("free -h") + dut.shell("df -h") + + logger.info("Collecting core dumps after test on {}".format(dut.hostname)) + if "20191130" in dut.os_version: + cur_cores = dut.shell('ls /var/core/ | grep -v python || true')['stdout'].split() + else: + cur_cores = dut.shell('ls /var/core/')['stdout'].split() + duts_data[dut.hostname]["cur_core_dumps"] = cur_cores + + cur_core_dumps_set = set(duts_data[dut.hostname]["cur_core_dumps"]) + pre_core_dumps_set = set(duts_data[dut.hostname]["pre_core_dumps"]) + new_core_dumps[dut.hostname] = list(cur_core_dumps_set - pre_core_dumps_set) + + logger.info("Collecting running config after test on {}".format(dut.hostname)) + # get running config after running + duts_data[dut.hostname]["cur_running_config"] = {} + duts_data[dut.hostname]["cur_running_config"][None] = \ + json.loads(dut.shell("sonic-cfggen -d --print-data", verbose=False)['stdout']) + if dut.is_multi_asic: + for asic_index in range(0, dut.facts.get('num_asic')): + asic_ns = "asic{}".format(asic_index) + duts_data[dut.hostname]["cur_running_config"][asic_ns] = \ + json.loads(dut.shell("sonic-cfggen -n {} -d --print-data".format(asic_ns), + verbose=False)['stdout']) + + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost in duthosts: + executor.submit(collect_after_test, duthost) + + for duthost in duthosts: + if new_core_dumps[duthost.hostname]: + core_dump_check_failed = True + + base_dir = os.path.dirname(os.path.realpath(__file__)) + for new_core_dump in new_core_dumps[duthost.hostname]: + duthost.fetch(src="/var/core/{}".format(new_core_dump), dest=os.path.join(base_dir, "logs")) + + # The tables that we don't care + exclude_config_table_names = set([]) + # The keys that we don't care + # Current skipped keys: + # 1. "MUX_LINKMGR|LINK_PROBER" + # 2. "MUX_LINKMGR|TIMED_OSCILLATION" + # 3. "LOGGER|linkmgrd" + # NOTE: this key is edited by the `run_icmp_responder_session` or `run_icmp_responder` + # to account for the lower performance of the ICMP responder/mux simulator compared to + # real servers and mux cables. + # Linkmgrd is the only service to consume this table so it should not affect other test cases. + # Let's keep this setting in db and we don't want any config reload caused by this key, so + # let's skip checking it. + if "dualtor" in tbinfo["topo"]["name"]: + exclude_config_key_names = [ + 'MUX_LINKMGR|LINK_PROBER', + 'MUX_LINKMGR|TIMED_OSCILLATION', + 'LOGGER|linkmgrd' + ] + else: + exclude_config_key_names = [] + + def _remove_entry(table_name, key_name, config): + if table_name in config and key_name in config[table_name]: + config[table_name].pop(key_name) + if len(config[table_name]) == 0: + config.pop(table_name) + + for cfg_context in duts_data[duthost.hostname]['pre_running_config']: + pre_only_config[duthost.hostname][cfg_context] = {} + cur_only_config[duthost.hostname][cfg_context] = {} + inconsistent_config[duthost.hostname][cfg_context] = {} + + pre_running_config = duts_data[duthost.hostname]["pre_running_config"][cfg_context] + cur_running_config = duts_data[duthost.hostname]["cur_running_config"][cfg_context] + + # Remove ignored keys from base config + for exclude_key in exclude_config_key_names: + fields = exclude_key.split('|') + if len(fields) != 2: + continue + _remove_entry(fields[0], fields[1], pre_running_config) + _remove_entry(fields[0], fields[1], cur_running_config) + + pre_running_config_keys = set(pre_running_config.keys()) + cur_running_config_keys = set(cur_running_config.keys()) + + # Check if there are extra keys in pre running config + pre_config_extra_keys = list( + pre_running_config_keys - cur_running_config_keys - exclude_config_table_names) + for key in pre_config_extra_keys: + pre_only_config[duthost.hostname][cfg_context].update({key: pre_running_config[key]}) + + # Check if there are extra keys in cur running config + cur_config_extra_keys = list( + cur_running_config_keys - pre_running_config_keys - exclude_config_table_names) + for key in cur_config_extra_keys: + cur_only_config[duthost.hostname][cfg_context].update({key: cur_running_config[key]}) + + # Get common keys in pre running config and cur running config + common_config_keys = list(pre_running_config_keys & cur_running_config_keys - + exclude_config_table_names) + + # Check if the running config is modified after module running + for key in common_config_keys: + # TODO: remove these code when solve the problem of "FLEX_COUNTER_DELAY_STATUS" + if key == "FLEX_COUNTER_TABLE": + for sub_key, sub_value in list(pre_running_config[key].items()): + try: + pre_value = pre_running_config[key][sub_key] + cur_value = cur_running_config[key][sub_key] + if pre_value["FLEX_COUNTER_STATUS"] != cur_value["FLEX_COUNTER_STATUS"]: + inconsistent_config[duthost.hostname][cfg_context].update( + { + key: { + "pre_value": pre_running_config[key], + "cur_value": cur_running_config[key] + } + } + ) + except KeyError: + inconsistent_config[duthost.hostname][cfg_context].update( + { + key: { + "pre_value": pre_running_config[key], + "cur_value": cur_running_config[key] + } + } + ) + elif not compare_running_config(pre_running_config[key], cur_running_config[key]): + inconsistent_config[duthost.hostname][cfg_context].update( + { + key: { + "pre_value": pre_running_config[key], + "cur_value": cur_running_config[key] + } + } + ) + + if pre_only_config[duthost.hostname][cfg_context] or \ + cur_only_config[duthost.hostname][cfg_context] or \ + inconsistent_config[duthost.hostname][cfg_context]: + config_db_check_failed = True + + if core_dump_check_failed or config_db_check_failed: + check_result = { + "core_dump_check": { + "failed": core_dump_check_failed, + "new_core_dumps": new_core_dumps + }, + "config_db_check": { + "failed": config_db_check_failed, + "pre_only_config": pre_only_config, + "cur_only_config": cur_only_config, + "inconsistent_config": inconsistent_config + } + } + logger.warning("Core dump or config check failed for {}, results: {}" + .format(module_name, json.dumps(check_result))) + + restore_config_db_and_config_reload(duts_data, duthosts, request) + else: + logger.info("Core dump and config check passed for {}".format(module_name)) + + if check_result: + logger.debug("core_dump_and_config_check failed, check_result: {}".format(json.dumps(check_result))) + add_custom_msg(request, f"{DUT_CHECK_NAMESPACE}.core_dump_check_failed", core_dump_check_failed) + add_custom_msg(request, f"{DUT_CHECK_NAMESPACE}.config_db_check_failed", config_db_check_failed) + + +@pytest.fixture(scope="module", autouse=True) +def temporarily_disable_route_check(request, duthosts): + check_flag = False + for m in request.node.iter_markers(): + if m.name == "disable_route_check": + check_flag = True + break + + def wait_for_route_check_to_pass(dut): + + def run_route_check(): + res = dut.shell("sudo route_check.py", module_ignore_errors=True) + return res["rc"] == 0 + + pt_assert( + wait_until(180, 15, 0, run_route_check), + "route_check.py is still failing after timeout", + ) + + if check_flag: + # If a pytest.fail or any other exceptions are raised in the setup stage of a fixture (before the yield), + # the teardown code (after the yield) will not run, so we are using try...finally... to ensure the + # routeCheck monit will always be started after this fixture. + try: + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost in duthosts.frontend_nodes: + executor.submit(wait_for_route_check_to_pass, duthost) + + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost in duthosts.frontend_nodes: + executor.submit(stop_route_checker_on_duthost, duthost) + + yield + + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost in duthosts.frontend_nodes: + executor.submit(wait_for_route_check_to_pass, duthost) + finally: + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost in duthosts.frontend_nodes: + executor.submit(start_route_checker_on_duthost, duthost) + else: + logger.info("Skipping temporarily_disable_route_check fixture") + yield + logger.info("Skipping temporarily_disable_route_check fixture") + + +@pytest.fixture(scope="function") +def on_exit(): + ''' + Utility to register callbacks for cleanup. Runs callbacks despite assertion + failures. Callbacks are executed in reverse order of registration. + ''' + class OnExit(): + def __init__(self): + self.cbs = [] + + def register(self, fn): + self.cbs.append(fn) + + def cleanup(self): + while len(self.cbs) != 0: + self.cbs.pop()() + + on_exit = OnExit() + yield on_exit + on_exit.cleanup() + + +@pytest.fixture(scope="session", autouse=True) +def add_mgmt_test_mark(duthosts): + ''' + @summary: Create mark file at /etc/sonic/mgmt_test_mark, and DUT can use this mark to detect mgmt test. + @param duthosts: fixture to get DUT hosts + ''' + mark_file = "/etc/sonic/mgmt_test_mark" + duthosts.shell("touch %s" % mark_file, module_ignore_errors=True) + + +def verify_packets_any_fixed(test, pkt, ports=[], device_number=0, timeout=None): + """ + Check that a packet is received on _any_ of the specified ports belonging to + the given device (default device_number is 0). + + Also verifies that the packet is not received on any other ports for this + device, and that no other packets are received on the device (unless --relax + is in effect). + + The function is redefined here to workaround code bug in testutils.verify_packets_any + """ + received = False + failures = [] + for device, port in testutils.ptf_ports(): + if device != device_number: + continue + if port in ports: + logging.debug("Checking for pkt on device %d, port %d", device_number, port) + result = testutils.dp_poll(test, device_number=device, port_number=port, + timeout=timeout, exp_pkt=pkt) + if isinstance(result, test.dataplane.PollSuccess): + received = True + else: + failures.append((port, result)) + else: + testutils.verify_no_packet(test, pkt, (device, port)) + testutils.verify_no_other_packets(test) + + if not received: + def format_failure(port, failure): + return "On port %d:\n%s" % (port, failure.format()) + failure_report = "\n".join([format_failure(*f) for f in failures]) + test.fail("Did not receive expected packet on any of ports %r for device %d.\n%s" + % (ports, device_number, failure_report)) + + +# HACK: testutils.verify_packets_any to workaround code bug +# TODO: delete me when ptf version is advanced than https://github.com/p4lang/ptf/pull/139 +testutils.verify_packets_any = verify_packets_any_fixed + +# HACK: We are using set_do_not_care_scapy but it will be deprecated. +if not hasattr(Mask, "set_do_not_care_scapy"): + Mask.set_do_not_care_scapy = Mask.set_do_not_care_packet + + +def run_logrotate(duthost, stop_event): + logger.info("Start rotate_syslog on {}".format(duthost)) + while not stop_event.is_set(): + try: + # Run logrotate for rsyslog + duthost.shell("logrotate -f /etc/logrotate.conf", module_ignore_errors=True) + except subprocess.CalledProcessError as e: + logger.error("Error: {}".format(str(e))) + # Wait for 60 seconds before the next rotation + time.sleep(60) + + +@pytest.fixture(scope="function") +def rotate_syslog(duthosts, enum_rand_one_per_hwsku_frontend_hostname): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + stop_event = threading.Event() + thread = InterruptableThread( + target=run_logrotate, + args=(duthost, stop_event,) + ) + thread.daemon = True + thread.start() + + yield + stop_event.set() + try: + if thread.is_alive(): + thread.join(timeout=30) + logger.info("thread {} joined".format(thread)) + except Exception as e: + logger.debug("Exception occurred in thread {}".format(str(e))) + + logger.info("rotate_syslog exit {}".format(thread)) + + +@pytest.fixture(scope="module") +def gnxi_path(ptfhost): + """ + gnxi's location is updated from /gnxi to /root/gnxi + in RP https://github.com/sonic-net/sonic-buildimage/pull/10599. + But old docker-ptf images don't have this update, + test case will fail for these docker-ptf images, + because it should still call /gnxi files. + For avoiding this conflict, check gnxi path before test and set GNXI_PATH to correct value. + Add a new gnxi_path module fixture to make sure to set GNXI_PATH before test. + """ + path_exists = ptfhost.stat(path="/root/gnxi/") + if path_exists["stat"]["exists"] and path_exists["stat"]["isdir"]: + gnxipath = "/root/gnxi/" + else: + gnxipath = "/gnxi/" + return gnxipath + + +@pytest.fixture(scope="module") +def selected_asic_index(request): + asic_index = DEFAULT_ASIC_ID + if "enum_asic_index" in request.fixturenames: + asic_index = request.getfixturevalue("enum_asic_index") + elif "enum_frontend_asic_index" in request.fixturenames: + asic_index = request.getfixturevalue("enum_frontend_asic_index") + elif "enum_backend_asic_index" in request.fixturenames: + asic_index = request.getfixturevalue("enum_backend_asic_index") + elif "enum_rand_one_asic_index" in request.fixturenames: + asic_index = request.getfixturevalue("enum_rand_one_asic_index") + elif "enum_rand_one_frontend_asic_index" in request.fixturenames: + asic_index = request.getfixturevalue("enum_rand_one_frontend_asic_index") + logger.info(f"Selected asic_index {asic_index}") + return asic_index + + +@pytest.fixture(scope="module") +def ip_netns_namespace_prefix(request, selected_asic_index): + """ + Construct the formatted namespace prefix for executed commands inside the specific + network namespace or for linux commands. + """ + if selected_asic_index == DEFAULT_ASIC_ID: + return '' + else: + return f'sudo ip netns exec {NAMESPACE_PREFIX}{selected_asic_index}' + + +@pytest.fixture(scope="module") +def cli_namespace_prefix(request, selected_asic_index): + """ + Construct the formatted namespace prefix for executed commands inside the specific + network namespace or for CLI commands. + """ + if selected_asic_index == DEFAULT_ASIC_ID: + return '' + else: + return f'-n {NAMESPACE_PREFIX}{selected_asic_index}' + + +def pytest_collection_modifyitems(config, items): + # Skip all stress_tests if --run-stress-test is not set + if not config.getoption("--run-stress-tests"): + skip_stress_tests = pytest.mark.skip(reason="Stress tests run only if --run-stress-tests is passed") + for item in items: + if "stress_test" in item.keywords: + item.add_marker(skip_stress_tests) + + +def update_t1_test_ports(duthost, mg_facts, test_ports, tbinfo): + """ + Find out active IP interfaces and use the list to + remove inactive ports from test_ports + """ + ip_ifaces = duthost.get_active_ip_interfaces(tbinfo, asic_index=0) + port_list = [] + for iface in list(ip_ifaces.keys()): + if iface.startswith("PortChannel"): + port_list.extend( + mg_facts["minigraph_portchannels"][iface]["members"] + ) + else: + port_list.append(iface) + port_list_set = set(port_list) + for port in list(test_ports.keys()): + if port not in port_list_set: + del test_ports[port] + return test_ports + + +@pytest.fixture(scope="module", params=['IPv6', 'IPv4']) +def ip_version(request): + return request.param + + +@pytest.fixture(scope="module") +def setup_pfc_test( + duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, conn_graph_facts, tbinfo, ip_version, # noqa F811 +): + """ + Sets up all the parameters needed for the PFC Watchdog tests + + Args: + duthost: AnsibleHost instance for DUT + ptfhost: AnsibleHost instance for PTF + conn_graph_facts: fixture that contains the parsed topology info + + Yields: + setup_info: dictionary containing pfc timers, generated test ports and selected test ports + """ + SUPPORTED_T1_TOPOS = {"t1-lag", "t1-64-lag", "t1-56-lag", "t1-28-lag", "t1-32-lag"} + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + mg_facts = duthost.get_extended_minigraph_facts(tbinfo) + port_list = list(mg_facts['minigraph_ports'].keys()) + neighbors = conn_graph_facts['device_conn'].get(duthost.hostname, {}) + config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] + dut_eth0_ip = duthost.mgmt_ip + vlan_nw = None + + if mg_facts['minigraph_vlans']: + # Filter VLANs with one interface inside only(PortChannel interface in case of t0-56-po2vlan topo) + unexpected_vlans = [] + for vlan, vlan_data in list(mg_facts['minigraph_vlans'].items()): + if len(vlan_data['members']) < 2: + unexpected_vlans.append(vlan) + + # Update minigraph_vlan_interfaces with only expected VLAN interfaces + expected_vlan_ifaces = [] + for vlan in unexpected_vlans: + for mg_vl_iface in mg_facts['minigraph_vlan_interfaces']: + if vlan != mg_vl_iface['attachto']: + expected_vlan_ifaces.append(mg_vl_iface) + if expected_vlan_ifaces: + mg_facts['minigraph_vlan_interfaces'] = expected_vlan_ifaces + + # gather all vlan specific info + ip_index = 0 if ip_version == "IPv4" else 1 + vlan_addr = mg_facts['minigraph_vlan_interfaces'][ip_index]['addr'] + vlan_prefix = mg_facts['minigraph_vlan_interfaces'][ip_index]['prefixlen'] + vlan_dev = mg_facts['minigraph_vlan_interfaces'][ip_index]['attachto'] + vlan_ips = duthost.get_ip_in_range( + num=1, prefix="{}/{}".format(vlan_addr, vlan_prefix), + exclude_ips=[vlan_addr])['ansible_facts']['generated_ips'] + vlan_nw = vlan_ips[0].split('/')[0] + + topo = tbinfo["topo"]["name"] + # build the port list for the test + config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] + if ip_version == "IPv4": + ip_version_num = 4 + elif ip_version == "IPv6": + ip_version_num = 6 + else: + pytest.fail(f"Invalid IP version: {input}", pytrace=True) + + tp_handle = TrafficPorts(mg_facts, neighbors, vlan_nw, topo, config_facts, ip_version_num) + test_ports = tp_handle.build_port_list() + + # In T1 topology update test ports by removing inactive ports + if topo in SUPPORTED_T1_TOPOS: + test_ports = update_t1_test_ports( + duthost, mg_facts, test_ports, tbinfo + ) + # select a subset of ports from the generated port list + selected_ports = select_test_ports(test_ports) + + setup_info = {'test_ports': test_ports, + 'port_list': port_list, + 'selected_test_ports': selected_ports, + 'pfc_timers': set_pfc_timers(), + 'neighbors': neighbors, + 'eth0_ip': dut_eth0_ip, + 'ip_version': ip_version + } + + if mg_facts['minigraph_vlans']: + setup_info['vlan'] = {'addr': vlan_addr, + 'prefix': vlan_prefix, + 'dev': vlan_dev + } + else: + setup_info['vlan'] = None + + # stop pfcwd + logger.info("--- Stopping Pfcwd ---") + duthost.command("pfcwd stop") + + # set poll interval + duthost.command("pfcwd interval {}".format(setup_info['pfc_timers']['pfc_wd_poll_time'])) + + # set bulk counter chunk size + logger.info("--- Setting bulk counter polling chunk size ---") + duthost.command('redis-cli -n 4 hset "FLEX_COUNTER_TABLE|PORT" BULK_CHUNK_SIZE 64' + ' BULK_CHUNK_SIZE_PER_PREFIX "SAI_PORT_STAT_IF_OUT_QLEN:0;SAI_PORT_STAT_IF_IN_FEC:32"') + + logger.info("setup_info : {}".format(setup_info)) + yield setup_info + + +@pytest.fixture(scope="session") +def setup_gnmi_server(request, localhost, duthost): + """ + SAI validation library uses gNMI to access sonic-db data + objects. This fixture is used by tests to set up gNMI server + """ + disable_sai_validation = request.config.getoption("--disable_sai_validation") + if disable_sai_validation: + logger.info("SAI validation is disabled") + yield duthost, None + return + gnmi_insecure = request.config.getoption("--gnmi_insecure") + if gnmi_insecure: + logger.info("gNMI insecure mode is enabled") + yield duthost, None + return + else: + checkpoint_name = "before-applying-gnmi-certs" + cert_path = pathlib.Path("/tmp/gnmi_certificates") + gnmi_setup.create_certificates(localhost, duthost.mgmt_ip, cert_path) + gnmi_setup.copy_certificates_to_dut(cert_path, duthost) + gnmi_setup.apply_certs(duthost, checkpoint_name) + yield duthost, cert_path + gnmi_setup.remove_certs(duthost, checkpoint_name) + + +@pytest.fixture(scope="session") +def setup_connection(request, setup_gnmi_server): + duthost, cert_path = setup_gnmi_server + disable_sai_validation = request.config.getoption("--disable_sai_validation") + if disable_sai_validation: + logger.info("SAI validation is disabled") + yield None + return + else: + # Dynamically import create_gnmi_stub + gnmi_client_module = importlib.import_module("tests.common.sai_validation.gnmi_client") + create_gnmi_stub = getattr(gnmi_client_module, "create_gnmi_stub") + + # if cert_path is None then it is insecure mode + gnmi_insecure = request.config.getoption("--gnmi_insecure") + gnmi_target_port = int(request.config.getoption("--gnmi_port")) + duthost_mgmt_ip = duthost.mgmt_ip + channel = None + gnmi_connection = None + if gnmi_insecure: + channel, gnmi_connection = create_gnmi_stub(ip=duthost_mgmt_ip, + port=gnmi_target_port, secure=False) + else: + root_cert = str(cert_path / 'gnmiCA.pem') + client_cert = str(cert_path / 'gnmiclient.crt') + client_key = str(cert_path / 'gnmiclient.key') + channel, gnmi_connection = create_gnmi_stub(ip=duthost_mgmt_ip, + port=gnmi_target_port, secure=True, + root_cert_path=root_cert, + client_cert_path=client_cert, + client_key_path=client_key) + yield gnmi_connection + channel.close() + + +@pytest.fixture(scope="module", autouse=True) +def restore_golden_config_db(duthost): + if file_exists_on_dut(duthost, GOLDEN_CONFIG_DB_PATH_ORI): + duthost.shell("cp {} {}".format(GOLDEN_CONFIG_DB_PATH_ORI, GOLDEN_CONFIG_DB_PATH)) + logger.info("[restore_golden_config_db] Restored {}".format(GOLDEN_CONFIG_DB_PATH)) + yield + + +@pytest.fixture(scope="session") +def gnmi_connection(request, setup_connection): + connection = setup_connection + yield connection + + +class DualtorMuxPortSetupConfig(enum.Flag): + """Dualtor mux port setup config.""" + DUALTOR_SKIP_SETUP_MUX_PORTS = enum.auto() + DUALTOR_SETUP_MUX_PORT_MANUAL_MODE = enum.auto() + + # active-standby mux setup configs + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_ENUM_TOR = enum.auto() + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_ENUM_TOR_MANUAL_MODE = \ + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_ENUM_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_UPPER_TOR = enum.auto() + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_UPPER_TOR_MANUAL_MODE = \ + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_UPPER_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_LOWER_TOR = enum.auto() + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_LOWER_TOR_MANUAL_MODE = \ + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_LOWER_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_TOR = enum.auto() + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_TOR_MANUAL_MODE = \ + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_UNSELECTED_TOR = enum.auto() + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_UNSELECTED_TOR_MANUAL_MODE = \ + DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_UNSELECTED_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE + + # active-active mux setup configs + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_ENUM_TOR = enum.auto() + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_ENUM_TOR_MANUAL_MODE = \ + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_ENUM_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_UPPER_TOR = enum.auto() + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_UPPER_TOR_MANUAL_MODE = \ + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_UPPER_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_LOWER_TOR = enum.auto() + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_LOWER_TOR_MANUAL_MODE = \ + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_LOWER_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_TOR = enum.auto() + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_TOR_MANUAL_MODE = \ + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_UNSELECTED_TOR = enum.auto() + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_UNSELECTED_TOR_MANUAL_MODE = \ + DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_UNSELECTED_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE + + +@pytest.fixture(autouse=True) +def setup_dualtor_mux_ports(active_active_ports, duthost, duthosts, tbinfo, request, mux_server_url): # noqa:F811 + """Setup dualtor mux ports.""" + def _get_enumerated_dut_hostname(request): + for k, v in request.node.callspec.params.items(): + if k in ("enum_dut_hostname", + "enum_frontend_dut_hostname", + "enum_supervisor_dut_hostname", + "enum_rand_one_per_hwsku_hostname", + "enum_rand_one_per_hwsku_frontend_hostname"): + return v + return None + + def _get_peer_dut_hostname(local_dut_hostname): + """Get the peer DUT hostname.""" + for dut in duthosts: + if dut.hostname != local_dut_hostname: + return dut.hostname + return None + + def _is_dut_hostname_valid(dut_hostname): + """Check if the dut hostname is valid/present in the tb.""" + return dut_hostname in duthosts.duts + + topo_name = tbinfo["topo"]["name"] + is_dualtor = "dualtor" in topo_name + + if not is_dualtor: + logging.info("skip setup dualtor mux cables on non-dualtor testbed") + yield False + return + + is_dualtor_aa = "dualtor-aa" in topo_name + # read setup configs from pytest markers + dualtor_setup_config = DualtorMuxPortSetupConfig(0) + for marker in request.node.iter_markers(): + try: + dualtor_setup_config |= DualtorMuxPortSetupConfig[marker.name.upper()] + except KeyError: + continue + logging.debug("dualtor mux port setup config: %s", dualtor_setup_config) + + if dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_SKIP_SETUP_MUX_PORTS: + logging.info("skip setup dualtor mux cables") + yield False + return + + is_test_func_parametrized = hasattr(request.node, "callspec") + is_enum = is_test_func_parametrized and \ + any(param.startswith("enum_") for param in request.node.callspec.params.keys()) + rand_one_unselected_dut_hostname = _get_peer_dut_hostname(rand_one_dut_hostname_var) + + if is_dualtor_aa: + # NOTE: Skip setup mux ports if the test explicitly calls + # an active-active mux toggle fixture. + for fixture in request.fixturenames: + if fixture == "config_active_active_dualtor_active_standby": + logging.info("Skip setup dualtor mux cables as toggle " + "fixture %s is explicitly called", + fixture) + yield False + return + + active_dut_hostname = None + standby_dut_hostname = None + + if dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_ENUM_TOR: + if is_test_func_parametrized: + standby_dut_hostname = _get_enumerated_dut_hostname(request) + active_dut_hostname = _get_peer_dut_hostname(standby_dut_hostname) + elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_UPPER_TOR: + standby_dut_hostname = duthosts[0].hostname + active_dut_hostname = duthosts[-1].hostname + elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_LOWER_TOR: + standby_dut_hostname = duthosts[-1].hostname + active_dut_hostname = duthosts[0].hostname + elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_TOR: + standby_dut_hostname = rand_one_dut_hostname_var + active_dut_hostname = rand_one_unselected_dut_hostname + elif dualtor_setup_config & \ + DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_UNSELECTED_TOR: + standby_dut_hostname = rand_one_unselected_dut_hostname + active_dut_hostname = rand_one_dut_hostname_var + else: + # NOTE: If no marker is explicitly specified on active-active dualtor, let's + # leave the ToRs pair in active-active by default. + pass + + if (_is_dut_hostname_valid(active_dut_hostname) and + _is_dut_hostname_valid(standby_dut_hostname)): + logging.info("Setup active-active dualtor, DUT %s as active side, " + "DUT %s as standby side", + active_dut_hostname, standby_dut_hostname) + config_active_active_dualtor( + duthosts[active_dut_hostname], + duthosts[standby_dut_hostname], + active_active_ports, + dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_SETUP_MUX_PORT_MANUAL_MODE + ) + else: + yield False + return + + else: + # NOTE: Skip setup mux ports if the test explicitly calls + # an active-standby mux toggle fixture. + for fixture in request.fixturenames: + if fixture.startswith("toggle_") and fixture in dir(mux_simulator_control): + logging.info("Skip setup dualtor mux cables as toggle " + "fixture %s is explicitly called", + fixture) + yield False + return + + if is_enum and not (dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_ENUM_TOR): + logging.info( + "skip setup dualtor mux cables on test with enum fixture") + yield False + return + + target_dut_hostname = None + test_func_args = [_ for _ in inspect.signature(request.node.function).parameters.keys()] + + if dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_ENUM_TOR: + # retrieve the current enumerated dut hostname + if is_test_func_parametrized: + target_dut_hostname = _get_enumerated_dut_hostname(request) + elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_UPPER_TOR: + target_dut_hostname = duthosts[0].hostname + elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_LOWER_TOR: + target_dut_hostname = duthosts[-1].hostname + elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_TOR: + target_dut_hostname = rand_one_dut_hostname_var + elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_UNSELECTED_TOR: + target_dut_hostname = rand_one_unselected_dut_hostname + else: + # if the test function uses `rand_selected_dut`, toggle active side to `rand_selected_dut` + rand_dut_fixture = "rand_one_dut_hostname" + duthost_fixture = "duthost" + if rand_dut_fixture in test_func_args or rand_dut_fixture in request.fixturenames: + logging.debug("Select random selected DUT %s as the toggle target", + rand_one_dut_hostname_var) + target_dut_hostname = rand_one_dut_hostname_var + # if the test function uses `duthost`, toggle active side to `duthost`. + elif duthost_fixture in test_func_args or duthost_fixture in request.fixturenames: + logging.debug("Select duthost DUT %s as the toggle target", + duthost.hostname) + target_dut_hostname = duthost.hostname + + if not _is_dut_hostname_valid(target_dut_hostname): + logging.warn("Invalid DUT selected %s as the toggle target, fallback to use the upper ToR %s", + target_dut_hostname, duthosts[0].hostname) + target_dut_hostname = duthosts[0].hostname + logging.info("Toggle mux ports to the target DUT %s", + target_dut_hostname) + mux_simulator_control._toggle_all_simulator_ports_to_target_dut(target_dut_hostname, + duthosts, + mux_server_url, + tbinfo) + + if dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_SETUP_MUX_PORT_MANUAL_MODE: + logger.info("Set all mux ports to manual mode on all ToRs") + duthosts.shell("config muxcable mode manual all") + + yield True + + if dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_SETUP_MUX_PORT_MANUAL_MODE: + logger.info("Set all muxcable to auto mode on all ToRs") + duthosts.shell("config muxcable mode auto all") + duthosts.shell("config save -y") + + +def pytest_runtest_setup(item): + # Let's place `setup_dualtor_mux_ports` at the tail of fixture list + # to make it running as last as possible. + fixtureinfo = item._fixtureinfo + for fixturedef in fixtureinfo.name2fixturedefs.values(): + fixturedef = fixturedef[0] + if fixturedef.argname == "setup_dualtor_mux_ports": + fixtureinfo.names_closure.remove("setup_dualtor_mux_ports") + fixtureinfo.names_closure.append("setup_dualtor_mux_ports") + + +@pytest.fixture(scope="module", autouse=True) +def yang_validation_check(request, duthosts): + """ + YANG validation check that runs before and after each test module + """ + skip_yang = request.config.getoption("--skip_yang") + + if skip_yang: + logger.info("Skipping YANG validation check due to --skip_yang flag") + return + + def run_yang_validation(stage): + """Run YANG validation and return results""" + validation_results = {} + + for duthost in duthosts: + logger.info(f"Running YANG validation on {duthost.hostname} ({stage})") + try: + result = duthost.shell( + 'echo "[]" | sudo config apply-patch /dev/stdin', + module_ignore_errors=True + ) + + if result['rc'] != 0: + validation_results[duthost.hostname] = { + 'failed': True, + 'error': result.get('stderr', result.get('stdout', 'Unknown error')) + } + logger.error(f"YANG validation failed on {duthost.hostname} ({stage}): " + f"{validation_results[duthost.hostname]['error']}") + else: + validation_results[duthost.hostname] = {'failed': False} + logger.info(f"YANG validation passed on {duthost.hostname} ({stage})") - for interface in selected_interfaces: - self.shutdown_and_startup_interfaces(dut, interface) + except Exception as e: + validation_results[duthost.hostname] = { + 'failed': True, + 'error': str(e) + } + logger.error(f"Exception during YANG validation on {duthost.hostname} ({stage}): {str(e)}") - pytest_assert(self.get_interface_status(dut, interface) == "up", - "Interface {} was not up before disabling/enabling tx-output using sfputil".format(interface)) + return validation_results - remote_fault_before = self.get_mac_fault_count(dut, interface, "mac remote fault") - logging.info("Initial MAC remote fault count on {}: {}".format(interface, remote_fault_before)) + # pre-test YANG validation + pre_results = run_yang_validation("pre-test") - dut.shell("sudo sfputil debug tx-output {} disable".format(interface)) - time.sleep(5) - pytest_assert(self.get_interface_status(dut, interface) == "down", - "Interface {iface} did not go down after 'sudo sfputil debug tx-output {iface} disable'" - .format(iface=interface)) + # Check if any pre-test validation failed + pre_failures = {host: result for host, result in pre_results.items() if result['failed']} + if pre_failures: + error_summary = [] + for host, result in pre_failures.items(): + error_summary.append(f"{host}: {result['error']}") - dut.shell("sudo sfputil debug tx-output {} enable".format(interface)) - time.sleep(20) + pt_assert(False, "pre-test YANG validation failed:\n" + "\n".join(error_summary)) - pytest_assert(self.get_interface_status(dut, interface) == "up", - "Interface {iface} did not come up after 'sudo sfputil debug tx-output {iface} enable'" - .format(iface=interface)) + yield - remote_fault_after = self.get_mac_fault_count(dut, interface, "mac remote fault") - logging.info("MAC remote fault count after disabling/enabling tx-output using sfputil {}: {}".format( - interface, remote_fault_after)) + # post-test YANG validation + post_results = run_yang_validation("post-test") - pytest_assert(remote_fault_after > remote_fault_before, - "MAC remote fault count did not increment after disabling/enabling tx-output on the device") + # Check if any post-test validation failed + post_failures = {host: result for host, result in post_results.items() if result['failed']} + if post_failures: + error_summary = [] + for host, result in post_failures.items(): + error_summary.append(f"{host}: {result['error']}") - pytest_assert(len(failed_api_ports) == 0, "Interfaces with failed API ports: {}".format(failed_api_ports)) + pt_assert(False, "post-test YANG validation failed:\n" + "\n".join(error_summary)) From ec29fab69cbdb9ec53a842b5c6d6783111539811 Mon Sep 17 00:00:00 2001 From: gshemesh2 Date: Sun, 21 Dec 2025 13:53:22 +0200 Subject: [PATCH 3/5] Update test_port_error.py Signed-off-by: Guy Shemesh --- tests/layer1/test_port_error.py | 3818 ++----------------------------- 1 file changed, 158 insertions(+), 3660 deletions(-) diff --git a/tests/layer1/test_port_error.py b/tests/layer1/test_port_error.py index 51455a61de8..deb92bb3cba 100644 --- a/tests/layer1/test_port_error.py +++ b/tests/layer1/test_port_error.py @@ -1,3716 +1,214 @@ -from functools import lru_cache -import enum -import os -import json import logging -import random -import re -import sys - import pytest -import yaml -import copy +import random import time -import subprocess -import threading -import pathlib -import importlib -import inspect - -from datetime import datetime -from ipaddress import ip_interface, IPv4Interface -from tests.common.multi_servers_utils import MultiServersUtils -from tests.common.fixtures.conn_graph_facts import conn_graph_facts # noqa: F401 -from tests.common.devices.local import Localhost -from tests.common.devices.ptf import PTFHost -from tests.common.devices.eos import EosHost -from tests.common.devices.sonic import SonicHost -from tests.common.devices.fanout import FanoutHost -from tests.common.devices.k8s import K8sMasterHost -from tests.common.devices.k8s import K8sMasterCluster -from tests.common.devices.duthosts import DutHosts -from tests.common.devices.vmhost import VMHost -from tests.common.devices.base import NeighborDevice -from tests.common.devices.cisco import CiscoHost -from tests.common.fixtures.duthost_utils import backup_and_restore_config_db_session, \ - stop_route_checker_on_duthost, start_route_checker_on_duthost # noqa: F401 -from tests.common.fixtures.ptfhost_utils import ptf_portmap_file # noqa: F401 -from tests.common.fixtures.ptfhost_utils import ptf_test_port_map_active_active # noqa: F401 -from tests.common.fixtures.ptfhost_utils import run_icmp_responder_session # noqa: F401 -from tests.common.dualtor.dual_tor_utils import disable_timed_oscillation_active_standby # noqa: F401 -from tests.common.dualtor.dual_tor_utils import config_active_active_dualtor -from tests.common.dualtor.dual_tor_common import active_active_ports # noqa: F401 -from tests.common.dualtor import mux_simulator_control # noqa: F401 - -from tests.common.helpers.constants import ( - ASIC_PARAM_TYPE_ALL, ASIC_PARAM_TYPE_FRONTEND, DEFAULT_ASIC_ID, NAMESPACE_PREFIX, - ASICS_PRESENT, DUT_CHECK_NAMESPACE -) -from tests.common.helpers.custom_msg_utils import add_custom_msg -from tests.common.helpers.dut_ports import encode_dut_port_name -from tests.common.helpers.dut_utils import encode_dut_and_container_name -from tests.common.helpers.parallel_utils import ParallelCoordinator, ParallelStatus, ParallelRunContext -from tests.common.helpers.pfcwd_helper import TrafficPorts, select_test_ports, set_pfc_timers -from tests.common.system_utils import docker -from tests.common.testbed import TestbedInfo -from tests.common.utilities import get_inventory_files, wait_until -from tests.common.utilities import get_host_vars -from tests.common.utilities import get_host_visible_vars -from tests.common.utilities import get_test_server_host -from tests.common.utilities import str2bool -from tests.common.utilities import safe_filename -from tests.common.utilities import get_duts_from_host_pattern -from tests.common.utilities import get_upstream_neigh_type, file_exists_on_dut -from tests.common.helpers.dut_utils import is_supervisor_node, is_frontend_node, create_duthost_console, creds_on_dut, \ - is_enabled_nat_for_dpu, get_dpu_names_and_ssh_ports, enable_nat_for_dpus, is_macsec_capable_node -from tests.common.cache import FactsCache -from tests.common.config_reload import config_reload -from tests.common.helpers.assertions import pytest_assert as pt_assert -from tests.common.helpers.inventory_utils import trim_inventory -from tests.common.utilities import InterruptableThread -from tests.common.plugins.ptfadapter.dummy_testutils import DummyTestUtils -from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor - -import tests.common.gnmi_setup as gnmi_setup - -try: - from tests.common.macsec import MacsecPluginT2, MacsecPluginT0 -except ImportError as e: - logging.error(e) - -from tests.common.platform.args.advanced_reboot_args import add_advanced_reboot_args -from tests.common.platform.args.cont_warm_reboot_args import add_cont_warm_reboot_args -from tests.common.platform.args.normal_reboot_args import add_normal_reboot_args -from ptf import testutils -from ptf.mask import Mask - - -logger = logging.getLogger(__name__) -cache = FactsCache() - -DUTHOSTS_FIXTURE_FAILED_RC = 15 -CUSTOM_MSG_PREFIX = "sonic_custom_msg" -GOLDEN_CONFIG_DB_PATH = "/etc/sonic/golden_config_db.json" -GOLDEN_CONFIG_DB_PATH_ORI = "/etc/sonic/golden_config_db.json.origin.backup" - -pytest_plugins = ('tests.common.plugins.ptfadapter', - 'tests.common.plugins.ansible_fixtures', - 'tests.common.plugins.dut_monitor', - 'tests.common.plugins.loganalyzer', - 'tests.common.plugins.pdu_controller', - 'tests.common.plugins.sanity_check', - 'tests.common.plugins.custom_markers', - 'tests.common.plugins.test_completeness', - 'tests.common.plugins.log_section_start', - 'tests.common.plugins.custom_fixtures', - 'tests.common.dualtor', - 'tests.decap', - 'tests.platform_tests.api', - 'tests.common.plugins.allure_server', - 'tests.common.plugins.conditional_mark', - 'tests.common.plugins.random_seed', - 'tests.common.plugins.memory_utilization', - 'tests.common.fixtures.duthost_utils') - - -def pytest_addoption(parser): - parser.addoption("--testbed", action="store", default=None, help="testbed name") - parser.addoption("--testbed_file", action="store", default=None, help="testbed file name") - parser.addoption("--uhd_config", action="store", help="Enable UHD config mode") - parser.addoption("--save_uhd_config", action="store_true", help="Save UHD config mode") - parser.addoption("--npu_dpu_startup", action="store_true", help="Startup NPU and DPUs and install configurations") - parser.addoption("--l47_trafficgen", action="store_true", help="Enable L47 trafficgen config") - parser.addoption("--save_l47_trafficgen", action="store_true", help="Save L47 trafficgen config") - - # test_vrf options - parser.addoption("--vrf_capacity", action="store", default=None, type=int, help="vrf capacity of dut (4-1000)") - parser.addoption("--vrf_test_count", action="store", default=None, type=int, - help="number of vrf to be tested (1-997)") - - # qos_sai options - parser.addoption("--ptf_portmap", action="store", default=None, type=str, - help="PTF port index to DUT port alias map") - parser.addoption("--qos_swap_syncd", action="store", type=str2bool, default=True, - help="Swap syncd container with syncd-rpc container") - - # Kubernetes master options - parser.addoption("--kube_master", action="store", default=None, type=str, - help="Name of k8s master group used in k8s inventory, format: k8s_vms{msetnumber}_{servernumber}") - - # neighbor device type - parser.addoption("--neighbor_type", action="store", default="eos", type=str, choices=["eos", "sonic", "cisco"], - help="Neighbor devices type") - - # ceos neighbor lacp multiplier - parser.addoption("--ceos_neighbor_lacp_multiplier", action="store", default=3, type=int, - help="LACP multiplier for ceos neighbors") - - # FWUtil options - parser.addoption('--fw-pkg', action='store', help='Firmware package file') - - ##################################### - # dash, vxlan, route shared options # - ##################################### - parser.addoption("--skip_cleanup", action="store_true", help="Skip config cleanup after test (tests: dash, vxlan)") - parser.addoption("--num_routes", action="store", default=None, type=int, - help="Number of routes (tests: route, vxlan)") - - ############################ - # pfc_asym options # - ############################ - parser.addoption("--server_ports_num", action="store", default=20, type=int, help="Number of server ports to use") - parser.addoption("--fanout_inventory", action="store", default="lab", help="Inventory with defined fanout hosts") - - ############################ - # test_techsupport options # - ############################ - parser.addoption("--loop_num", action="store", default=2, type=int, - help="Change default loop range for show techsupport command") - parser.addoption("--loop_delay", action="store", default=2, type=int, - help="Change default loops delay") - parser.addoption("--logs_since", action="store", type=int, - help="number of minutes for show techsupport command") - parser.addoption("--collect_techsupport", action="store", default=True, type=str2bool, - help="Enable/Disable tech support collection. Default is enabled (True)") - - ############################ - # sanity_check options # - ############################ - parser.addoption("--skip_sanity", action="store_true", default=False, - help="Skip sanity check") - parser.addoption("--allow_recover", action="store_true", default=False, - help="Allow recovery attempt in sanity check in case of failure") - parser.addoption("--check_items", action="store", default=False, - help="Change (add|remove) check items in the check list") - parser.addoption("--post_check", action="store_true", default=False, - help="Perform post test sanity check if sanity check is enabled") - parser.addoption("--post_check_items", action="store", default=False, - help="Change (add|remove) post test check items based on pre test check items") - parser.addoption("--recover_method", action="store", default="adaptive", - help="Set method to use for recover if sanity failed") - - ######################## - # pre-test options # - ######################## - parser.addoption("--deep_clean", action="store_true", default=False, - help="Deep clean DUT before tests (remove old logs, cores, dumps)") - parser.addoption("--py_saithrift_url", action="store", default=None, type=str, - help="Specify the url of the saithrift package to be installed on the ptf " - "(should be http:///path/python-saithrift_0.9.4_amd64.deb") - - ######################### - # post-test options # - ######################### - parser.addoption("--posttest_show_tech_since", action="store", default="yesterday", - help="collect show techsupport since . should be a string which can " - "be parsed by bash command 'date --d '. Default value is yesterday. " - "To collect all time spans, please use '@0' as the value.") - - ############################ - # keysight ixanvl options # - ############################ - parser.addoption("--testnum", action="store", default=None, type=str) - parser.addoption("--enable-snappi-dynamic-ports", action="store_true", default=False, - help="Force to use dynamic port allocation for snappi port selections") - - ################################## - # advance-reboot,upgrade options # - ################################## - add_advanced_reboot_args(parser) - add_cont_warm_reboot_args(parser) - add_normal_reboot_args(parser) - - ############################ - # loop_times options # - ############################ - parser.addoption("--loop_times", metavar="LOOP_TIMES", action="store", default=1, type=int, - help="Define the loop times of the test") - ############################ - # collect logs option # - ############################ - parser.addoption("--collect_db_data", action="store_true", default=False, help="Collect db info if test failed") - - ############################ - # macsec options # - ############################ - parser.addoption("--enable_macsec", action="store_true", default=False, - help="Enable macsec on some links of testbed") - parser.addoption("--macsec_profile", action="store", default="all", - type=str, help="profile name list in macsec/profile.json") - - ############################ - # QoS options # - ############################ - parser.addoption("--public_docker_registry", action="store_true", default=False, - help="To use public docker registry for syncd swap, by default is disabled (False)") - - ############################## - # ansible inventory option # - ############################## - parser.addoption("--trim_inv", action="store_true", default=False, help="Trim inventory files") - - ############################## - # gnmi connection options # - ############################## - # The gNMI target port number to connect to the DUT gNMI server. - parser.addoption("--gnmi_port", action="store", default="8080", type=str, - help="gNMI target port number") - parser.addoption("--gnmi_insecure", action="store_true", default=True, - help="Use insecure connection to gNMI target") - parser.addoption("--disable_sai_validation", action="store_true", default=True, - help="Disable SAI validation") - ############################ - # Parallel run options # - ############################ - parser.addoption("--target_hostname", action="store", default=None, type=str, - help="Target hostname to run the test in parallel") - parser.addoption("--parallel_state_file", action="store", default=None, type=str, - help="File to store the state of the parallel run") - parser.addoption("--is_parallel_leader", action="store_true", default=False, help="Is the parallel leader") - parser.addoption("--parallel_followers", action="store", default=0, type=int, help="Number of parallel followers") - parser.addoption("--parallel_mode", action="store", default=None, type=str, - help="Parallel mode to run the test. Either FULL_PARALLEL or RP_FIRST if parallel run enabled") - - ############################ - # SmartSwitch options # - ############################ - parser.addoption("--dpu-pattern", action="store", default="all", help="dpu host name") - - ################################## - # Container Upgrade options # - ################################## - parser.addoption("--containers", action="store", default=None, type=str, - help="Container bundle to test on each iteration") - parser.addoption("--os_versions", action="store", default=None, type=str, - help="OS Versions to install, one per iteration") - parser.addoption("--image_url_template", action="store", default=None, type=str, - help="Template url to use to download image") - parser.addoption("--parameters_file", action="store", default=None, type=str, - help="File that containers parameters for each container") - parser.addoption("--testcase_file", action="store", default=None, type=str, - help="File that contains testcases to execute per iteration") - - ################################# - # Stress test options # - ################################# - parser.addoption("--run-stress-tests", action="store_true", default=False, help="Run only tests stress tests") - - ################################# - # Container upgrade test options # - ################################# - parser.addoption("--container_test", action="store", default="", - help="This flag indicates that the test is being run by the container test.") - - ################################# - # YANG validation options # - ################################# - parser.addoption("--skip_yang", action="store_true", default=False, - help="Skip YANG validation") - ################################# - # Port error test options # - ################################# - parser.addoption("--collected-ports-num", action="store", default=5, type=int, - help="Number of ports to collect for testing (default: 5)") - - -def pytest_configure(config): - if config.getoption("enable_macsec"): - topo = config.getoption("topology") - if topo is not None and "t2" in topo: - config.pluginmanager.register(MacsecPluginT2()) - else: - config.pluginmanager.register(MacsecPluginT0()) - - -@pytest.fixture(scope="session", autouse=True) -def enhance_inventory(request, tbinfo): - """ - This fixture is to enhance the capability of parsing the value of pytest cli argument '--inventory'. - The pytest-ansible plugin always assumes that the value of cli argument '--inventory' is a single - inventory file. With this enhancement, we can pass in multiple inventory files using the cli argument - '--inventory'. The multiple inventory files can be separated by comma ','. - - For example: - pytest --inventory "inventory1, inventory2" - pytest --inventory inventory1,inventory2 - - This fixture is automatically applied, you don't need to declare it in your test script. - """ - inv_opt = request.config.getoption("ansible_inventory") - if isinstance(inv_opt, list): - return - inv_files = [inv_file.strip() for inv_file in inv_opt.split(",")] - - if request.config.getoption("trim_inv"): - target_hostname = get_target_hostname(request) - trim_inventory(inv_files, tbinfo, target_hostname) - - try: - logger.info(f"Inventory file: {inv_files}") - setattr(request.config.option, "ansible_inventory", inv_files) - except AttributeError: - logger.error("Failed to set enhanced 'ansible_inventory' to request.config.option") - - -def pytest_collection(session): - """Workaround to reduce messy plugin logs generated during collection only - - Args: - session (ojb): Pytest session object - """ - if session.config.option.collectonly: - root_logger = logging.getLogger() - root_logger.setLevel(logging.WARNING) - - -def get_target_hostname(request): - return request.config.getoption("--target_hostname") - - -def get_parallel_state_file(request): - return request.config.getoption("--parallel_state_file") - - -def is_parallel_run(request): - return get_target_hostname(request) is not None - - -def is_parallel_leader(request): - return request.config.getoption("--is_parallel_leader") - - -def get_parallel_followers(request): - return request.config.getoption("--parallel_followers") - - -def get_parallel_mode(request): - return request.config.getoption("--parallel_mode") - - -def get_tbinfo(request): - """ - Helper function to create and return testbed information - """ - tbname = request.config.getoption("--testbed") - tbfile = request.config.getoption("--testbed_file") - if tbname is None or tbfile is None: - raise ValueError("testbed and testbed_file are required!") - - testbedinfo = cache.read(tbname, 'tbinfo') - if testbedinfo is cache.NOTEXIST: - testbedinfo = TestbedInfo(tbfile) - cache.write(tbname, 'tbinfo', testbedinfo) +import os +import re - return tbname, testbedinfo.testbed_topo.get(tbname, {}) +from tests.common.helpers.assertions import pytest_assert +from tests.common.utilities import skip_release +from tests.common.platform.transceiver_utils import parse_sfp_eeprom_infos +from tests.common.mellanox_data import get_supported_available_optical_interfaces +from tests.common.utilities import wait_until +pytestmark = [ + pytest.mark.disable_loganalyzer, # disable automatic loganalyzer + pytest.mark.topology('any') +] -@pytest.fixture(scope="session") -def tbinfo(request): - """ - Create and return testbed information - """ - _, testbedinfo = get_tbinfo(request) - return testbedinfo +SUPPORTED_PLATFORMS = ["arista_7060x6", "nvidia_sn5640", "nvidia_sn5600"] +cmd_sfp_presence = "sudo sfpshow presence" @pytest.fixture(scope="session") -def parallel_run_context(request): - return ParallelRunContext( - is_parallel_run(request), - get_target_hostname(request), - is_parallel_leader(request), - get_parallel_followers(request), - get_parallel_state_file(request), - get_parallel_mode(request), - ) - - -def get_specified_device_info(request, device_pattern): - """ - Get a list of device hostnames specified with the --host-pattern or --dpu-pattern CLI option +def collected_ports_num(request): """ - tbname, tbinfo = get_tbinfo(request) - testbed_duts = tbinfo['duts'] - - if is_parallel_run(request): - return [get_target_hostname(request)] - - host_pattern = request.config.getoption(device_pattern) - if host_pattern == 'all': - if device_pattern == '--dpu-pattern': - testbed_duts = [dut for dut in testbed_duts if 'dpu' in dut] - logger.info(f"dpu duts: {testbed_duts}") - return testbed_duts - else: - specified_duts = get_duts_from_host_pattern(host_pattern) - - if any([dut not in testbed_duts for dut in specified_duts]): - pytest.fail("One of the specified DUTs {} does not belong to the testbed {}".format(specified_duts, tbname)) - - if len(testbed_duts) != specified_duts: - duts = specified_duts - logger.debug("Different DUTs specified than in testbed file, using {}" - .format(str(duts))) - - return duts - - -def get_specified_duts(request): - """ - Get a list of DUT hostnames specified with the --host-pattern CLI option - or -d if using `run_tests.sh` + Fixture to get the number of ports to collect from command line argument """ - return get_specified_device_info(request, "--host-pattern") + return request.config.getoption("--collected-ports-num") -def get_specified_dpus(request): - """ - Get a list of DUT hostnames specified with the --dpu-pattern CLI option - """ - return get_specified_device_info(request, "--dpu-pattern") - - -def pytest_sessionstart(session): - # reset all the sonic_custom_msg keys from cache - # reset here because this fixture will always be very first fixture to be called - cache_dir = session.config.cache._cachedir - keys = [p.name for p in cache_dir.glob('**/*') if p.is_file() and p.name.startswith(CUSTOM_MSG_PREFIX)] - for key in keys: - logger.debug("reset existing key: {}".format(key)) - session.config.cache.set(key, None) +class TestMACFault(object): + @pytest.fixture(scope="class", autouse=True) + def is_supported_nvidia_platform_with_sw_control_disabled(self, duthost): + return 'nvidia' in duthost.facts['platform'].lower() and not self.is_sw_control_feature_enabled(duthost) - # Invoke the build-gnmi-stubs.sh script - script_path = os.path.join(os.path.dirname(__file__), "build-gnmi-stubs.sh") - base_dir = os.getcwd() # Use the current working directory as the base directory - logger.info(f"Invoking {script_path} with base directory: {base_dir}") + @pytest.fixture(scope="class", autouse=True) + def is_supported_nvidia_platform_with_sw_control_enabled(self, duthost): + return 'nvidia' in duthost.facts['platform'].lower() and self.is_sw_control_feature_enabled(duthost) - try: - result = subprocess.run( - [script_path, base_dir], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - check=False # Do not raise an exception automatically on non-zero exit - ) - logger.info(f"Output of {script_path}:\n{result.stdout}") - # logger.error(f"Error output of {script_path}:\n{result.stderr}") + @pytest.fixture(scope="class", autouse=True) + def is_supported_platform(self, duthost, tbinfo, is_supported_nvidia_platform_with_sw_control_disabled): + if 'ptp' not in tbinfo['topo']['name']: + pytest.skip("Skipping test: Not applicable for non-PTP topology") - if result.returncode != 0: - logger.error(f"{script_path} failed with exit code {result.returncode}") - session.exitstatus = 1 # Fail the pytest session + if any(platform in duthost.facts['platform'] for platform in SUPPORTED_PLATFORMS): + skip_release(duthost, ["201811", "201911", "202012", "202205", "202211", "202305", "202405"]) else: - # Add the generated directory to sys.path for module imports - generated_path = os.path.join(base_dir, "common", "sai_validation", "generated") - if generated_path not in sys.path: - sys.path.insert(0, generated_path) - logger.info(f"Added {generated_path} to sys.path") - except Exception as e: - logger.error(f"Exception occurred while invoking {script_path}: {e}") - session.exitstatus = 1 # Fail the pytest session - - -def pytest_sessionfinish(session, exitstatus): - if session.config.cache.get("duthosts_fixture_failed", None): - session.config.cache.set("duthosts_fixture_failed", None) - session.exitstatus = DUTHOSTS_FIXTURE_FAILED_RC - + pytest.skip("DUT has platform {}, test is not supported".format(duthost.facts['platform'])) -@pytest.fixture(name="duthosts", scope="session") -def fixture_duthosts(enhance_inventory, ansible_adhoc, tbinfo, request): - """ - @summary: fixture to get DUT hosts defined in testbed. - @param enhance_inventory: fixture to enhance the capability of parsing the value of pytest cli argument - @param ansible_adhoc: Fixture provided by the pytest-ansible package. - Source of the various device objects. It is - mandatory argument for the class constructors. - @param tbinfo: fixture provides information about testbed. - @param request: pytest request object - """ - try: - host = DutHosts(ansible_adhoc, tbinfo, request, get_specified_duts(request), - target_hostname=get_target_hostname(request), is_parallel_leader=is_parallel_leader(request)) - return host - except BaseException as e: - logger.error("Failed to initialize duthosts.") - request.config.cache.set("duthosts_fixture_failed", True) - pt_assert(False, "!!!!!!!!!!!!!!!! duthosts fixture failed !!!!!!!!!!!!!!!!" - "Exception: {}".format(repr(e))) - - -@pytest.fixture(scope="session") -def duthost(duthosts, request): - ''' - @summary: Shortcut fixture for getting DUT host. For a lengthy test case, test case module can - pass a request to disable sh time out mechanis on dut in order to avoid ssh timeout. - After test case completes, the fixture will restore ssh timeout. - @param duthosts: fixture to get DUT hosts - @param request: request parameters for duthost test fixture - ''' - dut_index = getattr(request.session, "dut_index", 0) - assert dut_index < len(duthosts), \ - "DUT index '{0}' is out of bound '{1}'".format(dut_index, - len(duthosts)) - - duthost = duthosts[dut_index] - - return duthost - - -@pytest.fixture(scope="session") -def enable_nat_for_dpuhosts(duthosts, ansible_adhoc, request): - """ - @summary: fixture to enable nat for dpuhost. - @param duthosts: fixture to get DUT hosts - @param ansible_adhoc: Fixture provided by the pytest-ansible package. - Source of the various device objects. It is - mandatory argument for the class constructors. - @param request: request parameters for duthost test fixture - """ - dpuhost_names = get_specified_dpus(request) - if dpuhost_names: - logging.info(f"dpuhost_names: {dpuhost_names}") - for duthost in duthosts: - if not is_enabled_nat_for_dpu(duthost, request): - dpu_name_ssh_port_dict = get_dpu_names_and_ssh_ports(duthost, dpuhost_names, ansible_adhoc) - enable_nat_for_dpus(duthost, dpu_name_ssh_port_dict, request) - - -@pytest.fixture(name="dpuhosts", scope="session") -def fixture_dpuhosts(enhance_inventory, ansible_adhoc, tbinfo, request, enable_nat_for_dpuhosts): - """ - @summary: fixture to get DPU hosts defined in testbed. - @param ansible_adhoc: Fixture provided by the pytest-ansible package. - Source of the various device objects. It is - mandatory argument for the class constructors. - @param tbinfo: fixture provides information about testbed. - """ - # Before calling dpuhosts, we must enable NAT on NPU. - # E.g. run sonic-dpu-mgmt-traffic.sh on NPU to enable NAT - # sonic-dpu-mgmt-traffic.sh inbound -e --dpus all --ports 5021,5022,5023,5024 - try: - host = DutHosts(ansible_adhoc, tbinfo, request, get_specified_dpus(request), - target_hostname=get_target_hostname(request), is_parallel_leader=is_parallel_leader(request)) - return host - except BaseException as e: - logger.error("Failed to initialize dpuhosts.") - request.config.cache.set("dpuhosts_fixture_failed", True) - pt_assert(False, "!!!!!!!!!!!!!!!! dpuhosts fixture failed !!!!!!!!!!!!!!!!" - "Exception: {}".format(repr(e))) + if is_supported_nvidia_platform_with_sw_control_disabled: + pytest.skip("SW control feature is not enabled on Nvidia platform") + @staticmethod + def get_mac_fault_count(dut, interface, fault_type): + output = dut.show_and_parse("show int errors {}".format(interface)) + logging.info("Raw output for show int errors on {}: {}".format(interface, output)) -@pytest.fixture(scope="session") -def dpuhost(dpuhosts, request): - ''' - @summary: Shortcut fixture for getting DPU host. For a lengthy test case, test case module can - pass a request to disable sh time out mechanis on dut in order to avoid ssh timeout. - After test case completes, the fixture will restore ssh timeout. - @param duthosts: fixture to get DPU hosts - @param request: request parameters for duphost test fixture - ''' - dpu_index = getattr(request.session, "dpu_index", 0) - assert dpu_index < len(dpuhosts), \ - "DPU index '{0}' is out of bound '{1}'".format(dpu_index, - len(dpuhosts)) - - duthost = dpuhosts[dpu_index] - - return duthost - - -@pytest.fixture(scope="session") -def mg_facts(duthost): - return duthost.minigraph_facts(host=duthost.hostname)['ansible_facts'] - - -@pytest.fixture(scope="session") -def macsec_duthost(duthosts, tbinfo): - # get the first macsec capable node - macsec_dut = None - if 't2' in tbinfo['topo']['name']: - # currently in the T2 topo only the uplink linecard will have - # macsec enabled - for duthost in duthosts: - if duthost.is_macsec_capable_node(): - macsec_dut = duthost + fault_count = 0 + for error_info in output: + if error_info['port errors'] == fault_type: + fault_count = int(error_info['count']) break - else: - return duthosts[0] - return macsec_dut - - -@pytest.fixture(scope="session") -def is_macsec_enabled_for_test(duthosts): - # If macsec is enabled, use the override option to get macsec profile from golden config - macsec_en = False - request = duthosts.request - if request: - macsec_en = request.config.getoption("--enable_macsec", default=False) - return macsec_en - - -# Make sure in same test module, always use same random DUT -rand_one_dut_hostname_var = None - - -def set_rand_one_dut_hostname(request): - global rand_one_dut_hostname_var - if rand_one_dut_hostname_var is None: - dut_hostnames = generate_params_dut_hostname(request) - if len(dut_hostnames) > 1: - dut_hostnames = random.sample(dut_hostnames, 1) - rand_one_dut_hostname_var = dut_hostnames[0] - logger.info("Randomly select dut {} for testing".format(rand_one_dut_hostname_var)) + logging.info("{} count on {}: {}".format(fault_type, interface, fault_count)) + return fault_count -@pytest.fixture(scope="module") -def rand_one_dut_hostname(request): - """ - """ - global rand_one_dut_hostname_var - if rand_one_dut_hostname_var is None: - set_rand_one_dut_hostname(request) - return rand_one_dut_hostname_var - - -@pytest.fixture(scope="module") -def rand_selected_dut(duthosts, rand_one_dut_hostname): - """ - Return the randomly selected duthost - """ - return duthosts[rand_one_dut_hostname] - - -@pytest.fixture(scope="module") -def selected_rand_dut(request): - global rand_one_dut_hostname_var - if rand_one_dut_hostname_var is None: - set_rand_one_dut_hostname(request) - return rand_one_dut_hostname_var - - -@pytest.fixture(scope="module") -def rand_one_dut_front_end_hostname(request): - """ - """ - dut_hostnames = generate_params_frontend_hostname(request) - if len(dut_hostnames) > 1: - dut_hostnames = random.sample(dut_hostnames, 1) - logger.info("Randomly select dut {} for testing".format(dut_hostnames[0])) - return dut_hostnames[0] - - -@pytest.fixture(scope="module") -def rand_one_tgen_dut_hostname(request, tbinfo, rand_one_dut_front_end_hostname, rand_one_dut_hostname): - """ - Return the randomly selected duthost for TGEN test cases - """ - # For T2, we need to skip supervisor, only use linecards. - if 't2' in tbinfo['topo']['name']: - return rand_one_dut_front_end_hostname - return rand_one_dut_hostname - - -@pytest.fixture(scope="module") -def rand_selected_front_end_dut(duthosts, rand_one_dut_front_end_hostname): - """ - Return the randomly selected duthost - """ - return duthosts[rand_one_dut_front_end_hostname] - - -@pytest.fixture(scope="module") -def rand_unselected_dut(request, duthosts, rand_one_dut_hostname): - """ - Return the left duthost after random selection. - Return None for non dualtor testbed - """ - dut_hostnames = generate_params_dut_hostname(request) - if len(dut_hostnames) <= 1: - return None - idx = dut_hostnames.index(rand_one_dut_hostname) - return duthosts[dut_hostnames[1 - idx]] - - -@pytest.fixture(scope="module") -def selected_rand_one_per_hwsku_hostname(request): - """ - Return the selected hostnames for the given module. - This fixture will return the list of selected dut hostnames - when another fixture like enum_rand_one_per_hwsku_hostname - or enum_rand_one_per_hwsku_frontend_hostname is used. - """ - if request.module in _hosts_per_hwsku_per_module: - return _hosts_per_hwsku_per_module[request.module] - else: - return [] - - -@pytest.fixture(scope="module") -def rand_one_dut_portname_oper_up(request): - oper_up_ports = generate_port_lists(request, "oper_up_ports") - if len(oper_up_ports) > 1: - oper_up_ports = random.sample(oper_up_ports, 1) - return oper_up_ports[0] - - -@pytest.fixture(scope="module") -def rand_one_dut_lossless_prio(request): - lossless_prio_list = generate_priority_lists(request, 'lossless') - if len(lossless_prio_list) > 1: - lossless_prio_list = random.sample(lossless_prio_list, 1) - return lossless_prio_list[0] + @staticmethod + def get_interface_status(dut, interface): + return dut.show_and_parse("show interfaces status {}".format(interface))[0].get("oper", "unknown") + @pytest.fixture(scope="class", autouse=True) + def reboot_dut(self, duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname): + from tests.common.reboot import reboot + reboot(duthosts[enum_rand_one_per_hwsku_frontend_hostname], + localhost, safe_reboot=True, check_intf_up_ports=True) -@pytest.fixture(scope="module", autouse=True) -def reset_critical_services_list(duthosts): - """ - Resets the critical services list between test modules to ensure that it is - left in a known state after tests finish running. - """ - [a_dut.critical_services_tracking_list() for a_dut in duthosts] - - -@pytest.fixture(scope="session") -def localhost(ansible_adhoc): - return Localhost(ansible_adhoc) - - -@pytest.fixture(scope="session") -def ptfhost(ptfhosts): - if not ptfhosts: - return ptfhosts - return ptfhosts[0] # For backward compatibility, this is for single ptfhost testbed. - - -@pytest.fixture(scope="session") -def ptfhosts(enhance_inventory, ansible_adhoc, tbinfo, duthost, request): - _hosts = [] - if 'ptp' in tbinfo['topo']['name']: - return None - if tbinfo['topo']['name'].startswith("nut-"): - return None - if "ptf_image_name" in tbinfo and "docker-keysight-api-server" in tbinfo["ptf_image_name"]: - return None - if "ptf" in tbinfo: - _hosts.append(PTFHost(ansible_adhoc, tbinfo["ptf"], duthost, tbinfo, - macsec_enabled=request.config.option.enable_macsec)) - elif "servers" in tbinfo: - for server in tbinfo["servers"].values(): - if "ptf" in server and server["ptf"]: - _host = PTFHost(ansible_adhoc, server["ptf"], duthost, tbinfo, - macsec_enabled=request.config.option.enable_macsec) - _hosts.append(_host) - else: - # when no ptf defined in testbed.csv - # try to parse it from inventory - ptf_host = duthost.host.options["inventory_manager"].get_host(duthost.hostname).get_vars()["ptf_host"] - _hosts.apend(PTFHost(ansible_adhoc, ptf_host, duthost, tbinfo, - macsec_enabled=request.config.option.enable_macsec)) - return _hosts - - -@pytest.fixture(scope="module") -def k8smasters(enhance_inventory, ansible_adhoc, request): - """ - Shortcut fixture for getting Kubernetes master hosts - """ - k8s_master_ansible_group = request.config.getoption("--kube_master") - master_vms = {} - inv_files = request.config.getoption("ansible_inventory") - k8s_inv_file = None - for inv_file in inv_files: - if "k8s" in inv_file: - k8s_inv_file = inv_file - if not k8s_inv_file: - pytest.skip("k8s inventory not found, skipping tests") - with open('../ansible/{}'.format(k8s_inv_file), 'r') as kinv: - k8sinventory = yaml.safe_load(kinv) - for hostname, attributes in list(k8sinventory[k8s_master_ansible_group]['hosts'].items()): - if 'haproxy' in attributes: - is_haproxy = True - else: - is_haproxy = False - master_vms[hostname] = {'host': K8sMasterHost(ansible_adhoc, - hostname, - is_haproxy)} - return master_vms - + @pytest.fixture(scope="class") + def get_dut_and_supported_available_optical_interfaces(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, + is_supported_nvidia_platform_with_sw_control_enabled): + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] -@pytest.fixture(scope="module") -def k8scluster(k8smasters): - k8s_master_cluster = K8sMasterCluster(k8smasters) - return k8s_master_cluster + sfp_presence = dut.command(cmd_sfp_presence) + parsed_presence = {line.split()[0]: line.split()[1] for line in sfp_presence["stdout_lines"][2:]} + supported_available_optical_interfaces = [] + failed_api_ports = [] + if is_supported_nvidia_platform_with_sw_control_enabled: -@pytest.fixture(scope="session") -def nbrhosts(enhance_inventory, ansible_adhoc, tbinfo, creds, request): - """ - Shortcut fixture for getting VM host - """ - logger.info("Fixture nbrhosts started") - devices = {} - if ('vm_base' in tbinfo and not tbinfo['vm_base'] and 'tgen' in tbinfo['topo']['name']) or \ - 'ptf' in tbinfo['topo']['name'] or \ - 'ixia' in tbinfo['topo']['name']: - logger.info("No VMs exist for this topology: {}".format(tbinfo['topo']['name'])) - return devices - - neighbor_type = request.config.getoption("--neighbor_type") - if 'VMs' not in tbinfo['topo']['properties']['topology']: - logger.info("No VMs exist for this topology: {}".format(tbinfo['topo']['properties']['topology'])) - return devices + eeprom_infos = dut.shell("sudo sfputil show eeprom -d")['stdout'] + eeprom_infos = parse_sfp_eeprom_infos(eeprom_infos) - def initial_neighbor(neighbor_name, vm_name): - logger.info(f"nbrhosts started: {neighbor_name}_{vm_name}") - if neighbor_type == "eos": - device = NeighborDevice( - { - 'host': EosHost( - ansible_adhoc, - vm_name, - creds['eos_login'], - creds['eos_password'], - shell_user=creds['eos_root_user'] if 'eos_root_user' in creds else None, - shell_passwd=creds['eos_root_password'] if 'eos_root_password' in creds else None - ), - 'conf': tbinfo['topo']['properties']['configuration'][neighbor_name] - } - ) - elif neighbor_type == "sonic": - device = NeighborDevice( - { - 'host': SonicHost( - ansible_adhoc, - vm_name, - ssh_user=creds['sonic_login'] if 'sonic_login' in creds else None, - ssh_passwd=creds['sonic_password'] if 'sonic_password' in creds else None - ), - 'conf': tbinfo['topo']['properties']['configuration'][neighbor_name] - } - ) - elif neighbor_type == "cisco": - device = NeighborDevice( - { - 'host': CiscoHost( - ansible_adhoc, - vm_name, - creds['cisco_login'], - creds['cisco_password'], - ), - 'conf': tbinfo['topo']['properties']['configuration'][neighbor_name] - } + supported_available_optical_interfaces, failed_api_ports = ( + get_supported_available_optical_interfaces( + eeprom_infos, parsed_presence, return_failed_api_ports=True + ) ) - else: - raise ValueError("Unknown neighbor type %s" % (neighbor_type,)) - devices[neighbor_name] = device - logger.info(f"nbrhosts finished: {neighbor_name}_{vm_name}") - - servers = [] - if 'servers' in tbinfo: - servers.extend(tbinfo['servers'].values()) - elif 'server' in tbinfo: - servers.append(tbinfo) - else: - logger.warning("Unknown testbed schema for setup nbrhosts") - - with SafeThreadPoolExecutor(max_workers=8) as executor: - for server in servers: - vm_base = int(server['vm_base'][2:]) - vm_name_fmt = 'VM%0{}d'.format(len(server['vm_base']) - 2) - vms = MultiServersUtils.get_vms_by_dut_interfaces( - tbinfo['topo']['properties']['topology']['VMs'], - server['dut_interfaces'] - ) if 'dut_interfaces' in server else tbinfo['topo']['properties']['topology']['VMs'] - for neighbor_name, neighbor in vms.items(): - vm_name = vm_name_fmt % (vm_base + neighbor['vm_offset']) - executor.submit(initial_neighbor, neighbor_name, vm_name) - - logger.info("Fixture nbrhosts finished") - return devices - - -@pytest.fixture(scope="module") -def fanouthosts(enhance_inventory, ansible_adhoc, tbinfo, conn_graph_facts, creds, duthosts): # noqa: F811 - """ - Shortcut fixture for getting Fanout hosts - """ - - dev_conn = conn_graph_facts.get('device_conn', {}) - fanout_hosts = {} - - if tbinfo['topo']['name'].startswith('nut-'): - # Nut topology has no fanout - return fanout_hosts - - # WA for virtual testbed which has no fanout - for dut_host, value in list(dev_conn.items()): - duthost = duthosts[dut_host] - if duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0': - continue # skip for kvm platform which has no fanout - mg_facts = duthost.minigraph_facts(host=duthost.hostname)['ansible_facts'] - for dut_port in list(value.keys()): - fanout_rec = value[dut_port] - fanout_host = str(fanout_rec['peerdevice']) - fanout_port = str(fanout_rec['peerport']) - - if fanout_host in list(fanout_hosts.keys()): - fanout = fanout_hosts[fanout_host] - else: - host_vars = ansible_adhoc().options[ - 'inventory_manager'].get_host(fanout_host).vars - os_type = host_vars.get('os', 'eos') - if 'fanout_tacacs_user' in creds: - fanout_user = creds['fanout_tacacs_user'] - fanout_password = creds['fanout_tacacs_password'] - elif 'fanout_tacacs_{}_user'.format(os_type) in creds: - fanout_user = creds['fanout_tacacs_{}_user'.format(os_type)] - fanout_password = creds['fanout_tacacs_{}_password'.format(os_type)] - elif os_type == 'sonic': - fanout_user = creds.get('fanout_sonic_user', None) - fanout_password = creds.get('fanout_sonic_password', None) - elif os_type == 'eos': - fanout_user = creds.get('fanout_network_user', None) - fanout_password = creds.get('fanout_network_password', None) - elif os_type == 'onyx': - fanout_user = creds.get('fanout_mlnx_user', None) - fanout_password = creds.get('fanout_mlnx_password', None) - elif os_type == 'ixia': - # Skip for ixia device which has no fanout - continue - else: - # when os is mellanox, not supported - pytest.fail("os other than sonic and eos not supported") - - eos_shell_user = None - eos_shell_password = None - if os_type == "eos": - admin_user = creds['fanout_admin_user'] - admin_password = creds['fanout_admin_password'] - eos_shell_user = creds.get('fanout_shell_user', admin_user) - eos_shell_password = creds.get('fanout_shell_password', admin_password) - - fanout = FanoutHost(ansible_adhoc, - os_type, - fanout_host, - 'FanoutLeaf', - fanout_user, - fanout_password, - eos_shell_user=eos_shell_user, - eos_shell_passwd=eos_shell_password) - fanout.dut_hostnames = [dut_host] - fanout_hosts[fanout_host] = fanout - - if fanout.os == 'sonic': - ifs_status = fanout.host.get_interfaces_status() - for key, interface_info in list(ifs_status.items()): - fanout.fanout_port_alias_to_name[interface_info['alias']] = interface_info['interface'] - logging.info("fanout {} fanout_port_alias_to_name {}" - .format(fanout_host, fanout.fanout_port_alias_to_name)) - - fanout.add_port_map(encode_dut_port_name(dut_host, dut_port), fanout_port) - - # Add port name to fanout port mapping port if dut_port is alias. - if dut_port in mg_facts['minigraph_port_alias_to_name_map']: - mapped_port = mg_facts['minigraph_port_alias_to_name_map'][dut_port] - # only add the mapped port which isn't in device_conn ports to avoid overwriting port map wrongly, - # it happens when an interface has the same name with another alias, for example: - # Interface Alias - # -------------------- - # Ethernet108 Ethernet32 - # Ethernet32 Ethernet13/1 - if mapped_port not in list(value.keys()): - fanout.add_port_map(encode_dut_port_name(dut_host, mapped_port), fanout_port) - - if dut_host not in fanout.dut_hostnames: - fanout.dut_hostnames.append(dut_host) - - return fanout_hosts - - -@pytest.fixture(scope="session") -def vmhost(vmhosts): - if not vmhosts: - return vmhosts - return vmhosts[0] # For backward compatibility, this is for single vmhost testbed. - - -@pytest.fixture(scope="session") -def vmhosts(enhance_inventory, ansible_adhoc, request, tbinfo): - hosts = [] - inv_files = get_inventory_files(request) - if 'ptp' in tbinfo['topo']['name']: - return None - elif "servers" in tbinfo: - for server in tbinfo["servers"].keys(): - vmhost = get_test_server_host(inv_files, server) - hosts.append(VMHost(ansible_adhoc, vmhost.name)) - elif "server" in tbinfo: - server = tbinfo["server"] - vmhost = get_test_server_host(inv_files, server) - hosts.append(VMHost(ansible_adhoc, vmhost.name)) - else: - logger.info("No VM host exist for this topology: {}".format(tbinfo['topo']['name'])) - return hosts - - -@pytest.fixture(scope='session') -def eos(): - """ read and yield eos configuration """ - with open('eos/eos.yml') as stream: - eos = yaml.safe_load(stream) - return eos - - -@pytest.fixture(scope='session') -def sonic(): - """ read and yield sonic configuration """ - with open('sonic/sonic.yml') as stream: - eos = yaml.safe_load(stream) - return eos + pytest_assert(supported_available_optical_interfaces, + "No interfaces with SFP detected. Cannot proceed with tests.") + logging.info("Available Optical interfaces for tests: {}".format(supported_available_optical_interfaces)) + else: + interfaces = list(dut.show_and_parse("show interfaces status")) + supported_available_optical_interfaces = [ + intf["interface"] for intf in interfaces + if parsed_presence.get(intf["interface"]) == "Present" + ] + pytest_assert(supported_available_optical_interfaces, + "No interfaces with SFP detected. Cannot proceed with tests.") -@pytest.fixture(scope='session') -def pdu(): - """ read and yield pdu configuration """ - with open('../ansible/group_vars/pdu/pdu.yml') as stream: - pdu = yaml.safe_load(stream) - return pdu + return dut, supported_available_optical_interfaces, failed_api_ports + def is_sw_control_feature_enabled(self, duthost): + """ + Check if SW control feature is enabled. + """ + try: + platform_name = duthost.facts['platform'] + hwsku = duthost.facts.get('hwsku', '') + sai_profile_path = os.path.join('/usr/share/sonic/device', platform_name, hwsku, 'sai.profile') + cmd = duthost.shell('cat {}'.format(sai_profile_path), module_ignore_errors=True) + if cmd['rc'] == 0 and 'SAI_INDEPENDENT_MODULE_MODE' in cmd['stdout']: + sc_enabled = re.search(r"SAI_INDEPENDENT_MODULE_MODE=(\d?)", cmd['stdout']) + if sc_enabled and sc_enabled.group(1) == '1': + return True + except Exception as e: + logging.error("Error checking SW control feature on Nvidia platform: {}".format(e)) + return False -@pytest.fixture(scope="session") -def creds(duthost): - return creds_on_dut(duthost) + def shutdown_and_startup_interfaces(self, dut, interface): + dut.command("sudo config interface shutdown {}".format(interface)) + pytest_assert(wait_until(30, 2, 0, lambda: self.get_interface_status(dut, interface) == "down"), + "Interface {} did not go down after shutdown".format(interface)) + dut.command("sudo config interface startup {}".format(interface)) + pytest_assert(wait_until(30, 2, 0, lambda: self.get_interface_status(dut, interface) == "up"), + "Interface {} did not come up after startup".format(interface)) -@pytest.fixture(scope="session") -def topo_bgp_routes(localhost, ptfhosts, tbinfo): - bgp_routes = {} - topo_name = tbinfo['topo']['name'] - servers_dut_interfaces = None - if 'servers' in tbinfo: - servers_dut_interfaces = {value['ptf_ip'].split("/")[0]: value['dut_interfaces'] - for value in tbinfo['servers'].values()} - for ptfhost in ptfhosts: - ptf_ip = ptfhost.mgmt_ip - res = localhost.announce_routes( - topo_name=topo_name, - ptf_ip=ptf_ip, - action='generate', - path="../ansible/", - log_path="logs", - dut_interfaces=servers_dut_interfaces.get(ptf_ip) if servers_dut_interfaces else None, + def test_mac_local_fault_increment(self, get_dut_and_supported_available_optical_interfaces, + collected_ports_num): + dut, supported_available_optical_interfaces, failed_api_ports = ( + get_dut_and_supported_available_optical_interfaces() ) - if 'topo_routes' not in res: - logger.warning("No routes generated.") - else: - for host in res['topo_routes'].keys(): - if host in bgp_routes: - pytest.fail("Duplicate vm name={} on multiple servers".format(host)) - bgp_routes[host] = res['topo_routes'][host] - return bgp_routes - - -@pytest.fixture(scope='module') -def creds_all_duts(duthosts): - creds_all_duts = dict() - for duthost in duthosts.nodes: - creds_all_duts[duthost.hostname] = creds_on_dut(duthost) - return creds_all_duts - - -def update_custom_msg(custom_msg, key, value): - if custom_msg is None: - custom_msg = {} - chunks = key.split('.') - if chunks[0] == CUSTOM_MSG_PREFIX: - chunks = chunks[1:] - if len(chunks) == 1: - custom_msg.update({chunks[0]: value}) - return custom_msg - if chunks[0] not in custom_msg: - custom_msg[chunks[0]] = {} - custom_msg[chunks[0]] = update_custom_msg(custom_msg[chunks[0]], '.'.join(chunks[1:]), value) - return custom_msg - - -def log_custom_msg(item): - # temp log output to track module name - logger.debug("[log_custom_msg] item: {}".format(item)) + selected_interfaces = random.sample(supported_available_optical_interfaces, + min(collected_ports_num, len(supported_available_optical_interfaces))) + logging.info("Selected interfaces for tests: {}".format(selected_interfaces)) - cache_dir = item.session.config.cache._cachedir - keys = [p.name for p in cache_dir.glob('**/*') if p.is_file() and p.name.startswith(CUSTOM_MSG_PREFIX)] + for interface in selected_interfaces: + self.shutdown_and_startup_interfaces(dut, interface) - custom_msg = {} - for key in keys: - value = item.session.config.cache.get(key, None) - if value is not None: - custom_msg = update_custom_msg(custom_msg, key, value) + pytest_assert(self.get_interface_status(dut, interface) == "up", + "Interface {} was not up before disabling/enabling rx-output using sfputil".format(interface)) - if custom_msg: - logger.debug("append custom_msg: {}".format(custom_msg)) - item.user_properties.append(('CustomMsg', json.dumps(custom_msg))) + local_fault_before = self.get_mac_fault_count(dut, interface, "mac local fault") + logging.info("Initial MAC local fault count on {}: {}".format(interface, local_fault_before)) + dut.shell("sudo sfputil debug rx-output {} disable".format(interface)) + time.sleep(5) + pytest_assert(self.get_interface_status(dut, interface) == "down", + "Interface {iface} did not go down after 'sudo sfputil debug rx-output {iface} disable'" + .format(iface=interface)) -# This function is a pytest hook implementation that is called to create a test report. -# By placing the call to log_custom_msg in the 'teardown' phase, we ensure that it is executed -# at the end of each test, after all other fixture teardowns. This guarantees that any custom -# messages are logged at the latest possible stage in the test lifecycle. -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_makereport(item, call): + dut.shell("sudo sfputil debug rx-output {} enable".format(interface)) + time.sleep(20) + pytest_assert(self.get_interface_status(dut, interface) == "up", + "Interface {iface} did not come up after 'sudo sfputil debug rx-output {iface} enable'" + .format(iface=interface)) - if call.when == 'setup': - item.user_properties.append(('start', str(datetime.fromtimestamp(call.start)))) - elif call.when == 'teardown': - if item.nodeid == item.session.items[-1].nodeid: - log_custom_msg(item) - item.user_properties.append(('end', str(datetime.fromtimestamp(call.stop)))) - - # Filter out unnecessary logs captured on "stdout" and "stderr" - item._report_sections = list([report for report in item._report_sections if report[1] not in ("stdout", "stderr")]) - - # execute all other hooks to obtain the report object - outcome = yield - rep = outcome.get_result() - - # set a report attribute for each phase of a call, which can - # be "setup", "call", "teardown" - - setattr(item, "rep_" + rep.when, rep) - - -# This function is a pytest hook implementation that is called in runtest call stage. -# We are using this hook to set ptf.testutils to DummyTestUtils if the test is marked with "skip_traffic_test", -# DummyTestUtils would always return True for all verify function in ptf.testutils. -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_call(item): - # See tests/common/plugins/conditional_mark/tests_mark_conditions_skip_traffic_test.yaml - if "skip_traffic_test" in item.keywords: - logger.info("Got skip_traffic_test marker, will skip traffic test") - with DummyTestUtils(): - logger.info("Set ptf.testutils to DummyTestUtils to skip traffic test") - yield - logger.info("Reset ptf.testutils") - else: - yield - - -def collect_techsupport_on_dut(request, a_dut): - # request.node is an "item" because we use the default - # "function" scope - testname = request.node.name - if request.config.getoption("--collect_techsupport") and request.node.rep_call.failed: - res = a_dut.shell("generate_dump -s \"-2 hours\"") - fname = res['stdout_lines'][-1] - a_dut.fetch(src=fname, dest="logs/{}".format(testname)) - - logging.info("########### Collected tech support for test {} ###########".format(testname)) - - -@pytest.fixture -def collect_techsupport(request, duthosts, enum_dut_hostname): - yield - # request.node is an "item" because we use the default - # "function" scope - duthost = duthosts[enum_dut_hostname] - collect_techsupport_on_dut(request, duthost) - - -@pytest.fixture -def collect_techsupport_all_duts(request, duthosts): - yield - [collect_techsupport_on_dut(request, a_dut) for a_dut in duthosts] - - -@pytest.fixture -def collect_techsupport_all_nbrs(request, nbrhosts): - yield - if request.config.getoption("neighbor_type") == "sonic": - [collect_techsupport_on_dut(request, nbrhosts[nbrhost]['host']) for nbrhost in nbrhosts] - - -@pytest.fixture(scope="session", autouse=True) -def tag_test_report(request, pytestconfig, tbinfo, duthost, record_testsuite_property): - if not request.config.getoption("--junit-xml"): - return - - # Test run information - record_testsuite_property("topology", tbinfo["topo"]["name"]) - record_testsuite_property("testbed", tbinfo["conf-name"]) - record_testsuite_property("timestamp", datetime.utcnow()) - - # Device information - record_testsuite_property("host", duthost.hostname) - record_testsuite_property("asic", duthost.facts["asic_type"]) - record_testsuite_property("platform", duthost.facts["platform"]) - record_testsuite_property("hwsku", duthost.facts["hwsku"]) - record_testsuite_property("os_version", duthost.os_version) - - -@pytest.fixture(scope="module", autouse=True) -def clear_neigh_entries(duthosts, tbinfo): - """ - This is a stop bleeding change for dualtor testbed. Because dualtor duts will - learn the same set of arp entries during tests. But currently the test only - cleans up on the dut under test. So the other dut will accumulate arp entries - until kernel start to barf. - Adding this fixture to flush out IPv4/IPv6 static ARP entries after each test - moduel is done. - """ - - yield - - if 'dualtor' in tbinfo['topo']['name']: - for dut in duthosts: - dut.command("sudo ip neigh flush nud permanent") - - -@pytest.fixture(scope="module") -def patch_lldpctl(): - def patch_lldpctl(localhost, duthost): - output = localhost.shell('ansible --version') - if 'ansible 2.8.12' in output['stdout']: - """ - Work around a known lldp module bug in ansible version 2.8.12: - When neighbor sent more than one unknown tlv. Ansible will throw - exception. - This function applies the patch before test. - """ - duthost.shell( - 'sudo sed -i -e \'s/lldp lldpctl "$@"$/lldp lldpctl "$@" | grep -v "unknown-tlvs"/\' /usr/bin/lldpctl' - ) - - return patch_lldpctl - - -@pytest.fixture(scope="module") -def unpatch_lldpctl(): - def unpatch_lldpctl(localhost, duthost): - output = localhost.shell('ansible --version') - if 'ansible 2.8.12' in output['stdout']: - """ - Work around a known lldp module bug in ansible version 2.8.12: - When neighbor sent more than one unknown tlv. Ansible will throw - exception. - This function removes the patch after the test is done. - """ - duthost.shell( - 'sudo sed -i -e \'s/lldp lldpctl "$@"$/lldp lldpctl "$@" | grep -v "unknown-tlvs"/\' /usr/bin/lldpctl' - ) + local_fault_after = self.get_mac_fault_count(dut, interface, "mac local fault") + logging.info("MAC local fault count after disabling/enabling rx-output using sfputil {}: {}".format( + interface, local_fault_after)) - return unpatch_lldpctl + pytest_assert(local_fault_after > local_fault_before, + "MAC local fault count did not increment after disabling/enabling rx-output on the device") + pytest_assert(len(failed_api_ports) == 0, "Interfaces with failed API ports: {}".format(failed_api_ports)) -@pytest.fixture(scope="module") -def disable_container_autorestart(): - def disable_container_autorestart(duthost, testcase="", feature_list=None): - ''' - @summary: Disable autorestart of the features present in feature_list. - - @param duthosts: Instance of DutHost - @param testcase: testcase name used to save pretest autorestart state. Later to be used for restoration. - @feature_list: List of features to disable autorestart. If None, autorestart of all the features will be - disabled. - ''' - command_output = duthost.shell("show feature autorestart", module_ignore_errors=True) - if command_output['rc'] != 0: - logging.info("Feature autorestart utility not supported. Error: {}".format(command_output['stderr'])) - logging.info("Skipping disable_container_autorestart") - return - container_autorestart_states = duthost.get_container_autorestart_states() - state_file_name = "/tmp/autorestart_state_{}_{}.json".format(duthost.hostname, testcase) - # Dump autorestart state to file - with open(state_file_name, "w") as f: - json.dump(container_autorestart_states, f) - # Disable autorestart for all containers - logging.info("Disable container autorestart") - cmd_disable = "config feature autorestart {} disabled" - cmds_disable = [] - for name, state in list(container_autorestart_states.items()): - if state == "enabled" and (feature_list is None or name in feature_list): - cmds_disable.append(cmd_disable.format(name)) - # Write into config_db - cmds_disable.append("config save -y") - duthost.shell_cmds(cmds=cmds_disable) - - return disable_container_autorestart - - -@pytest.fixture(scope="module") -def enable_container_autorestart(): - def enable_container_autorestart(duthost, testcase="", feature_list=None): - ''' - @summary: Enable autorestart of the features present in feature_list. - - @param duthosts: Instance of DutHost - @param testcase: testcase name used to find corresponding file to restore autorestart state. - @feature_list: List of features to enable autorestart. If None, autorestart of all the features will - be disabled. - ''' - state_file_name = "/tmp/autorestart_state_{}_{}.json".format(duthost.hostname, testcase) - if not os.path.exists(state_file_name): - return - stored_autorestart_states = {} - with open(state_file_name, "r") as f: - stored_autorestart_states = json.load(f) - container_autorestart_states = duthost.get_container_autorestart_states() - # Recover autorestart states - logging.info("Recover container autorestart") - cmd_enable = "config feature autorestart {} enabled" - cmds_enable = [] - for name, state in list(container_autorestart_states.items()): - if state == "disabled" and (feature_list is None or name in feature_list) \ - and name in stored_autorestart_states \ - and stored_autorestart_states[name] == "enabled": - cmds_enable.append(cmd_enable.format(name)) - # Write into config_db - cmds_enable.append("config save -y") - duthost.shell_cmds(cmds=cmds_enable) - os.remove(state_file_name) - - return enable_container_autorestart - - -@pytest.fixture(scope='module') -def swapSyncd(request, duthosts, enum_rand_one_per_hwsku_frontend_hostname, creds, tbinfo, lower_tor_host): - """ - Swap syncd on DUT host - - Args: - request (Fixture): pytest request object - duthost (AnsibleHost): Device Under Test (DUT) - - Returns: - None - """ - if 'dualtor' in tbinfo['topo']['name']: - duthost = lower_tor_host - else: - duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - swapSyncd = request.config.getoption("--qos_swap_syncd") - public_docker_reg = request.config.getoption("--public_docker_registry") - try: - if swapSyncd: - if public_docker_reg: - new_creds = copy.deepcopy(creds) - new_creds['docker_registry_host'] = new_creds['public_docker_registry_host'] - new_creds['docker_registry_username'] = '' - new_creds['docker_registry_password'] = '' - else: - new_creds = creds - docker.swap_syncd(duthost, new_creds) - - yield - finally: - if swapSyncd: - docker.restore_default_syncd(duthost, new_creds) - - -def get_host_data(request, dut): - ''' - This function parses multple inventory files and returns the dut information present in the inventory - ''' - inv_files = get_inventory_files(request) - return get_host_vars(inv_files, dut) - - -def generate_params_frontend_hostname(request, macsec_only=False): - frontend_duts = [] - tbname, tbinfo = get_tbinfo(request) - duts = get_specified_duts(request) - inv_files = get_inventory_files(request) - host_type = "frontend" - - if macsec_only: - host_type = "macsec" - if 't2' in tbinfo['topo']['name'] and request.config.option.enable_macsec: - # currently in the T2 topo only the uplink linecard will have macsec enabled - # Please add "macsec_card = True" param to inventory the inventory file - # under Line Card with macsec capability. - for dut in duts: - if is_frontend_node(inv_files, dut) and is_macsec_capable_node(inv_files, dut): - frontend_duts.append(dut) - if not frontend_duts: - logging.info("no macsec card found") - else: - frontend_duts.append(duts[0]) - else: - for dut in duts: - if is_frontend_node(inv_files, dut): - frontend_duts.append(dut) - - assert len(frontend_duts) > 0, \ - "Test selected require at-least one {} node, " \ - "none of the DUTs '{}' in testbed '{}' are a {} node".format(host_type, duts, tbname, host_type) - return frontend_duts - - -def generate_params_hostname_rand_per_hwsku(request, frontend_only=False, macsec_only=False): - hosts = get_specified_duts(request) - if frontend_only: - hosts = generate_params_frontend_hostname(request, macsec_only=macsec_only) - - hosts_per_hwsku = get_hosts_per_hwsku(request, hosts) - return hosts_per_hwsku - - -def get_hosts_per_hwsku(request, hosts): - inv_files = get_inventory_files(request) - # Create a list of hosts per hwsku - host_hwskus = {} - for a_host in hosts: - host_vars = get_host_visible_vars(inv_files, a_host) - a_host_hwsku = None - if 'hwsku' in host_vars: - a_host_hwsku = host_vars['hwsku'] - else: - # Lets try 'sonic_hwsku' as well - if 'sonic_hwsku' in host_vars: - a_host_hwsku = host_vars['sonic_hwsku'] - if a_host_hwsku: - if a_host_hwsku not in host_hwskus: - host_hwskus[a_host_hwsku] = [a_host] - else: - host_hwskus[a_host_hwsku].append(a_host) - else: - pytest.fail("Test selected require a node per hwsku, but 'hwsku' for '{}' not defined in the inventory" - .format(a_host)) - - hosts_per_hwsku = [] - for hosts in list(host_hwskus.values()): - if len(hosts) == 1: - hosts_per_hwsku.append(hosts[0]) - else: - hosts_per_hwsku.extend(random.sample(hosts, 1)) - - return hosts_per_hwsku - - -def generate_params_supervisor_hostname(request): - duts = get_specified_duts(request) - if len(duts) == 1: - # We have a single node - dealing with pizza box, return it - return [duts[0]] - inv_files = get_inventory_files(request) - for dut in duts: - # Expecting only a single supervisor node - if is_supervisor_node(inv_files, dut): - return [dut] - # If there are no supervisor cards in a multi-dut tesbed, we are dealing with all pizza box in the testbed, - # pick the first DUT - return [duts[0]] - - -def generate_param_asic_index(request, dut_hostnames, param_type, random_asic=False): - _, tbinfo = get_tbinfo(request) - inv_files = get_inventory_files(request) - logging.info("generating {} asic indicies for DUT [{}] in ".format(param_type, dut_hostnames)) - - asic_index_params = [] - for dut in dut_hostnames: - inv_data = get_host_visible_vars(inv_files, dut) - # if the params are not present treat the device as a single asic device - dut_asic_params = [DEFAULT_ASIC_ID] - if inv_data: - if param_type == ASIC_PARAM_TYPE_ALL and ASIC_PARAM_TYPE_ALL in inv_data: - if int(inv_data[ASIC_PARAM_TYPE_ALL]) == 1: - dut_asic_params = [DEFAULT_ASIC_ID] - else: - if ASICS_PRESENT in inv_data: - dut_asic_params = inv_data[ASICS_PRESENT] - else: - dut_asic_params = list(range(int(inv_data[ASIC_PARAM_TYPE_ALL]))) - elif param_type == ASIC_PARAM_TYPE_FRONTEND and ASIC_PARAM_TYPE_FRONTEND in inv_data: - dut_asic_params = inv_data[ASIC_PARAM_TYPE_FRONTEND] - logging.info("dut name {} asics params = {}".format(dut, dut_asic_params)) - - if random_asic: - asic_index_params.append(random.sample(dut_asic_params, 1)) - else: - asic_index_params.append(dut_asic_params) - return asic_index_params - - -def generate_params_dut_index(request): - tbname, _ = get_tbinfo(request) - num_duts = len(get_specified_duts(request)) - logging.info("Using {} duts from testbed '{}'".format(num_duts, tbname)) - - return list(range(num_duts)) - - -def generate_params_dut_hostname(request): - tbname, _ = get_tbinfo(request) - duts = get_specified_duts(request) - logging.info("Using DUTs {} in testbed '{}'".format(str(duts), tbname)) - - return duts - - -def get_completeness_level_metadata(request): - completeness_level = request.config.getoption("--completeness_level") - # if completeness_level is not set or an unknown completeness_level is set - # return "thorough" to run all test set - if not completeness_level or completeness_level not in ["debug", "basic", "confident", "thorough"]: - return "debug" - return completeness_level - - -def get_testbed_metadata(request): - """ - Get the metadata for the testbed name. Return None if tbname is - not provided, or metadata file not found or metadata does not - contain tbname - """ - tbname = request.config.getoption("--testbed") - if not tbname: - return None - - folder = 'metadata' - filepath = os.path.join(folder, tbname + '.json') - metadata = None - - try: - with open(filepath, 'r') as yf: - metadata = json.load(yf) - except IOError: - return None - - return metadata.get(tbname) - - -def get_snappi_testbed_metadata(request): - """ - Get the metadata for the testbed name. Return None if tbname is - not provided, or metadata file not found or metadata does not - contain tbname - """ - tbname = request.config.getoption("--testbed") - if not tbname: - return None - - folder = 'metadata/snappi_tests' - filepath = os.path.join(folder, tbname + '.json') - metadata = None - - try: - with open(filepath, 'r') as yf: - metadata = json.load(yf) - except IOError: - return None - - return metadata.get(tbname) - - -def generate_port_lists(request, port_scope, with_completeness_level=False): - empty = [encode_dut_port_name('unknown', 'unknown')] - if 'ports' in port_scope: - scope = 'Ethernet' - elif 'pcs' in port_scope: - scope = 'PortChannel' - else: - return empty - - if 'all' in port_scope: - state = None - elif 'oper_up' in port_scope: - state = 'oper_state' - elif 'admin_up' in port_scope: - state = 'admin_state' - else: - return empty - - dut_ports = get_testbed_metadata(request) - - if dut_ports is None: - return empty - - dut_port_map = {} - for dut, val in list(dut_ports.items()): - dut_port_pairs = [] - if 'intf_status' not in val: - continue - for intf, status in list(val['intf_status'].items()): - if scope in intf and (not state or status[state] == 'up'): - dut_port_pairs.append(encode_dut_port_name(dut, intf)) - dut_port_map[dut] = dut_port_pairs - logger.info("Generate dut_port_map: {}".format(dut_port_map)) - - if with_completeness_level: - completeness_level = get_completeness_level_metadata(request) - # if completeness_level in ["debug", "basic", "confident"], - # only select several ports on every DUT to save test time - - def trim_dut_port_lists(dut_port_list, target_len): - if len(dut_port_list) <= target_len: - return dut_port_list - # for diversity, fetch the ports from both the start and the end of the original list - pos_1 = target_len // 2 - pos_2 = target_len - pos_1 - return dut_ports[:pos_1] + dut_ports[-pos_2:] - - if completeness_level in ["debug"]: - for dut, dut_ports in list(dut_port_map.items()): - dut_port_map[dut] = trim_dut_port_lists(dut_ports, 1) - elif completeness_level in ["basic", "confident"]: - for dut, dut_ports in list(dut_port_map.items()): - dut_port_map[dut] = trim_dut_port_lists(dut_ports, 4) - - ret = sum(list(dut_port_map.values()), []) - logger.info("Generate port_list: {}".format(ret)) - return ret if ret else empty - - -def generate_dut_feature_container_list(request): - """ - Generate list of containers given the list of features. - List of features and container names are both obtained from - metadata file - """ - empty = [encode_dut_and_container_name("unknown", "unknown")] - - meta = get_testbed_metadata(request) - - if meta is None: - return empty - - container_list = [] - - for dut, val in list(meta.items()): - if "features" not in val: - continue - for feature in list(val["features"].keys()): - if "disabled" in val["features"][feature]: - continue - - dut_info = meta[dut] - - if "asic_services" in dut_info and dut_info["asic_services"].get(feature) is not None: - for service in dut_info["asic_services"].get(feature): - container_list.append(encode_dut_and_container_name(dut, service)) - else: - container_list.append(encode_dut_and_container_name(dut, feature)) - - return container_list - - -def generate_dut_feature_list(request, duts_selected, asics_selected): - """ - Generate a list of features. - The list of features willl be obtained from - metadata file. - This list will be features that can be stopped - or restarted. - """ - meta = get_testbed_metadata(request) - tuple_list = [] - - if meta is None: - return tuple_list - - skip_feature_list = ['database', 'database-chassis', 'gbsyncd'] - - for a_dut_index, a_dut in enumerate(duts_selected): - if len(asics_selected): - for a_asic in asics_selected[a_dut_index]: - # Create tuple of dut and asic index - if "features" in meta[a_dut]: - for a_feature in list(meta[a_dut]["features"].keys()): - if a_feature not in skip_feature_list: - tuple_list.append((a_dut, a_asic, a_feature)) - else: - tuple_list.append((a_dut, a_asic, None)) - else: - if "features" in meta[a_dut]: - for a_feature in list(meta[a_dut]["features"].keys()): - if a_feature not in skip_feature_list: - tuple_list.append((a_dut, None, a_feature)) - else: - tuple_list.append((a_dut, None, None)) - return tuple_list - - -def generate_dut_backend_asics(request, duts_selected): - dut_asic_list = [] - - metadata = get_testbed_metadata(request) - - if metadata is None: - return [[None]]*len(duts_selected) - - for dut in duts_selected: - mdata = metadata.get(dut) - if mdata is None: - dut_asic_list.append([None]) - dut_asic_list.append(mdata.get("backend_asics", [None])) - - return dut_asic_list - - -def generate_priority_lists(request, prio_scope, with_completeness_level=False, one_dut_only=False): - empty = [] - - tbname = request.config.getoption("--testbed") - if not tbname: - return empty - - folder = 'priority' - filepath = os.path.join(folder, tbname + '-' + prio_scope + '.json') - - try: - with open(filepath, 'r') as yf: - info = json.load(yf) - except IOError: - return empty - - if tbname not in info: - return empty - - dut_prio = info[tbname] - ret = [] - - for dut, priorities in list(dut_prio.items()): - for p in priorities: - ret.append('{}|{}'.format(dut, p)) - - if one_dut_only: - break - - if with_completeness_level: - completeness_level = get_completeness_level_metadata(request) - # if completeness_level in ["debug", "basic", "confident"], - # select a small subnet to save test time - # if completeness_level in ["debug"], only select one item - # if completeness_level in ["basic", "confident"], select 1 priority per DUT - - if completeness_level in ["debug"] and ret: - ret = random.sample(ret, 1) - elif completeness_level in ["basic", "confident"]: - ret = [] - for dut, priorities in list(dut_prio.items()): - if priorities: - p = random.choice(priorities) - ret.append('{}|{}'.format(dut, p)) - - if one_dut_only: - break - - return ret if ret else empty - - -def pfc_pause_delay_test_params(request): - empty = [] - - tbname = request.config.getoption("--testbed") - if not tbname: - return empty - - folder = 'pfc_headroom_test_params' - filepath = os.path.join(folder, tbname + '.json') - - try: - with open(filepath, 'r') as yf: - info = json.load(yf) - except IOError: - return empty - - if tbname not in info: - return empty - - dut_pfc_delay_params = info[tbname] - ret = [] - - for dut, pfc_pause_delay_params in list(dut_pfc_delay_params.items()): - for pfc_delay, headroom_result in list(pfc_pause_delay_params.items()): - ret.append('{}|{}|{}'.format(dut, pfc_delay, headroom_result)) - - return ret if ret else empty - - -_frontend_hosts_per_hwsku_per_module = {} -_hosts_per_hwsku_per_module = {} -_rand_one_asic_per_module = {} -_rand_one_frontend_asic_per_module = {} -_macsec_frontend_hosts_per_hwsku_per_module = {} -def pytest_generate_tests(metafunc): # noqa: E302 - # The topology always has atleast 1 dut - dut_fixture_name = None - duts_selected = None - global _frontend_hosts_per_hwsku_per_module, _hosts_per_hwsku_per_module - global _macsec_frontend_hosts_per_hwsku_per_module - global _rand_one_asic_per_module, _rand_one_frontend_asic_per_module - # Enumerators for duts are mutually exclusive - target_hostname = get_target_hostname(metafunc) - if target_hostname: - duts_selected = [target_hostname] - if "enum_dut_hostname" in metafunc.fixturenames: - dut_fixture_name = "enum_dut_hostname" - elif "enum_supervisor_dut_hostname" in metafunc.fixturenames: - dut_fixture_name = "enum_supervisor_dut_hostname" - elif "enum_frontend_dut_hostname" in metafunc.fixturenames: - dut_fixture_name = "enum_frontend_dut_hostname" - elif "enum_rand_one_per_hwsku_hostname" in metafunc.fixturenames: - if metafunc.module not in _hosts_per_hwsku_per_module: - _hosts_per_hwsku_per_module[metafunc.module] = duts_selected - - dut_fixture_name = "enum_rand_one_per_hwsku_hostname" - elif "enum_rand_one_per_hwsku_frontend_hostname" in metafunc.fixturenames: - if metafunc.module not in _frontend_hosts_per_hwsku_per_module: - _frontend_hosts_per_hwsku_per_module[metafunc.module] = duts_selected - - dut_fixture_name = "enum_rand_one_per_hwsku_frontend_hostname" - elif "enum_rand_one_per_hwsku_macsec_frontend_hostname" in metafunc.fixturenames: - if metafunc.module not in _macsec_frontend_hosts_per_hwsku_per_module: - _macsec_frontend_hosts_per_hwsku_per_module[metafunc.module] = duts_selected - dut_fixture_name = "enum_rand_one_per_hwsku_macsec_frontend_hostname" - else: - if "enum_dut_hostname" in metafunc.fixturenames: - duts_selected = generate_params_dut_hostname(metafunc) - dut_fixture_name = "enum_dut_hostname" - elif "enum_supervisor_dut_hostname" in metafunc.fixturenames: - duts_selected = generate_params_supervisor_hostname(metafunc) - dut_fixture_name = "enum_supervisor_dut_hostname" - elif "enum_frontend_dut_hostname" in metafunc.fixturenames: - duts_selected = generate_params_frontend_hostname(metafunc) - dut_fixture_name = "enum_frontend_dut_hostname" - elif "enum_rand_one_per_hwsku_hostname" in metafunc.fixturenames: - if metafunc.module not in _hosts_per_hwsku_per_module: - hosts_per_hwsku = generate_params_hostname_rand_per_hwsku(metafunc) - _hosts_per_hwsku_per_module[metafunc.module] = hosts_per_hwsku - duts_selected = _hosts_per_hwsku_per_module[metafunc.module] - dut_fixture_name = "enum_rand_one_per_hwsku_hostname" - elif "enum_rand_one_per_hwsku_frontend_hostname" in metafunc.fixturenames: - if metafunc.module not in _frontend_hosts_per_hwsku_per_module: - hosts_per_hwsku = generate_params_hostname_rand_per_hwsku(metafunc, frontend_only=True) - _frontend_hosts_per_hwsku_per_module[metafunc.module] = hosts_per_hwsku - duts_selected = _frontend_hosts_per_hwsku_per_module[metafunc.module] - dut_fixture_name = "enum_rand_one_per_hwsku_frontend_hostname" - elif "enum_rand_one_per_hwsku_macsec_frontend_hostname" in metafunc.fixturenames: - if metafunc.module not in _macsec_frontend_hosts_per_hwsku_per_module: - hosts_per_hwsku = generate_params_hostname_rand_per_hwsku( - metafunc, frontend_only=True, macsec_only=True - ) - _macsec_frontend_hosts_per_hwsku_per_module[metafunc.module] = hosts_per_hwsku - duts_selected = _macsec_frontend_hosts_per_hwsku_per_module[metafunc.module] - dut_fixture_name = "enum_rand_one_per_hwsku_macsec_frontend_hostname" - - asics_selected = None - asic_fixture_name = None - - tbname, tbinfo = get_tbinfo(metafunc) - if duts_selected is None: - duts_selected = [tbinfo["duts"][0]] - - possible_asic_enums = ["enum_asic_index", "enum_frontend_asic_index", "enum_backend_asic_index", - "enum_rand_one_asic_index", "enum_rand_one_frontend_asic_index"] - enums_asic_fixtures = set(metafunc.fixturenames).intersection(possible_asic_enums) - assert len(enums_asic_fixtures) < 2, \ - "The number of asic_enum fixtures should be 1 or zero, " \ - "the following fixtures conflict one with each other: {}".format(str(enums_asic_fixtures)) - - if "enum_asic_index" in metafunc.fixturenames: - asic_fixture_name = "enum_asic_index" - asics_selected = generate_param_asic_index(metafunc, duts_selected, ASIC_PARAM_TYPE_ALL) - elif "enum_frontend_asic_index" in metafunc.fixturenames: - asic_fixture_name = "enum_frontend_asic_index" - asics_selected = generate_param_asic_index(metafunc, duts_selected, ASIC_PARAM_TYPE_FRONTEND) - elif "enum_backend_asic_index" in metafunc.fixturenames: - asic_fixture_name = "enum_backend_asic_index" - asics_selected = generate_dut_backend_asics(metafunc, duts_selected) - elif "enum_rand_one_asic_index" in metafunc.fixturenames: - asic_fixture_name = "enum_rand_one_asic_index" - if metafunc.module not in _rand_one_asic_per_module: - asics_selected = generate_param_asic_index(metafunc, duts_selected, - ASIC_PARAM_TYPE_ALL, random_asic=True) - _rand_one_asic_per_module[metafunc.module] = asics_selected - asics_selected = _rand_one_asic_per_module[metafunc.module] - elif "enum_rand_one_frontend_asic_index" in metafunc.fixturenames: - asic_fixture_name = "enum_rand_one_frontend_asic_index" - if metafunc.module not in _rand_one_frontend_asic_per_module: - asics_selected = generate_param_asic_index(metafunc, duts_selected, - ASIC_PARAM_TYPE_FRONTEND, random_asic=True) - _rand_one_frontend_asic_per_module[metafunc.module] = asics_selected - asics_selected = _rand_one_frontend_asic_per_module[metafunc.module] - - # Create parameterization tuple of dut_fixture_name, asic_fixture_name and feature to parameterize - if dut_fixture_name and asic_fixture_name and ("enum_dut_feature" in metafunc.fixturenames): - tuple_list = generate_dut_feature_list(metafunc, duts_selected, asics_selected) - feature_fixture = "enum_dut_feature" - metafunc.parametrize(dut_fixture_name + "," + asic_fixture_name + "," + feature_fixture, - tuple_list, scope="module", indirect=True) - # Create parameterization tuple of dut_fixture_name and asic_fixture_name to parameterize - elif dut_fixture_name and asic_fixture_name: - # parameterize on both - create tuple for each - tuple_list = [] - for a_dut_index, a_dut in enumerate(duts_selected): - if len(asics_selected): - for a_asic in asics_selected[a_dut_index]: - # Create tuple of dut and asic index - tuple_list.append((a_dut, a_asic)) - else: - tuple_list.append((a_dut, None)) - metafunc.parametrize(dut_fixture_name + "," + asic_fixture_name, tuple_list, scope="module", indirect=True) - elif dut_fixture_name: - # parameterize only on DUT - metafunc.parametrize(dut_fixture_name, duts_selected, scope="module", indirect=True) - elif asic_fixture_name: - # We have no duts selected, so need asic list for the first DUT - if len(asics_selected): - metafunc.parametrize(asic_fixture_name, asics_selected[0], scope="module", indirect=True) - else: - metafunc.parametrize(asic_fixture_name, [None], scope="module", indirect=True) - - # When selected_dut used and select a dut for test, parameterize dut for enable TACACS on all UT - if dut_fixture_name and "selected_dut" in metafunc.fixturenames: - metafunc.parametrize("selected_dut", duts_selected, scope="module", indirect=True) - - if "enum_dut_portname" in metafunc.fixturenames: - metafunc.parametrize("enum_dut_portname", generate_port_lists(metafunc, "all_ports")) - - def format_portautoneg_test_id(param): - speeds = param['speeds'] if 'speeds' in param else [param['speed']] - return "{}|{}|{}".format(param['dutname'], param['port'], ','.join(speeds)) - - if "enum_dut_portname_module_fixture" in metafunc.fixturenames or \ - "enum_speed_per_dutport_fixture" in metafunc.fixturenames: - autoneg_tests_data = get_autoneg_tests_data() - if "enum_dut_portname_module_fixture" in metafunc.fixturenames: - metafunc.parametrize( - "enum_dut_portname_module_fixture", - autoneg_tests_data, - scope="module", - ids=format_portautoneg_test_id, - indirect=True - ) - - if "enum_speed_per_dutport_fixture" in metafunc.fixturenames: - metafunc.parametrize( - "enum_speed_per_dutport_fixture", - parametrise_per_supported_port_speed(autoneg_tests_data), - scope="module", - ids=format_portautoneg_test_id, - indirect=True - ) - - if "enum_dut_portname_oper_up" in metafunc.fixturenames: - metafunc.parametrize("enum_dut_portname_oper_up", generate_port_lists(metafunc, "oper_up_ports")) - if "enum_dut_portname_admin_up" in metafunc.fixturenames: - metafunc.parametrize("enum_dut_portname_admin_up", generate_port_lists(metafunc, "admin_up_ports")) - if "enum_dut_portchannel" in metafunc.fixturenames: - metafunc.parametrize("enum_dut_portchannel", generate_port_lists(metafunc, "all_pcs")) - if "enum_dut_portchannel_oper_up" in metafunc.fixturenames: - metafunc.parametrize("enum_dut_portchannel_oper_up", generate_port_lists(metafunc, "oper_up_pcs")) - if "enum_dut_portchannel_admin_up" in metafunc.fixturenames: - metafunc.parametrize("enum_dut_portchannel_admin_up", generate_port_lists(metafunc, "admin_up_pcs")) - if "enum_dut_portchannel_with_completeness_level" in metafunc.fixturenames: - metafunc.parametrize("enum_dut_portchannel_with_completeness_level", - generate_port_lists(metafunc, "all_pcs", with_completeness_level=True)) - if "enum_dut_feature_container" in metafunc.fixturenames: - metafunc.parametrize( - "enum_dut_feature_container", generate_dut_feature_container_list(metafunc) + def test_mac_remote_fault_increment(self, get_dut_and_supported_available_optical_interfaces, collected_ports_num): + dut, supported_available_optical_interfaces, failed_api_ports = ( + get_dut_and_supported_available_optical_interfaces() ) - if 'enum_dut_all_prio' in metafunc.fixturenames: - metafunc.parametrize("enum_dut_all_prio", generate_priority_lists(metafunc, 'all')) - if 'enum_dut_lossless_prio' in metafunc.fixturenames: - metafunc.parametrize("enum_dut_lossless_prio", generate_priority_lists(metafunc, 'lossless')) - if 'enum_one_dut_lossless_prio' in metafunc.fixturenames: - metafunc.parametrize("enum_one_dut_lossless_prio", - generate_priority_lists(metafunc, 'lossless', one_dut_only=True)) - if 'enum_dut_lossless_prio_with_completeness_level' in metafunc.fixturenames: - metafunc.parametrize("enum_dut_lossless_prio_with_completeness_level", - generate_priority_lists(metafunc, 'lossless', with_completeness_level=True)) - if 'enum_one_dut_lossless_prio_with_completeness_level' in metafunc.fixturenames: - metafunc.parametrize("enum_one_dut_lossless_prio_with_completeness_level", - generate_priority_lists(metafunc, 'lossless', with_completeness_level=True, - one_dut_only=True)) - if 'enum_dut_lossy_prio' in metafunc.fixturenames: - metafunc.parametrize("enum_dut_lossy_prio", generate_priority_lists(metafunc, 'lossy')) - if 'enum_one_dut_lossy_prio' in metafunc.fixturenames: - metafunc.parametrize("enum_one_dut_lossy_prio", - generate_priority_lists(metafunc, 'lossy', one_dut_only=True)) - if 'enum_dut_lossy_prio_with_completeness_level' in metafunc.fixturenames: - metafunc.parametrize("enum_dut_lossy_prio_with_completeness_level", - generate_priority_lists(metafunc, 'lossy', with_completeness_level=True)) - if 'enum_one_dut_lossy_prio_with_completeness_level' in metafunc.fixturenames: - metafunc.parametrize("enum_one_dut_lossy_prio_with_completeness_level", - generate_priority_lists(metafunc, 'lossy', with_completeness_level=True, - one_dut_only=True)) - if 'enum_pfc_pause_delay_test_params' in metafunc.fixturenames: - metafunc.parametrize("enum_pfc_pause_delay_test_params", pfc_pause_delay_test_params(metafunc)) - - if 'topo_scenario' in metafunc.fixturenames: - if tbinfo['topo']['type'] == 'm0' and 'topo_scenario' in metafunc.fixturenames: - metafunc.parametrize('topo_scenario', ['m0_vlan_scenario', 'm0_l3_scenario'], scope='module') - else: - metafunc.parametrize('topo_scenario', ['default'], scope='module') - - if 'tgen_port_info' in metafunc.fixturenames: - metafunc.parametrize('tgen_port_info', generate_skeleton_port_info(metafunc), indirect=True) - - if 'vlan_name' in metafunc.fixturenames: - if tbinfo['topo']['type'] == 'm0' and 'topo_scenario' in metafunc.fixturenames: - if tbinfo['topo']['name'] == 'm0-2vlan': - metafunc.parametrize('vlan_name', ['Vlan1000', 'Vlan2000'], scope='module') - else: - metafunc.parametrize('vlan_name', ['Vlan1000'], scope='module') - # Non M0 topo - else: - try: - if tbinfo["topo"]["type"] in ["t0", "mx"]: - default_vlan_config = tbinfo["topo"]["properties"]["topology"][ - "DUT" - ]["vlan_configs"]["default_vlan_config"] - if default_vlan_config == "two_vlan_a": - logger.info("default_vlan_config is two_vlan_a") - vlan_list = list( - tbinfo["topo"]["properties"]["topology"]["DUT"][ - "vlan_configs" - ]["two_vlan_a"].keys() - ) - elif default_vlan_config == "one_vlan_a": - logger.info("default_vlan_config is one_vlan_a") - vlan_list = list( - tbinfo["topo"]["properties"]["topology"]["DUT"][ - "vlan_configs" - ]["one_vlan_a"].keys() - ) - else: - vlan_list = ["Vlan1000"] - logger.info("parametrize vlan_name: {}".format(vlan_list)) - metafunc.parametrize("vlan_name", vlan_list, scope="module") - else: - metafunc.parametrize("vlan_name", ["no_vlan"], scope="module") - except KeyError: - logger.error("topo {} keys are missing in the tbinfo={}".format(tbinfo['topo']['name'], tbinfo)) - if tbinfo['topo']['type'] in ['t0', 'mx']: - metafunc.parametrize('vlan_name', ['Vlan1000'], scope='module') - else: - metafunc.parametrize('vlan_name', ['no_vlan'], scope='module') - - -@lru_cache -def parse_override(testbed, field): - is_dynamic_only = "--enable-snappi-dynamic-ports" in sys.argv - - if is_dynamic_only and field != "pfcQueueGroupSize": - # Args "--enable-snappi-dynamic-ports" should not affect field `pfcQueueGroupSize` - return False, None - - override_file = "snappi_tests/variables.override.yml" - - with open(override_file, 'r') as f: - all_values = yaml.safe_load(f) - if testbed not in all_values or field not in all_values[testbed]: - return False, None - - return True, all_values[testbed][field] - - return False, None - - -def generate_skeleton_port_info(request): - """ - Return minimal port_info parameters to populate later in the format of -. i.e - - ["400.0-single_linecard_single_asic", "400.0-multiple_linecard_multiple_asic",...] - """ - is_override, override_data = parse_override( - request.config.getoption("--testbed"), - 'multidut_port_info' - ) - - if is_override: - return override_data - - dut_info = get_snappi_testbed_metadata(request) or [] - available_interfaces = {} - matrix = {} - for index, linecard in enumerate(dut_info): - interface_to_asic = {} - for asic in dut_info[linecard]["asic_to_interface"]: - for interface in dut_info[linecard]["asic_to_interface"][asic]: - interface_to_asic[interface] = asic - - available_interfaces[linecard] = [dut_info[linecard]['intf_status'][interface] - for interface in dut_info[linecard]['intf_status'] - if dut_info[linecard]['intf_status'][interface]["admin_state"] == "up"] - - for interface in available_interfaces[linecard]: - for key, value in dut_info[linecard]["asic_to_interface"].items(): - if interface['name'] in value: - interface['asic'] = key - - for interface in available_interfaces[linecard]: - speed = float(re.match(r"([\d.]+)", interface['speed']).group(0)) - asic = interface['asic'] - if (speed not in matrix): - matrix[speed] = {} - if (linecard not in matrix[speed]): - matrix[speed][linecard] = {} - if (asic not in matrix[speed][linecard]): - matrix[speed][linecard][asic] = 1 - else: - matrix[speed][linecard][asic] += 1 - - def build_params(speed, category): - return f"{speed}-{category}" - - flattened_list = set() - - for speed, linecards in matrix.items(): - if len(linecards) >= 2: - flattened_list.add(build_params(speed, 'multiple_linecard_multiple_asic')) - - for linecard, asic_list in linecards.items(): - if len(asic_list) >= 2: - flattened_list.add(build_params(speed, 'single_linecard_multiple_asic')) - - for asics, port_count in asic_list.items(): - if int(port_count) >= 2: - flattened_list.add(build_params(speed, 'single_linecard_single_asic')) - - return list(flattened_list) - - -def get_autoneg_tests_data(): - folder = 'metadata' - filepath = os.path.join(folder, 'autoneg-test-params.json') - if not os.path.exists(filepath): - logger.warning('Autoneg tests datafile is missing: {}. " \ - "Run test_pretest -k test_update_testbed_metadata to create it'.format(filepath)) - return [{'dutname': 'unknown', 'port': 'unknown', 'speeds': ['unknown']}] - data = {} - with open(filepath) as yf: - data = json.load(yf) - - return [ - {'dutname': dutname, 'port': dutport, 'speeds': portinfo['common_port_speeds']} - for dutname, ports in list(data.items()) - for dutport, portinfo in list(ports.items()) - ] - - -def parametrise_per_supported_port_speed(data): - return [ - {'dutname': conn_info['dutname'], 'port': conn_info['port'], 'speed': speed} - for conn_info in data for speed in conn_info['speeds'] - ] - - -# Override enum fixtures for duts and asics to ensure that parametrization happens once per module. -@pytest.fixture(scope="module") -def enum_dut_hostname(request): - return request.param - - -@pytest.fixture(scope="module") -def enum_supervisor_dut_hostname(request): - return request.param - - -@pytest.fixture(scope="module") -def enum_frontend_dut_hostname(request): - return request.param - - -@pytest.fixture(scope="module") -def selected_dut(request): - try: - logger.debug("selected_dut host: {}".format(request.param)) - return request.param - except AttributeError: - return None - - -@pytest.fixture(scope="module") -def enum_rand_one_per_hwsku_hostname(request): - return request.param - - -@pytest.fixture(scope="module") -def enum_rand_one_per_hwsku_frontend_hostname(request): - return request.param - - -@pytest.fixture(scope="module") -def enum_rand_one_per_hwsku_macsec_frontend_hostname(request): - return request.param - - -@pytest.fixture(scope="module") -def enum_asic_index(request): - return request.param - - -@pytest.fixture(scope="module") -def enum_frontend_asic_index(request): - return request.param - - -@pytest.fixture(scope="module") -def enum_backend_asic_index(request): - return request.param - - -@pytest.fixture(scope="module") -def enum_rand_one_asic_index(request): - return request.param - - -@pytest.fixture(scope="module") -def enum_dut_feature(request): - return request.param - - -@pytest.fixture(scope="module") -def enum_rand_one_frontend_asic_index(request): - return request.param - - -@pytest.fixture(scope='module') -def enum_upstream_dut_hostname(duthosts, tbinfo): - upstream_nbr_type = get_upstream_neigh_type(tbinfo, is_upper=True) - if upstream_nbr_type is None: - upstream_nbr_type = "T3" - - for a_dut in duthosts.frontend_nodes: - minigraph_facts = a_dut.get_extended_minigraph_facts(tbinfo) - minigraph_neighbors = minigraph_facts['minigraph_neighbors'] - for key, value in minigraph_neighbors.items(): - if upstream_nbr_type in value['name']: - return a_dut.hostname - - pytest.fail("Did not find a dut in duthosts that for topo type {} that has upstream nbr type {}". - format(tbinfo["topo"]["type"], upstream_nbr_type)) - - -@pytest.fixture(scope="module") -def duthost_console(duthosts, enum_supervisor_dut_hostname, localhost, conn_graph_facts, creds): # noqa: F811 - duthost = duthosts[enum_supervisor_dut_hostname] - host = create_duthost_console(duthost, localhost, conn_graph_facts, creds) - - yield host - host.disconnect() - - -@pytest.fixture(scope='session') -def cleanup_cache_for_session(request): - """ - This fixture allows developers to cleanup the cached data for all DUTs in the testbed before test. - Use cases: - - Running tests where some 'facts' about the DUT that get cached are changed. - - Running tests/regression without running test_pretest which has a test to clean up cache (PR#2978) - - Test case development phase to work out testbed information changes. - - This fixture is not automatically applied, if you want to use it, you have to add a call to it in your tests. - """ - tbname, tbinfo = get_tbinfo(request) - inv_files = get_inventory_files(request) - cache.cleanup(zone=tbname) - for a_dut in tbinfo['duts']: - cache.cleanup(zone=a_dut) - inv_data = get_host_visible_vars(inv_files, a_dut) - if 'num_asics' in inv_data and inv_data['num_asics'] > 1: - for asic_id in range(0, inv_data['num_asics']): - cache.cleanup(zone="{}-asic{}".format(a_dut, asic_id)) - - -def get_l2_info(dut): - """ - Helper function for l2 mode fixture - """ - config_facts = dut.get_running_config_facts() - mgmt_intf_table = config_facts['MGMT_INTERFACE'] - metadata_table = config_facts['DEVICE_METADATA']['localhost'] - mgmt_ip = None - for ip in list(mgmt_intf_table['eth0'].keys()): - if type(ip_interface(ip)) is IPv4Interface: - mgmt_ip = ip - mgmt_gw = mgmt_intf_table['eth0'][mgmt_ip]['gwaddr'] - hwsku = metadata_table['hwsku'] - - return mgmt_ip, mgmt_gw, hwsku - - -@pytest.fixture(scope='session') -def enable_l2_mode(duthosts, tbinfo, backup_and_restore_config_db_session): # noqa: F811 - """ - Configures L2 switch mode according to - https://github.com/sonic-net/SONiC/wiki/L2-Switch-mode - - Currently not compatible with version 201811 - - This fixture does not auto-cleanup after itself - A manual config reload is required to restore regular state - """ - base_config_db_cmd = 'echo \'{}\' | config reload /dev/stdin -y' - l2_preset_cmd = 'sonic-cfggen --preset l2 -p -H -k {} -a \'{}\' | config load /dev/stdin -y' - is_dualtor = 'dualtor' in tbinfo['topo']['name'] - - for dut in duthosts: - logger.info("Setting L2 mode on {}".format(dut)) - cmds = [] - mgmt_ip, mgmt_gw, hwsku = get_l2_info(dut) - # step 1 - base_config_db = { - "MGMT_INTERFACE": { - "eth0|{}".format(mgmt_ip): { - "gwaddr": "{}".format(mgmt_gw) - } - }, - "DEVICE_METADATA": { - "localhost": { - "hostname": "sonic" - } - } - } - - if is_dualtor: - base_config_db["DEVICE_METADATA"]["localhost"]["subtype"] = "DualToR" - cmds.append(base_config_db_cmd.format(json.dumps(base_config_db))) - - # step 2 - cmds.append('sonic-cfggen -H --write-to-db') - - # step 3 is optional and skipped here - # step 4 - if is_dualtor: - mg_facts = dut.get_extended_minigraph_facts(tbinfo) - all_ports = list(mg_facts['minigraph_ports'].keys()) - downlinks = [] - for vlan_info in list(mg_facts['minigraph_vlans'].values()): - downlinks.extend(vlan_info['members']) - uplinks = [intf for intf in all_ports if intf not in downlinks] - extra_args = { - 'is_dualtor': 'true', - 'uplinks': uplinks, - 'downlinks': downlinks - } - else: - extra_args = {} - cmds.append(l2_preset_cmd.format(hwsku, json.dumps(extra_args))) - - # extra step needed to render the feature table correctly - if is_dualtor: - cmds.append('while [ $(show feature config mux | awk \'{print $2}\' | tail -n 1) != "enabled" ]; ' - 'do sleep 1; done') - - # step 5 - cmds.append('config save -y') - - # step 6 - cmds.append('config reload -y') - - logger.debug("Commands to be run:\n{}".format(cmds)) - - dut.shell_cmds(cmds=cmds) - - -@pytest.fixture(scope='session') -def duts_running_config_facts(duthosts): - """Return running config facts for all multi-ASIC DUT hosts - - Args: - duthosts (DutHosts): Instance of DutHosts for interacting with DUT hosts. - - Returns: - dict: { - : [ - (asic0_idx, {asic0_cfg_facts}), - (asic1_idx, {asic1_cfg_facts}) - ] - } - """ - cfg_facts = {} - for duthost in duthosts: - cfg_facts[duthost.hostname] = [] - for asic in duthost.asics: - if asic.is_it_backend(): - continue - asic_cfg_facts = asic.config_facts(source='running')['ansible_facts'] - cfg_facts[duthost.hostname].append((asic.asic_index, asic_cfg_facts)) - return cfg_facts - - -@pytest.fixture(scope='class') -def dut_test_params_qos(duthosts, tbinfo, ptfhost, get_src_dst_asic_and_duts, lower_tor_host, creds, - mux_server_url, mux_status_from_nic_simulator, duts_running_config_facts, duts_minigraph_facts): - if 'dualtor' in tbinfo['topo']['name']: - all_duts = [lower_tor_host] - else: - all_duts = get_src_dst_asic_and_duts['all_duts'] - - src_asic = get_src_dst_asic_and_duts['src_asic'] - dst_asic = get_src_dst_asic_and_duts['dst_asic'] - - src_dut = get_src_dst_asic_and_duts['src_dut'] - src_dut_ip = src_dut.host.options['inventory_manager'].get_host(src_dut.hostname).vars['ansible_host'] - src_server = "{}:{}".format(src_dut_ip, src_asic.get_rpc_port_ssh_tunnel()) - - duthost = all_duts[0] - mgFacts = duthost.get_extended_minigraph_facts(tbinfo) - topo = tbinfo["topo"]["name"] - - rtn_dict = { - "topo": topo, - "hwsku": mgFacts["minigraph_hwsku"], - "basicParams": { - "router_mac": duthost.facts["router_mac"], - "src_server": src_server, - "port_map_file": ptf_test_port_map_active_active( - ptfhost, tbinfo, duthosts, mux_server_url, - duts_running_config_facts, duts_minigraph_facts, - mux_status_from_nic_simulator()), - "sonic_asic_type": duthost.facts['asic_type'], - "sonic_version": duthost.os_version, - "src_dut_index": get_src_dst_asic_and_duts['src_dut_index'], - "src_asic_index": get_src_dst_asic_and_duts['src_asic_index'], - "dst_dut_index": get_src_dst_asic_and_duts['dst_dut_index'], - "dst_asic_index": get_src_dst_asic_and_duts['dst_asic_index'], - "dut_username": creds['sonicadmin_user'], - "dut_password": creds['sonicadmin_password'] - }, - - } - - # Add dst server info if src and dst asic are different - if src_asic != dst_asic: - dst_dut = get_src_dst_asic_and_duts['dst_dut'] - dst_dut_ip = dst_dut.host.options['inventory_manager'].get_host(dst_dut.hostname).vars['ansible_host'] - rtn_dict["basicParams"]["dst_server"] = "{}:{}".format(dst_dut_ip, dst_asic.get_rpc_port_ssh_tunnel()) - - if 'platform_asic' in duthost.facts: - rtn_dict['basicParams']["platform_asic"] = duthost.facts['platform_asic'] - - yield rtn_dict - - -@pytest.fixture(scope='class') -def dut_test_params(duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinfo, - ptf_portmap_file, lower_tor_host, creds): # noqa: F811 - """ - Prepares DUT host test params - - Args: - duthost (AnsibleHost): Device Under Test (DUT) - tbinfo (Fixture, dict): Map containing testbed information - ptfPortMapFile (Fxiture, str): filename residing - on PTF host and contains port maps information - - Returns: - dut_test_params (dict): DUT host test params - """ - if 'dualtor' in tbinfo['topo']['name']: - duthost = lower_tor_host - else: - duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - mgFacts = duthost.get_extended_minigraph_facts(tbinfo) - topo = tbinfo["topo"]["name"] - - rtn_dict = { - "topo": topo, - "hwsku": mgFacts["minigraph_hwsku"], - "basicParams": { - "router_mac": duthost.facts["router_mac"], - "server": duthost.host.options['inventory_manager'].get_host( - duthost.hostname - ).vars['ansible_host'], - "port_map_file": ptf_portmap_file, - "sonic_asic_type": duthost.facts['asic_type'], - "sonic_version": duthost.os_version, - "dut_username": creds['sonicadmin_user'], - "dut_password": creds['sonicadmin_password'] - } - } - if 'platform_asic' in duthost.facts: - rtn_dict['basicParams']["platform_asic"] = duthost.facts['platform_asic'] - - yield rtn_dict - - -@pytest.fixture(scope='module') -def duts_minigraph_facts(duthosts, tbinfo): - """Return minigraph facts for all DUT hosts - - Args: - duthosts (DutHosts): Instance of DutHosts for interacting with DUT hosts. - tbinfo (object): Instance of TestbedInfo. - - Returns: - dict: { - : [ - (asic0_idx, {asic0_mg_facts}), - (asic1_idx, {asic1_mg_facts}) - ] - } - """ - mg_facts = {} - for duthost in duthosts: - mg_facts[duthost.hostname] = [] - for asic in duthost.asics: - if asic.is_it_backend(): - continue - asic_mg_facts = asic.get_extended_minigraph_facts(tbinfo) - mg_facts[duthost.hostname].append((asic.asic_index, asic_mg_facts)) - - return mg_facts - - -@pytest.fixture(scope="module", autouse=True) -def get_reboot_cause(duthost): - uptime_start = duthost.get_up_time() - yield - uptime_end = duthost.get_up_time() - if not uptime_end == uptime_start: - if "201811" in duthost.os_version or "201911" in duthost.os_version: - duthost.show_and_parse("show reboot-cause") - else: - duthost.show_and_parse("show reboot-cause history") - - -def collect_db_dump_on_duts(request, duthosts): - '''When test failed, this fixture will dump all the DBs on DUT and collect them to local - ''' - if hasattr(request.node, 'rep_call') and request.node.rep_call.failed: - dut_file_path = "/tmp/db_dump" - local_file_path = "./logs/db_dump" - - # Remove characters that can't be used in filename - nodename = safe_filename(request.node.nodeid) - db_dump_path = os.path.join(dut_file_path, nodename) - db_dump_tarfile = os.path.join(dut_file_path, "{}.tar.gz".format(nodename)) - - # We don't need to collect all DBs, db_names specify the DBs we want to collect - db_names = ["APPL_DB", "ASIC_DB", "COUNTERS_DB", "CONFIG_DB", "STATE_DB"] - raw_db_config = duthosts[0].shell("cat /var/run/redis/sonic-db/database_config.json")["stdout"] - db_config = json.loads(raw_db_config).get("DATABASES", {}) - db_ids = set() - for db_name in db_names: - # Skip STATE_DB dump on release 201911. - # JINJA2_CACHE can't be dumped by "redis-dump", and it is stored in STATE_DB on 201911 release. - # Please refer to issue: https://github.com/sonic-net/sonic-buildimage/issues/5587. - # The issue has been fixed in https://github.com/sonic-net/sonic-buildimage/pull/5646. - # However, the fix is not included in 201911 release. So we have to skip STATE_DB on release 201911 - # to avoid raising exception when dumping the STATE_DB. - if db_name == "STATE_DB" and duthosts[0].sonic_release in ['201911']: - continue - - if db_name in db_config: - db_ids.add(db_config[db_name].get("id", 0)) - - namespace_list = duthosts[0].get_asic_namespace_list() if duthosts[0].is_multi_asic else [] - if namespace_list: - for namespace in namespace_list: - # Collect DB dump - dump_dest_path = os.path.join(db_dump_path, namespace) - dump_cmds = ["mkdir -p {}".format(dump_dest_path)] - for db_id in db_ids: - dump_cmd = "ip netns exec {} redis-dump -d {} -y -o {}/{}" \ - .format(namespace, db_id, dump_dest_path, db_id) - dump_cmds.append(dump_cmd) - duthosts.shell_cmds(cmds=dump_cmds) - else: - # Collect DB dump - dump_dest_path = db_dump_path - dump_cmds = ["mkdir -p {}".format(dump_dest_path)] - for db_id in db_ids: - dump_cmd = "redis-dump -d {} -y -o {}/{}".format(db_id, dump_dest_path, db_id) - dump_cmds.append(dump_cmd) - duthosts.shell_cmds(cmds=dump_cmds) - - # compress dump file and fetch to docker - duthosts.shell("tar -czf {} -C {} {}".format(db_dump_tarfile, dut_file_path, nodename)) - duthosts.fetch(src=db_dump_tarfile, dest=local_file_path) - - # remove dump file from dut - duthosts.shell("rm -fr {} {}".format(db_dump_tarfile, db_dump_path)) - - -@pytest.fixture(autouse=True) -def collect_db_dump(request, duthosts): - """This autoused fixture is to generate DB dumps on DUT and collect them to local for later troubleshooting when - a test case failed. - """ - yield - if request.config.getoption("--collect_db_data"): - collect_db_dump_on_duts(request, duthosts) - - -def restore_config_db_and_config_reload(duts_data, duthosts, request): - # First copy the pre_running_config to the config_db.json files - for duthost in duthosts: - logger.info("dut reload called on {}".format(duthost.hostname)) - duthost.copy(content=json.dumps(duts_data[duthost.hostname]["pre_running_config"][None], indent=4), - dest='/etc/sonic/config_db.json', verbose=False) - - if duthost.is_multi_asic: - for asic_index in range(0, duthost.facts.get('num_asic')): - asic_ns = "asic{}".format(asic_index) - asic_cfg_file = "/tmp/{}_config_db{}.json".format(duthost.hostname, asic_index) - with open(asic_cfg_file, "w") as outfile: - outfile.write(json.dumps(duts_data[duthost.hostname]['pre_running_config'][asic_ns], indent=4)) - duthost.copy(src=asic_cfg_file, dest='/etc/sonic/config_db{}.json'.format(asic_index), verbose=False) - os.remove(asic_cfg_file) - - wait_for_bgp = False if request.config.getoption("skip_sanity") else True - - # Second execute config reload on all duthosts - with SafeThreadPoolExecutor(max_workers=8) as executor: - for duthost in duthosts: - executor.submit(config_reload, duthost, wait_before_force_reload=300, safe_reload=True, - check_intf_up_ports=True, wait_for_bgp=wait_for_bgp) - - -def compare_running_config(pre_running_config, cur_running_config): - if type(pre_running_config) != type(cur_running_config): - return False - if pre_running_config == cur_running_config: - return True - else: - if type(pre_running_config) is dict: - if set(pre_running_config.keys()) != set(cur_running_config.keys()): - return False - for key in pre_running_config.keys(): - if not compare_running_config(pre_running_config[key], cur_running_config[key]): - return False - return True - # We only have string in list in running config now, so we can ignore the order of the list. - elif type(pre_running_config) is list: - if set(pre_running_config) != set(cur_running_config): - return False - else: - return True - else: - return False - - -@pytest.fixture(scope="module", autouse=True) -def core_dump_and_config_check(duthosts, tbinfo, parallel_run_context, request, - # make sure the tear down of sanity_check happened after core_dump_and_config_check - sanity_check): - ''' - Check if there are new core dump files and if the running config is modified after the test case running. - If so, we will reload the running config after test case running. - ''' - - par_ctx = parallel_run_context - parallel_coordinator = ParallelCoordinator(par_ctx) if par_ctx.is_par_run else None - if par_ctx.is_par_run and not par_ctx.is_par_leader: - logger.info( - "Fixture core_dump_and_config_check setup for non-leader nodes in parallel run is skipped. " - "Please refer to the leader node log for core dump and config check status." - ) - - parallel_coordinator.wait_and_ack_status_for_followers( - ParallelStatus.SETUP_COMPLETED, - par_ctx.is_par_leader, - par_ctx.target_hostname, - ) - - parallel_coordinator.wait_for_all_followers_ack(ParallelStatus.SETUP_COMPLETED) - - yield {} - - parallel_coordinator.mark_and_wait_for_status( - ParallelStatus.TESTS_COMPLETED, - par_ctx.target_hostname, - par_ctx.is_par_leader, - ) - - logger.info( - "Fixture core_dump_and_config_check teardown for non-leader nodes in parallel run is skipped. " - "Please refer to the leader node log for core dump and config check status." - ) - else: - check_flag = True - if hasattr(request.config.option, 'enable_macsec') and request.config.option.enable_macsec: - check_flag = False - if hasattr(request.config.option, 'markexpr') and request.config.option.markexpr: - if "bsl" in request.config.option.markexpr: - check_flag = False - for m in request.node.iter_markers(): - if m.name == "skip_check_dut_health": - check_flag = False - - module_name = request.node.name - - duts_data = {} - - if check_flag: - - def collect_before_test(dut): - logger.info("Dumping Disk and Memory Space information before test on {}".format(dut.hostname)) - dut.shell("free -h") - dut.shell("df -h") - - logger.info("Collecting core dumps before test on {}".format(dut.hostname)) - duts_data[dut.hostname] = {} - - if "20191130" in dut.os_version: - pre_existing_core_dumps = dut.shell('ls /var/core/ | grep -v python || true')['stdout'].split() - else: - pre_existing_core_dumps = dut.shell('ls /var/core/')['stdout'].split() - duts_data[dut.hostname]["pre_core_dumps"] = pre_existing_core_dumps - - logger.info("Collecting running config before test on {}".format(dut.hostname)) - duts_data[dut.hostname]["pre_running_config"] = {} - if not dut.stat(path="/etc/sonic/running_golden_config.json")['stat']['exists']: - logger.info("Collecting running golden config before test on {}".format(dut.hostname)) - dut.shell("sonic-cfggen -d --print-data > /etc/sonic/running_golden_config.json") - duts_data[dut.hostname]["pre_running_config"][None] = \ - json.loads(dut.shell("cat /etc/sonic/running_golden_config.json", verbose=False)['stdout']) - - if dut.is_multi_asic: - for asic_index in range(0, dut.facts.get('num_asic')): - asic_ns = "asic{}".format(asic_index) - if not dut.stat( - path="/etc/sonic/running_golden_config{}.json".format(asic_index))['stat']['exists']: - dut.shell( - "sonic-cfggen -n {} -d --print-data > /etc/sonic/running_golden_config{}.json".format( - asic_ns, - asic_index, - ) - ) - duts_data[dut.hostname]['pre_running_config'][asic_ns] = \ - json.loads(dut.shell("cat /etc/sonic/running_golden_config{}.json".format(asic_index), - verbose=False)['stdout']) - - with SafeThreadPoolExecutor(max_workers=8) as executor: - for duthost in duthosts: - executor.submit(collect_before_test, duthost) - - if par_ctx.is_par_run and par_ctx.is_par_leader: - parallel_coordinator.set_new_status( - ParallelStatus.SETUP_COMPLETED, - par_ctx.is_par_leader, - par_ctx.target_hostname, - ) - - parallel_coordinator.wait_for_all_followers_ack(ParallelStatus.SETUP_COMPLETED) - - yield duts_data - - if par_ctx.is_par_run and par_ctx.is_par_leader: - parallel_coordinator.mark_and_wait_for_status( - ParallelStatus.TESTS_COMPLETED, - par_ctx.target_hostname, - par_ctx.is_par_leader, - ) - - parallel_coordinator.set_new_status( - ParallelStatus.TEARDOWN_STARTED, - par_ctx.is_par_leader, - par_ctx.target_hostname, - ) - - inconsistent_config = {} - pre_only_config = {} - cur_only_config = {} - new_core_dumps = {} - - core_dump_check_failed = False - config_db_check_failed = False - - check_result = {} - - if check_flag: - - def collect_after_test(dut): - inconsistent_config[dut.hostname] = {} - pre_only_config[dut.hostname] = {} - cur_only_config[dut.hostname] = {} - new_core_dumps[dut.hostname] = [] - - logger.info("Dumping Disk and Memory Space information after test on {}".format(dut.hostname)) - dut.shell("free -h") - dut.shell("df -h") - - logger.info("Collecting core dumps after test on {}".format(dut.hostname)) - if "20191130" in dut.os_version: - cur_cores = dut.shell('ls /var/core/ | grep -v python || true')['stdout'].split() - else: - cur_cores = dut.shell('ls /var/core/')['stdout'].split() - duts_data[dut.hostname]["cur_core_dumps"] = cur_cores - - cur_core_dumps_set = set(duts_data[dut.hostname]["cur_core_dumps"]) - pre_core_dumps_set = set(duts_data[dut.hostname]["pre_core_dumps"]) - new_core_dumps[dut.hostname] = list(cur_core_dumps_set - pre_core_dumps_set) - - logger.info("Collecting running config after test on {}".format(dut.hostname)) - # get running config after running - duts_data[dut.hostname]["cur_running_config"] = {} - duts_data[dut.hostname]["cur_running_config"][None] = \ - json.loads(dut.shell("sonic-cfggen -d --print-data", verbose=False)['stdout']) - if dut.is_multi_asic: - for asic_index in range(0, dut.facts.get('num_asic')): - asic_ns = "asic{}".format(asic_index) - duts_data[dut.hostname]["cur_running_config"][asic_ns] = \ - json.loads(dut.shell("sonic-cfggen -n {} -d --print-data".format(asic_ns), - verbose=False)['stdout']) - - with SafeThreadPoolExecutor(max_workers=8) as executor: - for duthost in duthosts: - executor.submit(collect_after_test, duthost) - - for duthost in duthosts: - if new_core_dumps[duthost.hostname]: - core_dump_check_failed = True - - base_dir = os.path.dirname(os.path.realpath(__file__)) - for new_core_dump in new_core_dumps[duthost.hostname]: - duthost.fetch(src="/var/core/{}".format(new_core_dump), dest=os.path.join(base_dir, "logs")) - - # The tables that we don't care - exclude_config_table_names = set([]) - # The keys that we don't care - # Current skipped keys: - # 1. "MUX_LINKMGR|LINK_PROBER" - # 2. "MUX_LINKMGR|TIMED_OSCILLATION" - # 3. "LOGGER|linkmgrd" - # NOTE: this key is edited by the `run_icmp_responder_session` or `run_icmp_responder` - # to account for the lower performance of the ICMP responder/mux simulator compared to - # real servers and mux cables. - # Linkmgrd is the only service to consume this table so it should not affect other test cases. - # Let's keep this setting in db and we don't want any config reload caused by this key, so - # let's skip checking it. - if "dualtor" in tbinfo["topo"]["name"]: - exclude_config_key_names = [ - 'MUX_LINKMGR|LINK_PROBER', - 'MUX_LINKMGR|TIMED_OSCILLATION', - 'LOGGER|linkmgrd' - ] - else: - exclude_config_key_names = [] - - def _remove_entry(table_name, key_name, config): - if table_name in config and key_name in config[table_name]: - config[table_name].pop(key_name) - if len(config[table_name]) == 0: - config.pop(table_name) - - for cfg_context in duts_data[duthost.hostname]['pre_running_config']: - pre_only_config[duthost.hostname][cfg_context] = {} - cur_only_config[duthost.hostname][cfg_context] = {} - inconsistent_config[duthost.hostname][cfg_context] = {} - - pre_running_config = duts_data[duthost.hostname]["pre_running_config"][cfg_context] - cur_running_config = duts_data[duthost.hostname]["cur_running_config"][cfg_context] - - # Remove ignored keys from base config - for exclude_key in exclude_config_key_names: - fields = exclude_key.split('|') - if len(fields) != 2: - continue - _remove_entry(fields[0], fields[1], pre_running_config) - _remove_entry(fields[0], fields[1], cur_running_config) - - pre_running_config_keys = set(pre_running_config.keys()) - cur_running_config_keys = set(cur_running_config.keys()) - - # Check if there are extra keys in pre running config - pre_config_extra_keys = list( - pre_running_config_keys - cur_running_config_keys - exclude_config_table_names) - for key in pre_config_extra_keys: - pre_only_config[duthost.hostname][cfg_context].update({key: pre_running_config[key]}) - - # Check if there are extra keys in cur running config - cur_config_extra_keys = list( - cur_running_config_keys - pre_running_config_keys - exclude_config_table_names) - for key in cur_config_extra_keys: - cur_only_config[duthost.hostname][cfg_context].update({key: cur_running_config[key]}) - - # Get common keys in pre running config and cur running config - common_config_keys = list(pre_running_config_keys & cur_running_config_keys - - exclude_config_table_names) - - # Check if the running config is modified after module running - for key in common_config_keys: - # TODO: remove these code when solve the problem of "FLEX_COUNTER_DELAY_STATUS" - if key == "FLEX_COUNTER_TABLE": - for sub_key, sub_value in list(pre_running_config[key].items()): - try: - pre_value = pre_running_config[key][sub_key] - cur_value = cur_running_config[key][sub_key] - if pre_value["FLEX_COUNTER_STATUS"] != cur_value["FLEX_COUNTER_STATUS"]: - inconsistent_config[duthost.hostname][cfg_context].update( - { - key: { - "pre_value": pre_running_config[key], - "cur_value": cur_running_config[key] - } - } - ) - except KeyError: - inconsistent_config[duthost.hostname][cfg_context].update( - { - key: { - "pre_value": pre_running_config[key], - "cur_value": cur_running_config[key] - } - } - ) - elif not compare_running_config(pre_running_config[key], cur_running_config[key]): - inconsistent_config[duthost.hostname][cfg_context].update( - { - key: { - "pre_value": pre_running_config[key], - "cur_value": cur_running_config[key] - } - } - ) - - if pre_only_config[duthost.hostname][cfg_context] or \ - cur_only_config[duthost.hostname][cfg_context] or \ - inconsistent_config[duthost.hostname][cfg_context]: - config_db_check_failed = True - - if core_dump_check_failed or config_db_check_failed: - check_result = { - "core_dump_check": { - "failed": core_dump_check_failed, - "new_core_dumps": new_core_dumps - }, - "config_db_check": { - "failed": config_db_check_failed, - "pre_only_config": pre_only_config, - "cur_only_config": cur_only_config, - "inconsistent_config": inconsistent_config - } - } - logger.warning("Core dump or config check failed for {}, results: {}" - .format(module_name, json.dumps(check_result))) - - restore_config_db_and_config_reload(duts_data, duthosts, request) - else: - logger.info("Core dump and config check passed for {}".format(module_name)) - - if check_result: - logger.debug("core_dump_and_config_check failed, check_result: {}".format(json.dumps(check_result))) - add_custom_msg(request, f"{DUT_CHECK_NAMESPACE}.core_dump_check_failed", core_dump_check_failed) - add_custom_msg(request, f"{DUT_CHECK_NAMESPACE}.config_db_check_failed", config_db_check_failed) - - -@pytest.fixture(scope="module", autouse=True) -def temporarily_disable_route_check(request, duthosts): - check_flag = False - for m in request.node.iter_markers(): - if m.name == "disable_route_check": - check_flag = True - break - - def wait_for_route_check_to_pass(dut): - - def run_route_check(): - res = dut.shell("sudo route_check.py", module_ignore_errors=True) - return res["rc"] == 0 - - pt_assert( - wait_until(180, 15, 0, run_route_check), - "route_check.py is still failing after timeout", - ) - - if check_flag: - # If a pytest.fail or any other exceptions are raised in the setup stage of a fixture (before the yield), - # the teardown code (after the yield) will not run, so we are using try...finally... to ensure the - # routeCheck monit will always be started after this fixture. - try: - with SafeThreadPoolExecutor(max_workers=8) as executor: - for duthost in duthosts.frontend_nodes: - executor.submit(wait_for_route_check_to_pass, duthost) - - with SafeThreadPoolExecutor(max_workers=8) as executor: - for duthost in duthosts.frontend_nodes: - executor.submit(stop_route_checker_on_duthost, duthost) - - yield - - with SafeThreadPoolExecutor(max_workers=8) as executor: - for duthost in duthosts.frontend_nodes: - executor.submit(wait_for_route_check_to_pass, duthost) - finally: - with SafeThreadPoolExecutor(max_workers=8) as executor: - for duthost in duthosts.frontend_nodes: - executor.submit(start_route_checker_on_duthost, duthost) - else: - logger.info("Skipping temporarily_disable_route_check fixture") - yield - logger.info("Skipping temporarily_disable_route_check fixture") - - -@pytest.fixture(scope="function") -def on_exit(): - ''' - Utility to register callbacks for cleanup. Runs callbacks despite assertion - failures. Callbacks are executed in reverse order of registration. - ''' - class OnExit(): - def __init__(self): - self.cbs = [] - - def register(self, fn): - self.cbs.append(fn) - - def cleanup(self): - while len(self.cbs) != 0: - self.cbs.pop()() - - on_exit = OnExit() - yield on_exit - on_exit.cleanup() - - -@pytest.fixture(scope="session", autouse=True) -def add_mgmt_test_mark(duthosts): - ''' - @summary: Create mark file at /etc/sonic/mgmt_test_mark, and DUT can use this mark to detect mgmt test. - @param duthosts: fixture to get DUT hosts - ''' - mark_file = "/etc/sonic/mgmt_test_mark" - duthosts.shell("touch %s" % mark_file, module_ignore_errors=True) - - -def verify_packets_any_fixed(test, pkt, ports=[], device_number=0, timeout=None): - """ - Check that a packet is received on _any_ of the specified ports belonging to - the given device (default device_number is 0). - - Also verifies that the packet is not received on any other ports for this - device, and that no other packets are received on the device (unless --relax - is in effect). - - The function is redefined here to workaround code bug in testutils.verify_packets_any - """ - received = False - failures = [] - for device, port in testutils.ptf_ports(): - if device != device_number: - continue - if port in ports: - logging.debug("Checking for pkt on device %d, port %d", device_number, port) - result = testutils.dp_poll(test, device_number=device, port_number=port, - timeout=timeout, exp_pkt=pkt) - if isinstance(result, test.dataplane.PollSuccess): - received = True - else: - failures.append((port, result)) - else: - testutils.verify_no_packet(test, pkt, (device, port)) - testutils.verify_no_other_packets(test) - - if not received: - def format_failure(port, failure): - return "On port %d:\n%s" % (port, failure.format()) - failure_report = "\n".join([format_failure(*f) for f in failures]) - test.fail("Did not receive expected packet on any of ports %r for device %d.\n%s" - % (ports, device_number, failure_report)) - - -# HACK: testutils.verify_packets_any to workaround code bug -# TODO: delete me when ptf version is advanced than https://github.com/p4lang/ptf/pull/139 -testutils.verify_packets_any = verify_packets_any_fixed - -# HACK: We are using set_do_not_care_scapy but it will be deprecated. -if not hasattr(Mask, "set_do_not_care_scapy"): - Mask.set_do_not_care_scapy = Mask.set_do_not_care_packet - - -def run_logrotate(duthost, stop_event): - logger.info("Start rotate_syslog on {}".format(duthost)) - while not stop_event.is_set(): - try: - # Run logrotate for rsyslog - duthost.shell("logrotate -f /etc/logrotate.conf", module_ignore_errors=True) - except subprocess.CalledProcessError as e: - logger.error("Error: {}".format(str(e))) - # Wait for 60 seconds before the next rotation - time.sleep(60) - - -@pytest.fixture(scope="function") -def rotate_syslog(duthosts, enum_rand_one_per_hwsku_frontend_hostname): - duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - - stop_event = threading.Event() - thread = InterruptableThread( - target=run_logrotate, - args=(duthost, stop_event,) - ) - thread.daemon = True - thread.start() - - yield - stop_event.set() - try: - if thread.is_alive(): - thread.join(timeout=30) - logger.info("thread {} joined".format(thread)) - except Exception as e: - logger.debug("Exception occurred in thread {}".format(str(e))) - - logger.info("rotate_syslog exit {}".format(thread)) - - -@pytest.fixture(scope="module") -def gnxi_path(ptfhost): - """ - gnxi's location is updated from /gnxi to /root/gnxi - in RP https://github.com/sonic-net/sonic-buildimage/pull/10599. - But old docker-ptf images don't have this update, - test case will fail for these docker-ptf images, - because it should still call /gnxi files. - For avoiding this conflict, check gnxi path before test and set GNXI_PATH to correct value. - Add a new gnxi_path module fixture to make sure to set GNXI_PATH before test. - """ - path_exists = ptfhost.stat(path="/root/gnxi/") - if path_exists["stat"]["exists"] and path_exists["stat"]["isdir"]: - gnxipath = "/root/gnxi/" - else: - gnxipath = "/gnxi/" - return gnxipath - - -@pytest.fixture(scope="module") -def selected_asic_index(request): - asic_index = DEFAULT_ASIC_ID - if "enum_asic_index" in request.fixturenames: - asic_index = request.getfixturevalue("enum_asic_index") - elif "enum_frontend_asic_index" in request.fixturenames: - asic_index = request.getfixturevalue("enum_frontend_asic_index") - elif "enum_backend_asic_index" in request.fixturenames: - asic_index = request.getfixturevalue("enum_backend_asic_index") - elif "enum_rand_one_asic_index" in request.fixturenames: - asic_index = request.getfixturevalue("enum_rand_one_asic_index") - elif "enum_rand_one_frontend_asic_index" in request.fixturenames: - asic_index = request.getfixturevalue("enum_rand_one_frontend_asic_index") - logger.info(f"Selected asic_index {asic_index}") - return asic_index - - -@pytest.fixture(scope="module") -def ip_netns_namespace_prefix(request, selected_asic_index): - """ - Construct the formatted namespace prefix for executed commands inside the specific - network namespace or for linux commands. - """ - if selected_asic_index == DEFAULT_ASIC_ID: - return '' - else: - return f'sudo ip netns exec {NAMESPACE_PREFIX}{selected_asic_index}' - - -@pytest.fixture(scope="module") -def cli_namespace_prefix(request, selected_asic_index): - """ - Construct the formatted namespace prefix for executed commands inside the specific - network namespace or for CLI commands. - """ - if selected_asic_index == DEFAULT_ASIC_ID: - return '' - else: - return f'-n {NAMESPACE_PREFIX}{selected_asic_index}' - - -def pytest_collection_modifyitems(config, items): - # Skip all stress_tests if --run-stress-test is not set - if not config.getoption("--run-stress-tests"): - skip_stress_tests = pytest.mark.skip(reason="Stress tests run only if --run-stress-tests is passed") - for item in items: - if "stress_test" in item.keywords: - item.add_marker(skip_stress_tests) - - -def update_t1_test_ports(duthost, mg_facts, test_ports, tbinfo): - """ - Find out active IP interfaces and use the list to - remove inactive ports from test_ports - """ - ip_ifaces = duthost.get_active_ip_interfaces(tbinfo, asic_index=0) - port_list = [] - for iface in list(ip_ifaces.keys()): - if iface.startswith("PortChannel"): - port_list.extend( - mg_facts["minigraph_portchannels"][iface]["members"] - ) - else: - port_list.append(iface) - port_list_set = set(port_list) - for port in list(test_ports.keys()): - if port not in port_list_set: - del test_ports[port] - return test_ports - - -@pytest.fixture(scope="module", params=['IPv6', 'IPv4']) -def ip_version(request): - return request.param - - -@pytest.fixture(scope="module") -def setup_pfc_test( - duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, conn_graph_facts, tbinfo, ip_version, # noqa F811 -): - """ - Sets up all the parameters needed for the PFC Watchdog tests - - Args: - duthost: AnsibleHost instance for DUT - ptfhost: AnsibleHost instance for PTF - conn_graph_facts: fixture that contains the parsed topology info - - Yields: - setup_info: dictionary containing pfc timers, generated test ports and selected test ports - """ - SUPPORTED_T1_TOPOS = {"t1-lag", "t1-64-lag", "t1-56-lag", "t1-28-lag", "t1-32-lag"} - duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - mg_facts = duthost.get_extended_minigraph_facts(tbinfo) - port_list = list(mg_facts['minigraph_ports'].keys()) - neighbors = conn_graph_facts['device_conn'].get(duthost.hostname, {}) - config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] - dut_eth0_ip = duthost.mgmt_ip - vlan_nw = None - - if mg_facts['minigraph_vlans']: - # Filter VLANs with one interface inside only(PortChannel interface in case of t0-56-po2vlan topo) - unexpected_vlans = [] - for vlan, vlan_data in list(mg_facts['minigraph_vlans'].items()): - if len(vlan_data['members']) < 2: - unexpected_vlans.append(vlan) - - # Update minigraph_vlan_interfaces with only expected VLAN interfaces - expected_vlan_ifaces = [] - for vlan in unexpected_vlans: - for mg_vl_iface in mg_facts['minigraph_vlan_interfaces']: - if vlan != mg_vl_iface['attachto']: - expected_vlan_ifaces.append(mg_vl_iface) - if expected_vlan_ifaces: - mg_facts['minigraph_vlan_interfaces'] = expected_vlan_ifaces - - # gather all vlan specific info - ip_index = 0 if ip_version == "IPv4" else 1 - vlan_addr = mg_facts['minigraph_vlan_interfaces'][ip_index]['addr'] - vlan_prefix = mg_facts['minigraph_vlan_interfaces'][ip_index]['prefixlen'] - vlan_dev = mg_facts['minigraph_vlan_interfaces'][ip_index]['attachto'] - vlan_ips = duthost.get_ip_in_range( - num=1, prefix="{}/{}".format(vlan_addr, vlan_prefix), - exclude_ips=[vlan_addr])['ansible_facts']['generated_ips'] - vlan_nw = vlan_ips[0].split('/')[0] - - topo = tbinfo["topo"]["name"] - # build the port list for the test - config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] - if ip_version == "IPv4": - ip_version_num = 4 - elif ip_version == "IPv6": - ip_version_num = 6 - else: - pytest.fail(f"Invalid IP version: {input}", pytrace=True) - - tp_handle = TrafficPorts(mg_facts, neighbors, vlan_nw, topo, config_facts, ip_version_num) - test_ports = tp_handle.build_port_list() - - # In T1 topology update test ports by removing inactive ports - if topo in SUPPORTED_T1_TOPOS: - test_ports = update_t1_test_ports( - duthost, mg_facts, test_ports, tbinfo - ) - # select a subset of ports from the generated port list - selected_ports = select_test_ports(test_ports) - - setup_info = {'test_ports': test_ports, - 'port_list': port_list, - 'selected_test_ports': selected_ports, - 'pfc_timers': set_pfc_timers(), - 'neighbors': neighbors, - 'eth0_ip': dut_eth0_ip, - 'ip_version': ip_version - } - - if mg_facts['minigraph_vlans']: - setup_info['vlan'] = {'addr': vlan_addr, - 'prefix': vlan_prefix, - 'dev': vlan_dev - } - else: - setup_info['vlan'] = None - - # stop pfcwd - logger.info("--- Stopping Pfcwd ---") - duthost.command("pfcwd stop") - - # set poll interval - duthost.command("pfcwd interval {}".format(setup_info['pfc_timers']['pfc_wd_poll_time'])) - - # set bulk counter chunk size - logger.info("--- Setting bulk counter polling chunk size ---") - duthost.command('redis-cli -n 4 hset "FLEX_COUNTER_TABLE|PORT" BULK_CHUNK_SIZE 64' - ' BULK_CHUNK_SIZE_PER_PREFIX "SAI_PORT_STAT_IF_OUT_QLEN:0;SAI_PORT_STAT_IF_IN_FEC:32"') - - logger.info("setup_info : {}".format(setup_info)) - yield setup_info - - -@pytest.fixture(scope="session") -def setup_gnmi_server(request, localhost, duthost): - """ - SAI validation library uses gNMI to access sonic-db data - objects. This fixture is used by tests to set up gNMI server - """ - disable_sai_validation = request.config.getoption("--disable_sai_validation") - if disable_sai_validation: - logger.info("SAI validation is disabled") - yield duthost, None - return - gnmi_insecure = request.config.getoption("--gnmi_insecure") - if gnmi_insecure: - logger.info("gNMI insecure mode is enabled") - yield duthost, None - return - else: - checkpoint_name = "before-applying-gnmi-certs" - cert_path = pathlib.Path("/tmp/gnmi_certificates") - gnmi_setup.create_certificates(localhost, duthost.mgmt_ip, cert_path) - gnmi_setup.copy_certificates_to_dut(cert_path, duthost) - gnmi_setup.apply_certs(duthost, checkpoint_name) - yield duthost, cert_path - gnmi_setup.remove_certs(duthost, checkpoint_name) - - -@pytest.fixture(scope="session") -def setup_connection(request, setup_gnmi_server): - duthost, cert_path = setup_gnmi_server - disable_sai_validation = request.config.getoption("--disable_sai_validation") - if disable_sai_validation: - logger.info("SAI validation is disabled") - yield None - return - else: - # Dynamically import create_gnmi_stub - gnmi_client_module = importlib.import_module("tests.common.sai_validation.gnmi_client") - create_gnmi_stub = getattr(gnmi_client_module, "create_gnmi_stub") - - # if cert_path is None then it is insecure mode - gnmi_insecure = request.config.getoption("--gnmi_insecure") - gnmi_target_port = int(request.config.getoption("--gnmi_port")) - duthost_mgmt_ip = duthost.mgmt_ip - channel = None - gnmi_connection = None - if gnmi_insecure: - channel, gnmi_connection = create_gnmi_stub(ip=duthost_mgmt_ip, - port=gnmi_target_port, secure=False) - else: - root_cert = str(cert_path / 'gnmiCA.pem') - client_cert = str(cert_path / 'gnmiclient.crt') - client_key = str(cert_path / 'gnmiclient.key') - channel, gnmi_connection = create_gnmi_stub(ip=duthost_mgmt_ip, - port=gnmi_target_port, secure=True, - root_cert_path=root_cert, - client_cert_path=client_cert, - client_key_path=client_key) - yield gnmi_connection - channel.close() - - -@pytest.fixture(scope="module", autouse=True) -def restore_golden_config_db(duthost): - if file_exists_on_dut(duthost, GOLDEN_CONFIG_DB_PATH_ORI): - duthost.shell("cp {} {}".format(GOLDEN_CONFIG_DB_PATH_ORI, GOLDEN_CONFIG_DB_PATH)) - logger.info("[restore_golden_config_db] Restored {}".format(GOLDEN_CONFIG_DB_PATH)) - yield - - -@pytest.fixture(scope="session") -def gnmi_connection(request, setup_connection): - connection = setup_connection - yield connection - - -class DualtorMuxPortSetupConfig(enum.Flag): - """Dualtor mux port setup config.""" - DUALTOR_SKIP_SETUP_MUX_PORTS = enum.auto() - DUALTOR_SETUP_MUX_PORT_MANUAL_MODE = enum.auto() - - # active-standby mux setup configs - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_ENUM_TOR = enum.auto() - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_ENUM_TOR_MANUAL_MODE = \ - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_ENUM_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_UPPER_TOR = enum.auto() - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_UPPER_TOR_MANUAL_MODE = \ - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_UPPER_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_LOWER_TOR = enum.auto() - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_LOWER_TOR_MANUAL_MODE = \ - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_LOWER_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_TOR = enum.auto() - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_TOR_MANUAL_MODE = \ - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_UNSELECTED_TOR = enum.auto() - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_UNSELECTED_TOR_MANUAL_MODE = \ - DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_UNSELECTED_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE - - # active-active mux setup configs - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_ENUM_TOR = enum.auto() - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_ENUM_TOR_MANUAL_MODE = \ - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_ENUM_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_UPPER_TOR = enum.auto() - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_UPPER_TOR_MANUAL_MODE = \ - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_UPPER_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_LOWER_TOR = enum.auto() - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_LOWER_TOR_MANUAL_MODE = \ - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_LOWER_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_TOR = enum.auto() - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_TOR_MANUAL_MODE = \ - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_UNSELECTED_TOR = enum.auto() - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_UNSELECTED_TOR_MANUAL_MODE = \ - DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_UNSELECTED_TOR | DUALTOR_SETUP_MUX_PORT_MANUAL_MODE - - -@pytest.fixture(autouse=True) -def setup_dualtor_mux_ports(active_active_ports, duthost, duthosts, tbinfo, request, mux_server_url): # noqa:F811 - """Setup dualtor mux ports.""" - def _get_enumerated_dut_hostname(request): - for k, v in request.node.callspec.params.items(): - if k in ("enum_dut_hostname", - "enum_frontend_dut_hostname", - "enum_supervisor_dut_hostname", - "enum_rand_one_per_hwsku_hostname", - "enum_rand_one_per_hwsku_frontend_hostname"): - return v - return None - - def _get_peer_dut_hostname(local_dut_hostname): - """Get the peer DUT hostname.""" - for dut in duthosts: - if dut.hostname != local_dut_hostname: - return dut.hostname - return None - - def _is_dut_hostname_valid(dut_hostname): - """Check if the dut hostname is valid/present in the tb.""" - return dut_hostname in duthosts.duts - - topo_name = tbinfo["topo"]["name"] - is_dualtor = "dualtor" in topo_name - - if not is_dualtor: - logging.info("skip setup dualtor mux cables on non-dualtor testbed") - yield False - return - - is_dualtor_aa = "dualtor-aa" in topo_name - # read setup configs from pytest markers - dualtor_setup_config = DualtorMuxPortSetupConfig(0) - for marker in request.node.iter_markers(): - try: - dualtor_setup_config |= DualtorMuxPortSetupConfig[marker.name.upper()] - except KeyError: - continue - logging.debug("dualtor mux port setup config: %s", dualtor_setup_config) - - if dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_SKIP_SETUP_MUX_PORTS: - logging.info("skip setup dualtor mux cables") - yield False - return - - is_test_func_parametrized = hasattr(request.node, "callspec") - is_enum = is_test_func_parametrized and \ - any(param.startswith("enum_") for param in request.node.callspec.params.keys()) - rand_one_unselected_dut_hostname = _get_peer_dut_hostname(rand_one_dut_hostname_var) - - if is_dualtor_aa: - # NOTE: Skip setup mux ports if the test explicitly calls - # an active-active mux toggle fixture. - for fixture in request.fixturenames: - if fixture == "config_active_active_dualtor_active_standby": - logging.info("Skip setup dualtor mux cables as toggle " - "fixture %s is explicitly called", - fixture) - yield False - return - - active_dut_hostname = None - standby_dut_hostname = None - - if dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_ENUM_TOR: - if is_test_func_parametrized: - standby_dut_hostname = _get_enumerated_dut_hostname(request) - active_dut_hostname = _get_peer_dut_hostname(standby_dut_hostname) - elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_UPPER_TOR: - standby_dut_hostname = duthosts[0].hostname - active_dut_hostname = duthosts[-1].hostname - elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_LOWER_TOR: - standby_dut_hostname = duthosts[-1].hostname - active_dut_hostname = duthosts[0].hostname - elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_TOR: - standby_dut_hostname = rand_one_dut_hostname_var - active_dut_hostname = rand_one_unselected_dut_hostname - elif dualtor_setup_config & \ - DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_ACTIVE_SETUP_STANDBY_ON_RANDOM_UNSELECTED_TOR: - standby_dut_hostname = rand_one_unselected_dut_hostname - active_dut_hostname = rand_one_dut_hostname_var - else: - # NOTE: If no marker is explicitly specified on active-active dualtor, let's - # leave the ToRs pair in active-active by default. - pass - - if (_is_dut_hostname_valid(active_dut_hostname) and - _is_dut_hostname_valid(standby_dut_hostname)): - logging.info("Setup active-active dualtor, DUT %s as active side, " - "DUT %s as standby side", - active_dut_hostname, standby_dut_hostname) - config_active_active_dualtor( - duthosts[active_dut_hostname], - duthosts[standby_dut_hostname], - active_active_ports, - dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_SETUP_MUX_PORT_MANUAL_MODE - ) - else: - yield False - return - - else: - # NOTE: Skip setup mux ports if the test explicitly calls - # an active-standby mux toggle fixture. - for fixture in request.fixturenames: - if fixture.startswith("toggle_") and fixture in dir(mux_simulator_control): - logging.info("Skip setup dualtor mux cables as toggle " - "fixture %s is explicitly called", - fixture) - yield False - return - - if is_enum and not (dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_ENUM_TOR): - logging.info( - "skip setup dualtor mux cables on test with enum fixture") - yield False - return - - target_dut_hostname = None - test_func_args = [_ for _ in inspect.signature(request.node.function).parameters.keys()] - - if dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_ENUM_TOR: - # retrieve the current enumerated dut hostname - if is_test_func_parametrized: - target_dut_hostname = _get_enumerated_dut_hostname(request) - elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_UPPER_TOR: - target_dut_hostname = duthosts[0].hostname - elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_LOWER_TOR: - target_dut_hostname = duthosts[-1].hostname - elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_TOR: - target_dut_hostname = rand_one_dut_hostname_var - elif dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_ACTIVE_STANDBY_TOGGLE_TO_RANDOM_UNSELECTED_TOR: - target_dut_hostname = rand_one_unselected_dut_hostname - else: - # if the test function uses `rand_selected_dut`, toggle active side to `rand_selected_dut` - rand_dut_fixture = "rand_one_dut_hostname" - duthost_fixture = "duthost" - if rand_dut_fixture in test_func_args or rand_dut_fixture in request.fixturenames: - logging.debug("Select random selected DUT %s as the toggle target", - rand_one_dut_hostname_var) - target_dut_hostname = rand_one_dut_hostname_var - # if the test function uses `duthost`, toggle active side to `duthost`. - elif duthost_fixture in test_func_args or duthost_fixture in request.fixturenames: - logging.debug("Select duthost DUT %s as the toggle target", - duthost.hostname) - target_dut_hostname = duthost.hostname - - if not _is_dut_hostname_valid(target_dut_hostname): - logging.warn("Invalid DUT selected %s as the toggle target, fallback to use the upper ToR %s", - target_dut_hostname, duthosts[0].hostname) - target_dut_hostname = duthosts[0].hostname - logging.info("Toggle mux ports to the target DUT %s", - target_dut_hostname) - mux_simulator_control._toggle_all_simulator_ports_to_target_dut(target_dut_hostname, - duthosts, - mux_server_url, - tbinfo) - - if dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_SETUP_MUX_PORT_MANUAL_MODE: - logger.info("Set all mux ports to manual mode on all ToRs") - duthosts.shell("config muxcable mode manual all") - - yield True - - if dualtor_setup_config & DualtorMuxPortSetupConfig.DUALTOR_SETUP_MUX_PORT_MANUAL_MODE: - logger.info("Set all muxcable to auto mode on all ToRs") - duthosts.shell("config muxcable mode auto all") - duthosts.shell("config save -y") - - -def pytest_runtest_setup(item): - # Let's place `setup_dualtor_mux_ports` at the tail of fixture list - # to make it running as last as possible. - fixtureinfo = item._fixtureinfo - for fixturedef in fixtureinfo.name2fixturedefs.values(): - fixturedef = fixturedef[0] - if fixturedef.argname == "setup_dualtor_mux_ports": - fixtureinfo.names_closure.remove("setup_dualtor_mux_ports") - fixtureinfo.names_closure.append("setup_dualtor_mux_ports") - - -@pytest.fixture(scope="module", autouse=True) -def yang_validation_check(request, duthosts): - """ - YANG validation check that runs before and after each test module - """ - skip_yang = request.config.getoption("--skip_yang") - - if skip_yang: - logger.info("Skipping YANG validation check due to --skip_yang flag") - return - - def run_yang_validation(stage): - """Run YANG validation and return results""" - validation_results = {} - - for duthost in duthosts: - logger.info(f"Running YANG validation on {duthost.hostname} ({stage})") - try: - result = duthost.shell( - 'echo "[]" | sudo config apply-patch /dev/stdin', - module_ignore_errors=True - ) - - if result['rc'] != 0: - validation_results[duthost.hostname] = { - 'failed': True, - 'error': result.get('stderr', result.get('stdout', 'Unknown error')) - } - logger.error(f"YANG validation failed on {duthost.hostname} ({stage}): " - f"{validation_results[duthost.hostname]['error']}") - else: - validation_results[duthost.hostname] = {'failed': False} - logger.info(f"YANG validation passed on {duthost.hostname} ({stage})") + selected_interfaces = random.sample(supported_available_optical_interfaces, + min(collected_ports_num, len(supported_available_optical_interfaces))) + logging.info("Selected interfaces for tests: {}".format(selected_interfaces)) - except Exception as e: - validation_results[duthost.hostname] = { - 'failed': True, - 'error': str(e) - } - logger.error(f"Exception during YANG validation on {duthost.hostname} ({stage}): {str(e)}") + for interface in selected_interfaces: + self.shutdown_and_startup_interfaces(dut, interface) - return validation_results + pytest_assert(self.get_interface_status(dut, interface) == "up", + "Interface {} was not up before disabling/enabling tx-output using sfputil".format(interface)) - # pre-test YANG validation - pre_results = run_yang_validation("pre-test") + remote_fault_before = self.get_mac_fault_count(dut, interface, "mac remote fault") + logging.info("Initial MAC remote fault count on {}: {}".format(interface, remote_fault_before)) - # Check if any pre-test validation failed - pre_failures = {host: result for host, result in pre_results.items() if result['failed']} - if pre_failures: - error_summary = [] - for host, result in pre_failures.items(): - error_summary.append(f"{host}: {result['error']}") + dut.shell("sudo sfputil debug tx-output {} disable".format(interface)) + time.sleep(5) + pytest_assert(self.get_interface_status(dut, interface) == "down", + "Interface {iface} did not go down after 'sudo sfputil debug tx-output {iface} disable'" + .format(iface=interface)) - pt_assert(False, "pre-test YANG validation failed:\n" + "\n".join(error_summary)) + dut.shell("sudo sfputil debug tx-output {} enable".format(interface)) + time.sleep(20) - yield + pytest_assert(self.get_interface_status(dut, interface) == "up", + "Interface {iface} did not come up after 'sudo sfputil debug tx-output {iface} enable'" + .format(iface=interface)) - # post-test YANG validation - post_results = run_yang_validation("post-test") + remote_fault_after = self.get_mac_fault_count(dut, interface, "mac remote fault") + logging.info("MAC remote fault count after disabling/enabling tx-output using sfputil {}: {}".format( + interface, remote_fault_after)) - # Check if any post-test validation failed - post_failures = {host: result for host, result in post_results.items() if result['failed']} - if post_failures: - error_summary = [] - for host, result in post_failures.items(): - error_summary.append(f"{host}: {result['error']}") + pytest_assert(remote_fault_after > remote_fault_before, + "MAC remote fault count did not increment after disabling/enabling tx-output on the device") - pt_assert(False, "post-test YANG validation failed:\n" + "\n".join(error_summary)) + pytest_assert(len(failed_api_ports) == 0, "Interfaces with failed API ports: {}".format(failed_api_ports)) From 84bb852b3b5e3ed159794d6001746612da042b6c Mon Sep 17 00:00:00 2001 From: Guy Shemesh Date: Mon, 22 Dec 2025 13:36:49 +0200 Subject: [PATCH 4/5] Change method name to is_innolight_cable Signed-off-by: Guy Shemesh --- tests/common/mellanox_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/common/mellanox_data.py b/tests/common/mellanox_data.py index fcbc1ebdc8f..fd39858a083 100644 --- a/tests/common/mellanox_data.py +++ b/tests/common/mellanox_data.py @@ -1262,8 +1262,8 @@ def get_hw_management_version(duthost): return full_version[len('1.mlnx.'):] -def is_pinewave_module(port_info): - """ Check if the given port info indicates an pinewave module and handle known issues """ +def is_innolight_cable(port_info): + """ Check if the given port info indicates an Innolight cable and handle known issues """ vendor_name = port_info.get('Vendor Name', '').upper() vendor_pn = port_info.get('Vendor PN', '').upper() manufacturer = port_info.get('manufacturer', '').upper() @@ -1275,7 +1275,7 @@ def is_pinewave_module(port_info): def is_unsupported_module(port_info, port_number): - if is_pinewave_module(port_info): + if is_innolight_cable(port_info): logger.info(f"Port {port_number} has an unsupported module, skipping it and continue to check other ports") return True return False From 05cd2395dc13bda5f2d7b3ae74f189f9893a6433 Mon Sep 17 00:00:00 2001 From: Guy Shemesh Date: Mon, 12 Jan 2026 14:00:05 +0200 Subject: [PATCH 5/5] Adjust test /telemetry/test_events.py to handle IPV6 only topologies Signed-off-by: Guy Shemesh --- tests/common/devices/multi_asic.py | 4 +- tests/common/devices/sonic.py | 385 +++++++++++++++++++- tests/common/mellanox_data.py | 86 ----- tests/conftest.py | 240 ++++++++---- tests/layer1/test_port_error.py | 154 ++------ tests/telemetry/events/bgp_events.py | 25 +- tests/telemetry/events/dhcp-relay_events.py | 2 +- tests/telemetry/events/host_events.py | 3 +- tests/telemetry/events/run_events_test.py | 16 +- tests/telemetry/events/swss_events.py | 8 +- tests/telemetry/telemetry_utils.py | 6 +- tests/telemetry/test_events.py | 4 +- 12 files changed, 609 insertions(+), 324 deletions(-) diff --git a/tests/common/devices/multi_asic.py b/tests/common/devices/multi_asic.py index c90f25d6686..4b42a10d5e5 100644 --- a/tests/common/devices/multi_asic.py +++ b/tests/common/devices/multi_asic.py @@ -424,8 +424,8 @@ def is_container_running(self, service): return False - def is_bgp_state_idle(self): - return self.sonichost.is_bgp_state_idle() + def is_bgp_state_idle(self, ip6=False): + return self.sonichost.is_bgp_state_idle(ip6) def is_service_running(self, service_name, docker_name=None): docker_name = service_name if docker_name is None else docker_name diff --git a/tests/common/devices/sonic.py b/tests/common/devices/sonic.py index 3acccf17839..0b016ac75d8 100644 --- a/tests/common/devices/sonic.py +++ b/tests/common/devices/sonic.py @@ -1,4 +1,3 @@ - import ipaddress import json import logging @@ -25,6 +24,19 @@ from tests.common.helpers.parallel import parallel_run_threaded from tests.common.errors import RunAnsibleModuleFail from tests.common import constants +from typing import TypedDict + + +class ShellResult(TypedDict): + cmd: str + rc: int + stdout: str + stderr: str + stdout_lines: list + stderr_lines: list + failed: bool + changed: bool + logger = logging.getLogger(__name__) PROCESS_TO_CONTAINER_MAP = { @@ -432,6 +444,25 @@ def is_frontend_node(self): """ return not self.is_supervisor_node() + def is_console_switch(self): + """ + Check if this device has console functionality enabled. + + Returns: + bool: True if console is enabled, False otherwise + """ + try: + result = self.shell( + 'sonic-db-cli CONFIG_DB hget "CONSOLE_SWITCH|console_mgmt" enabled', + module_ignore_errors=True + ) + if result["rc"] == 0: + output = result["stdout"].strip().lower() + return output == 'yes' + return False + except Exception: + return False + def is_macsec_capable_node(self): im = self.host.options['inventory_manager'] inv_files = im._sources @@ -2192,14 +2223,14 @@ def no_shutdown_bgp_neighbors(self, asn, neighbors=[]): logging.info('No shut BGP neighbors: {}'.format(json.dumps(neighbors))) return self.command(command) - def is_bgp_state_idle(self): + def is_bgp_state_idle(self, ip6=False): """ Check if all BGP peers are in IDLE state. Returns: True or False """ - bgp_summary = self.command("show ip bgp summary")["stdout_lines"] + bgp_summary = self.command("show {} bgp summary".format("ipv6" if ip6 else "ip"))["stdout_lines"] idle_count = 0 expected_idle_count = 0 @@ -2961,6 +2992,354 @@ def start_bgpd(self): logging.error(f"Error starting bgpd process: {str(e)}") return {'rc': 1, 'stdout': '', 'stderr': str(e)} + def is_file_existed(self, device_path: str) -> bool: + """Check if device path exists. Returns True if exists, False otherwise.""" + res: ShellResult = self.shell(f"test -e {device_path}", module_ignore_errors=True) + return True if res['rc'] == 0 else False + + def is_file_opened(self, device_path: str) -> bool: + """Check if device path is not in use. Returns True if file is opened, False otherwise.""" + res: ShellResult = self.shell(f"sudo lsof {device_path}", module_ignore_errors=True) + return True if res["stdout"] else False + + def _get_serial_device_prefix(self) -> str: + """ + Get the serial device prefix for the platform. + + Returns: + str: The device prefix (e.g., "/dev/C0-", "/dev/ttyUSB-") + """ + # Reads udevprefix.conf from the platform directory to determine the correct device prefix + # Falls back to /dev/ttyUSB- if the config file doesn't exist + script = ''' +from sonic_py_common import device_info +import os + +platform_path, _ = device_info.get_paths_to_platform_and_hwsku_dirs() +config_file = os.path.join(platform_path, "udevprefix.conf") + +if os.path.exists(config_file): + with open(config_file, 'r') as f: + device_prefix = "/dev/" + f.readline().rstrip() +else: + raise FileNotFoundError("Config file not found") + +print(device_prefix) +''' + cmd = f"python3 << 'EOF'\n{script}\nEOF" + res: ShellResult = self.shell(cmd, module_ignore_errors=True) + + if res['rc'] != 0 or not res['stdout'].strip(): + logging.warning("Failed to get serial device prefix, using default /dev/ttyUSB-") + device_prefix = "/dev/ttyUSB-" + else: + device_prefix = res['stdout'].strip() + + return device_prefix + + def _get_serial_device_path(self, port: int) -> str: + """ + Get the full serial device path for a given port. + + Args: + port: Port number (e.g., 1, 2) + + Returns: + str: The full device path (e.g., "/dev/C0-1", "/dev/ttyUSB-1") + """ + device_prefix = self._get_serial_device_prefix() + return f"{device_prefix}{port}" + + def set_loopback(self, port: int, baud_rate: int = 9600, flow_control: bool = False) -> None: + """Set loopback on the specified port. Raises RuntimeError on failure.""" + if not self.is_console_switch(): + error_msg = "This operation is only supported on console switches" + logging.error(error_msg) + raise RuntimeError(error_msg) + + device_path = self._get_serial_device_path(port) + + # Check if device path exists and is not in use or raise error + if not self.is_file_existed(device_path): + error_msg = f"Device path {device_path} does not exist" + logging.error(error_msg) + raise RuntimeError(error_msg) + if self.is_file_opened(device_path): + error_msg = f"Device path {device_path} is already in use" + logging.error(error_msg) + raise RuntimeError(error_msg) + + # Set hardware flow control option + crtscts_val = "1" if flow_control else "0" + + # Execute loopback command + command = ( + f"sudo socat -d -d " + f"FILE:{device_path},raw,echo=0,nonblock,b{baud_rate},cs8," + f"parenb=0,cstopb=0,ixon=0,ixoff=0,crtscts={crtscts_val},icrnl=0,onlcr=0,opost=0,isig=0,icanon=0 " + f"EXEC:'/bin/cat' " + f"& echo $! " + ) + + res: ShellResult = self.shell(command, module_ignore_errors=True) + if res['failed']: + error_msg = f"Failed to start socat on port {port}: {res.get('stderr', '')}" + logging.error(error_msg) + raise RuntimeError(error_msg) + + logging.info(f"Successfully started socat loopback on port {port}") + + def unset_loopback(self, port: int) -> None: + """Unset loopback on the specified port. Raises RuntimeError on failure.""" + if not self.is_console_switch(): + error_msg = "This operation is only supported on console switches" + logging.error(error_msg) + raise RuntimeError(error_msg) + + # Find all related socat processes + device_path = self._get_serial_device_path(port) + if not self.is_file_existed(device_path): + error_msg = f"Device path {device_path} does not exist" + logging.error(error_msg) + raise RuntimeError(error_msg) + + res: ShellResult = self.shell(f"pgrep -f 'socat .*{device_path}'", module_ignore_errors=True) + pids = res['stdout'].strip().split('\n') + + # Kill all related socat processes + for pid in pids: + self.shell(f"sudo kill {pid}", module_ignore_errors=True) + + time.sleep(0.5) + + # Confirm all related processes for the port have stopped + res: ShellResult = \ + self.shell(f"ps aux | grep 'socat .*{device_path}' | grep -v grep", module_ignore_errors=True) + + if res['stdout'].strip(): + error_msg = f"Failed to stop socat process for device path {device_path}" + logging.error(error_msg) + raise RuntimeError(error_msg) + + logging.info(f"Successfully stopped socat loopback on port {port}") + + def bridge(self, port1: int, port2: int, baud_rate: int = 9600, flow_control: bool = False) -> None: + """Bridge two ports together. Raises RuntimeError on failure.""" + if not self.is_console_switch(): + error_msg = "This operation is only supported on console switches" + logging.error(error_msg) + raise RuntimeError(error_msg) + + device_path1 = self._get_serial_device_path(port1) + device_path2 = self._get_serial_device_path(port2) + + # Check if both device paths exist and are not in use or raise error + if not self.is_file_existed(device_path1): + error_msg = f"Device path {device_path1} does not exist" + logging.error(error_msg) + raise RuntimeError(error_msg) + if not self.is_file_existed(device_path2): + error_msg = f"Device path {device_path2} does not exist" + logging.error(error_msg) + raise RuntimeError(error_msg) + if self.is_file_opened(device_path1): + error_msg = f"Device path {device_path1} is already in use" + logging.error(error_msg) + raise RuntimeError(error_msg) + if self.is_file_opened(device_path2): + error_msg = f"Device path {device_path2} is already in use" + logging.error(error_msg) + raise RuntimeError(error_msg) + + # Set hardware flow control option + crtscts_val = "1" if flow_control else "0" + + # Execute bridge command + command = ( + f"sudo socat -d -d " + f"FILE:{device_path1},raw,echo=0,nonblock,b{baud_rate},cs8," + f"parenb=0,cstopb=0,ixon=0,ixoff=0,crtscts={crtscts_val},icrnl=0,onlcr=0,opost=0,isig=0,icanon=0 " + f"FILE:{device_path2},raw,echo=0,nonblock,b{baud_rate},cs8," + f"parenb=0,cstopb=0,ixon=0,ixoff=0,crtscts={crtscts_val},icrnl=0,onlcr=0,opost=0,isig=0,icanon=0 " + f"& echo $! " + ) + + res: ShellResult = self.shell(command, module_ignore_errors=True) + if res['failed']: + error_msg = f"Failed to bridge ports {port1} and {port2}: {res.get('stderr', '')}" + logging.error(error_msg) + raise RuntimeError(error_msg) + + logging.info(f"Successfully bridged ports {port1} and {port2}") + + def unbridge(self, port1: int, port2: int) -> None: + """Remove bridge between two ports. Raises RuntimeError on failure.""" + if not self.is_console_switch(): + error_msg = "This operation is only supported on console switches" + logging.error(error_msg) + raise RuntimeError(error_msg) + + device_path1 = self._get_serial_device_path(port1) + device_path2 = self._get_serial_device_path(port2) + + if not self.is_file_existed(device_path1): + error_msg = f"Device path {device_path1} does not exist" + logging.error(error_msg) + raise RuntimeError(error_msg) + if not self.is_file_existed(device_path2): + error_msg = f"Device path {device_path2} does not exist" + logging.error(error_msg) + raise RuntimeError(error_msg) + + # Find all related socat processes for both ports + res: ShellResult = self.shell( + f"pgrep -f 'socat .*{device_path1}.*{device_path2}|socat .*{device_path2}.*{device_path1}'", + module_ignore_errors=True + ) + pids = res['stdout'].strip().split('\n') if res['stdout'].strip() else [] + + if not pids or pids == ['']: + error_msg = f"No bridge found between {device_path1} and {device_path2}" + logging.error(error_msg) + raise RuntimeError(error_msg) + + # Kill all related socat processes + for pid in pids: + if pid: # Skip empty strings + self.shell(f"sudo kill {pid}", module_ignore_errors=True) + + time.sleep(0.5) + + # Confirm all related processes have stopped + res: ShellResult = self.shell( + f"ps aux | " + f"grep -E 'socat.*{device_path1}.*{device_path2}|socat.*{device_path2}.*{device_path1}' | " + f"grep -v grep", + module_ignore_errors=True + ) + + if res['stdout'].strip(): + error_msg = f"Failed to stop bridge process between {device_path1} and {device_path2}" + logging.error(error_msg) + raise RuntimeError(error_msg) + + logging.info(f"Successfully unbridged ports {port1} and {port2}") + + def bridge_remote( + self, port: int, remote_host: str, remote_port: int, + baud_rate: int = 9600, flow_control: bool = False + ) -> None: + """Bridge a local serial port to a remote host's TCP port. Raises RuntimeError on failure.""" + if not self.is_console_switch(): + error_msg = "This operation is only supported on console switches" + logging.error(error_msg) + raise RuntimeError(error_msg) + + device_path = self._get_serial_device_path(port) + + # Check if device path exists and is not in use or raise error + if not self.is_file_existed(device_path): + error_msg = f"Device path {device_path} does not exist" + logging.error(error_msg) + raise RuntimeError(error_msg) + if self.is_file_opened(device_path): + error_msg = f"Device path {device_path} is already in use" + logging.error(error_msg) + raise RuntimeError(error_msg) + + # Set hardware flow control option + crtscts_val = "1" if flow_control else "0" + + # Execute bridge command to remote host + command = ( + f"sudo socat -d -d " + f"FILE:{device_path},raw,echo=0,nonblock,b{baud_rate},cs8," + f"parenb=0,cstopb=0,ixon=0,ixoff=0,crtscts={crtscts_val},icrnl=0,onlcr=0,opost=0,isig=0,icanon=0 " + f"TCP:{remote_host}:{remote_port} " + f"& echo $! " + ) + + res: ShellResult = self.shell(command, module_ignore_errors=True) + if res['failed']: + error_msg = f"Failed to bridge port {port} to {remote_host}:{remote_port}: {res.get('stderr', '')}" + logging.error(error_msg) + raise RuntimeError(error_msg) + + logging.info(f"Successfully bridged port {port} to {remote_host}:{remote_port}") + + def unbridge_remote(self, port: int) -> None: + """Remove bridge from a local port to any remote host. Raises RuntimeError on failure.""" + if not self.is_console_switch(): + error_msg = "This operation is only supported on console switches" + logging.error(error_msg) + raise RuntimeError(error_msg) + + device_path = self._get_serial_device_path(port) + + if not self.is_file_existed(device_path): + error_msg = f"Device path {device_path} does not exist" + logging.error(error_msg) + raise RuntimeError(error_msg) + + # Find all related socat processes for the port with TCP connection + res: ShellResult = self.shell( + f"pgrep -f 'socat .*{device_path}.*TCP:'", + module_ignore_errors=True + ) + pids = res['stdout'].strip().split('\n') if res['stdout'].strip() else [] + + if not pids or pids == ['']: + error_msg = f"No remote bridge found for port {port}" + logging.error(error_msg) + raise RuntimeError(error_msg) + + # Kill all related socat processes + for pid in pids: + if pid: # Skip empty strings + self.shell(f"sudo kill {pid}", module_ignore_errors=True) + + time.sleep(0.5) + + # Confirm all related processes have stopped + res: ShellResult = self.shell( + f"ps aux | grep 'socat .*{device_path}.*TCP:' | grep -v grep", + module_ignore_errors=True + ) + + if res['stdout'].strip(): + error_msg = f"Failed to stop remote bridge process for port {port}" + logging.error(error_msg) + raise RuntimeError(error_msg) + + logging.info(f"Successfully unbridged remote connection for port {port}") + + def cleanup_all_console_sessions(self) -> None: + """Clean up all console sessions. Raises RuntimeError on failure.""" + if not self.is_console_switch(): + error_msg = "This operation is only supported on console switches" + logging.error(error_msg) + raise RuntimeError(error_msg) + + device_prefix = self._get_serial_device_prefix() + pattern = f"{device_prefix}*" + + # Find all related serial port processes + res: ShellResult = self.shell(f"sudo lsof -t {pattern}", module_ignore_errors=True) + pids = res['stdout'].strip().split('\n') + + # Kill all related processes + for pid in pids: + self.shell(f"sudo kill {pid}", module_ignore_errors=True) + + # Check that no serial ports are in use + res: ShellResult = self.shell(f"sudo lsof {pattern}", module_ignore_errors=True) + if res['stdout'].strip() or res['stderr'].strip(): + error_msg = "Failed to clean up all console sessions: some ports are still in use" + logging.error(error_msg) + raise RuntimeError(error_msg) + + logging.info("Successfully cleaned up all console sessions") + def assert_exit_non_zero(shell_output): if shell_output['rc'] != 0: diff --git a/tests/common/mellanox_data.py b/tests/common/mellanox_data.py index fd39858a083..df6aff7028c 100644 --- a/tests/common/mellanox_data.py +++ b/tests/common/mellanox_data.py @@ -1,7 +1,4 @@ import functools -import pytest -import logging -logger = logging.getLogger(__name__) SPC1_HWSKUS = ["ACS-MSN2700", "Mellanox-SN2700", "Mellanox-SN2700-D48C8", "ACS-MSN2740", "ACS-MSN2100", "ACS-MSN2410", @@ -1260,86 +1257,3 @@ def get_hardware_version(duthost, platform): def get_hw_management_version(duthost): full_version = duthost.shell('dpkg-query --showformat=\'${Version}\' --show hw-management')['stdout'] return full_version[len('1.mlnx.'):] - - -def is_innolight_cable(port_info): - """ Check if the given port info indicates an Innolight cable and handle known issues """ - vendor_name = port_info.get('Vendor Name', '').upper() - vendor_pn = port_info.get('Vendor PN', '').upper() - manufacturer = port_info.get('manufacturer', '').upper() - - return ( - ('PINEWAVE' in vendor_name and 'T-OH8CNT-NMT' in vendor_pn) or - ('PINEWAVE' in manufacturer) - ) - - -def is_unsupported_module(port_info, port_number): - if is_innolight_cable(port_info): - logger.info(f"Port {port_number} has an unsupported module, skipping it and continue to check other ports") - return True - return False - - -def skip_on_unsupported_module(): - pytest.skip("All ports are with unsupported modules, skipping the test due to Github issue #21878") - - -def is_cmis_version_supported(cmis_version, min_required_version=5.0, failed_api_ports=None, port_name=None): - """ - Check if a CMIS version supports a specific feature by comparing it to a minimum required version - @param: cmis_version: CMIS version string (e.g., "5.0", "4.0", etc.) - @param: min_required_version: Minimum required CMIS version (default: 5.0) - @param: failed_api_ports: List to append failed ports to (optional) - @param: port_name: Port name to append to failed list if version check fails (optional) - @return: bool: True if CMIS version is supported, False otherwise - """ - try: - cmis_version_float = float(cmis_version) - return cmis_version_float >= min_required_version - except (ValueError, TypeError): - if failed_api_ports is not None and port_name is not None: - failed_api_ports.append(port_name) - return False - - -def get_supported_available_optical_interfaces(eeprom_infos, parsed_presence, - min_cmis_version=5.0, return_failed_api_ports=False): - """ - Filter available optical interfaces based on presence, EEPROM detection, media type, and CMIS version support - @param: eeprom_infos: Dictionary containing EEPROM information for each port - @param: parsed_presence: Dictionary containing presence status for each port - @param: min_cmis_version: Minimum required CMIS version (default: 5.0) - @param: return_failed_api_ports: If True, return both available_optical_interfaces and failed_api_ports. - If False, return only available_optical_interfaces (default: False) - @return: list or tuple: If return_failed_api_ports=False, returns list of available optical interface names. - If return_failed_api_ports=True, returns (available_optical_interfaces, failed_api_ports) - """ - available_optical_interfaces = [] - failed_api_ports = [] - - for port_name, eeprom_info in eeprom_infos.items(): - if parsed_presence.get(port_name) != "Present": - continue - if "SFP EEPROM detected" not in eeprom_info[port_name]: - continue - media_technology = eeprom_info.get("Media Interface Technology", "N/A").upper() - if "COPPER" in media_technology: - continue - if "N/A" in media_technology: - failed_api_ports.append(port_name) - continue - cmis_version = eeprom_info.get("CMIS Revision", "N/A") - if "N/A" in cmis_version: - failed_api_ports.append(port_name) - continue - elif not is_cmis_version_supported(cmis_version, min_cmis_version, failed_api_ports, port_name): - logging.info(f"Port {port_name} skipped: CMIS not supported on this port.") - continue - - available_optical_interfaces.append(port_name) - - if return_failed_api_ports: - return available_optical_interfaces, failed_api_ports - else: - return available_optical_interfaces diff --git a/tests/conftest.py b/tests/conftest.py index 7d4b3b11c95..136a1298056 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,8 +37,6 @@ from tests.common.fixtures.ptfhost_utils import ptf_portmap_file # noqa: F401 from tests.common.fixtures.ptfhost_utils import ptf_test_port_map_active_active # noqa: F401 from tests.common.fixtures.ptfhost_utils import run_icmp_responder_session # noqa: F401 -from tests.common.fixtures.grpc_fixtures import ptf_grpc, ptf_gnoi, ptf_grpc_custom, \ - setup_gnoi_tls_server, ptf_gnmi # noqa: F401 from tests.common.dualtor.dual_tor_utils import disable_timed_oscillation_active_standby # noqa: F401 from tests.common.dualtor.dual_tor_utils import config_active_active_dualtor from tests.common.dualtor.dual_tor_common import active_active_ports # noqa: F401 @@ -239,6 +237,8 @@ def pytest_addoption(parser): ############################ # macsec options # ############################ + parser.addoption("--snappi_macsec", action="store_true", default=False, + help="Enable macsec on tgen links of testbed") parser.addoption("--enable_macsec", action="store_true", default=False, help="Enable macsec on some links of testbed") parser.addoption("--macsec_profile", action="store", default="all", @@ -307,12 +307,6 @@ def pytest_addoption(parser): parser.addoption("--container_test", action="store", default="", help="This flag indicates that the test is being run by the container test.") - ################################# - # Port error test options # - ################################# - parser.addoption("--collected-ports-num", action="store", default=5, type=int, - help="Number of ports to collect for testing (default: 5)") - ################################# # YANG validation options # ################################# @@ -948,84 +942,136 @@ def initial_neighbor(neighbor_name, vm_name): def fanouthosts(enhance_inventory, ansible_adhoc, tbinfo, conn_graph_facts, creds, duthosts): # noqa: F811 """ Shortcut fixture for getting Fanout hosts + Supports both Ethernet connections and Serial connections + + For Ethernet connections: Uses device_conn from conn_graph_facts + For Serial connections: Uses device_serial_link from conn_graph_facts """ - dev_conn = conn_graph_facts.get('device_conn', {}) + # Internal helper functions + def create_or_get_fanout(fanout_hosts, fanout_name, dut_host) -> FanoutHost | None: + """ + Create FanoutHost if not exists, or return existing one. + Fanout creation logic for both Ethernet and Serial connections. + + Args: + fanout_hosts (dict): Dictionary of existing fanout hosts + fanout_name (str): Fanout device hostname + dut_host (str): DUT hostname that connects to this fanout + + Returns: + FanoutHost: Fanout host object + """ + # Return existing fanout if already created + if fanout_name in fanout_hosts: + fanout = fanout_hosts[fanout_name] + if dut_host not in fanout.dut_hostnames: + fanout.dut_hostnames.append(dut_host) + return fanout + + # Get fanout device info from inventory + try: + host_vars = ansible_adhoc().options['inventory_manager'].get_host(fanout_name).vars + except Exception as e: + logging.warning(f"Cannot get inventory for fanout {fanout_name}: {e}") + return None + + os_type = host_vars.get('os', 'eos') + + # Get credentials based on OS type + if 'fanout_tacacs_user' in creds: + fanout_user = creds['fanout_tacacs_user'] + fanout_password = creds['fanout_tacacs_password'] + elif 'fanout_tacacs_{}_user'.format(os_type) in creds: + fanout_user = creds['fanout_tacacs_{}_user'.format(os_type)] + fanout_password = creds['fanout_tacacs_{}_password'.format(os_type)] + elif os_type == 'sonic': + fanout_user = creds.get('fanout_sonic_user', None) + fanout_password = creds.get('fanout_sonic_password', None) + elif os_type == 'eos': + fanout_user = creds.get('fanout_network_user', None) + fanout_password = creds.get('fanout_network_password', None) + elif os_type == 'onyx': + fanout_user = creds.get('fanout_mlnx_user', None) + fanout_password = creds.get('fanout_mlnx_password', None) + elif os_type == 'ixia': + # Skip for ixia device which has no fanout + return None + else: + pytest.fail(f"Unsupported fanout OS type {os_type} for fanout {fanout_name}") + + # EOS specific shell credentials + eos_shell_user = None + eos_shell_password = None + if os_type == "eos": + admin_user = creds['fanout_admin_user'] + admin_password = creds['fanout_admin_password'] + eos_shell_user = creds.get('fanout_shell_user', admin_user) + eos_shell_password = creds.get('fanout_shell_password', admin_password) + + # Create FanoutHost object + fanout = FanoutHost( + ansible_adhoc, + os_type, + fanout_name, + 'FanoutLeaf', + fanout_user, + fanout_password, + eos_shell_user=eos_shell_user, + eos_shell_passwd=eos_shell_password + ) + fanout.dut_hostnames = [dut_host] + fanout_hosts[fanout_name] = fanout + + # For SONiC fanout, get port alias to name mapping + if fanout.os == 'sonic': + ifs_status = fanout.host.get_interfaces_status() + for key, interface_info in list(ifs_status.items()): + fanout.fanout_port_alias_to_name[interface_info['alias']] = interface_info['interface'] + logging.info(f"fanout {fanout_name} fanout_port_alias_to_name {fanout.fanout_port_alias_to_name}") + + return fanout + + # Main fixture logic + fanout_hosts = {} + # Skip special topologies that have no fanout if tbinfo['topo']['name'].startswith('nut-'): - # Nut topology has no fanout + logging.info("Nut topology has no fanout") return fanout_hosts - # WA for virtual testbed which has no fanout - for dut_host, value in list(dev_conn.items()): - duthost = duthosts[dut_host] + # Process Ethernet connections + + dev_conn = conn_graph_facts.get('device_conn', {}) + + for dut_name, ethernet_ports in dev_conn.items(): + + duthost = duthosts[dut_name] + + # Skip virtual testbed which has no fanout if duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0': - continue # skip for kvm platform which has no fanout + logging.info(f"Skipping kvm platform {dut_name}") + continue + + # Get minigraph facts for port alias mapping mg_facts = duthost.minigraph_facts(host=duthost.hostname)['ansible_facts'] - for dut_port in list(value.keys()): - fanout_rec = value[dut_port] + + # Process each Ethernet port connection + for dut_port, fanout_rec in ethernet_ports.items(): fanout_host = str(fanout_rec['peerdevice']) fanout_port = str(fanout_rec['peerport']) - if fanout_host in list(fanout_hosts.keys()): - fanout = fanout_hosts[fanout_host] - else: - host_vars = ansible_adhoc().options[ - 'inventory_manager'].get_host(fanout_host).vars - os_type = host_vars.get('os', 'eos') - if 'fanout_tacacs_user' in creds: - fanout_user = creds['fanout_tacacs_user'] - fanout_password = creds['fanout_tacacs_password'] - elif 'fanout_tacacs_{}_user'.format(os_type) in creds: - fanout_user = creds['fanout_tacacs_{}_user'.format(os_type)] - fanout_password = creds['fanout_tacacs_{}_password'.format(os_type)] - elif os_type == 'sonic': - fanout_user = creds.get('fanout_sonic_user', None) - fanout_password = creds.get('fanout_sonic_password', None) - elif os_type == 'eos': - fanout_user = creds.get('fanout_network_user', None) - fanout_password = creds.get('fanout_network_password', None) - elif os_type == 'onyx': - fanout_user = creds.get('fanout_mlnx_user', None) - fanout_password = creds.get('fanout_mlnx_password', None) - elif os_type == 'ixia': - # Skip for ixia device which has no fanout - continue - else: - # when os is mellanox, not supported - pytest.fail("os other than sonic and eos not supported") - - eos_shell_user = None - eos_shell_password = None - if os_type == "eos": - admin_user = creds['fanout_admin_user'] - admin_password = creds['fanout_admin_password'] - eos_shell_user = creds.get('fanout_shell_user', admin_user) - eos_shell_password = creds.get('fanout_shell_password', admin_password) - - fanout = FanoutHost(ansible_adhoc, - os_type, - fanout_host, - 'FanoutLeaf', - fanout_user, - fanout_password, - eos_shell_user=eos_shell_user, - eos_shell_passwd=eos_shell_password) - fanout.dut_hostnames = [dut_host] - fanout_hosts[fanout_host] = fanout - - if fanout.os == 'sonic': - ifs_status = fanout.host.get_interfaces_status() - for key, interface_info in list(ifs_status.items()): - fanout.fanout_port_alias_to_name[interface_info['alias']] = interface_info['interface'] - logging.info("fanout {} fanout_port_alias_to_name {}" - .format(fanout_host, fanout.fanout_port_alias_to_name)) - - fanout.add_port_map(encode_dut_port_name(dut_host, dut_port), fanout_port) - - # Add port name to fanout port mapping port if dut_port is alias. - if dut_port in mg_facts['minigraph_port_alias_to_name_map']: + # Create or get fanout object + fanout = create_or_get_fanout(fanout_hosts, fanout_host, dut_name) + if fanout is None: + continue + + # Add Ethernet port mapping: DUT port -> Fanout port + fanout.add_port_map(encode_dut_port_name(dut_name, dut_port), fanout_port) + + # Handle port alias mapping if available + if dut_port in mg_facts.get('minigraph_port_alias_to_name_map', {}): mapped_port = mg_facts['minigraph_port_alias_to_name_map'][dut_port] # only add the mapped port which isn't in device_conn ports to avoid overwriting port map wrongly, # it happens when an interface has the same name with another alias, for example: @@ -1033,11 +1079,43 @@ def fanouthosts(enhance_inventory, ansible_adhoc, tbinfo, conn_graph_facts, cred # -------------------- # Ethernet108 Ethernet32 # Ethernet32 Ethernet13/1 - if mapped_port not in list(value.keys()): - fanout.add_port_map(encode_dut_port_name(dut_host, mapped_port), fanout_port) + if mapped_port not in list(ethernet_ports.keys()): + fanout.add_port_map(encode_dut_port_name(dut_name, mapped_port), fanout_port) - if dut_host not in fanout.dut_hostnames: - fanout.dut_hostnames.append(dut_host) + # Process Serial connections + + dev_serial_link = conn_graph_facts.get('device_serial_link', {}) + + for dut_name, serial_ports_map in dev_serial_link.items(): + + duthost = duthosts[dut_name] + + # Skip virtual testbed which has no fanout + if duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0': + logging.info(f"Skipping kvm platform {dut_name} for serial links") + continue + + # Process each Serial port connection + for host_port, link_info in serial_ports_map.items(): + fanout_host = str(link_info['peerdevice']) + fanout_port = str(link_info['peerport']) + baud_rate = link_info.get('baud_rate', "9600") + flow_control = link_info.get('flow_control', "0") == "1" + + # Create or get fanout object + fanout = create_or_get_fanout(fanout_hosts, fanout_host, dut_name) + if fanout is None: + continue + + # Add Serial port mapping + fanout.add_serial_port_map(dut_name, host_port, fanout_port, baud_rate, flow_control) + + logging.debug( + f"Added serial port mapping: {dut_name} Console{host_port} -> " + f"{fanout_host}:{fanout_port} (baud={link_info.get('baud_rate', '9600')})" + ) + + logging.info(f"fanouthosts fixture initialized with {len(fanout_hosts)} fanout devices") return fanout_hosts @@ -1105,6 +1183,10 @@ def topo_bgp_routes(localhost, ptfhosts, tbinfo): if 'servers' in tbinfo: servers_dut_interfaces = {value['ptf_ip'].split("/")[0]: value['dut_interfaces'] for value in tbinfo['servers'].values()} + + # Check if logs directory exists, otherwise use /tmp + log_path = "logs" if os.path.isdir("logs") else "/tmp" + for ptfhost in ptfhosts: ptf_ip = ptfhost.mgmt_ip res = localhost.announce_routes( @@ -1112,7 +1194,7 @@ def topo_bgp_routes(localhost, ptfhosts, tbinfo): ptf_ip=ptf_ip, action='generate', path="../ansible/", - log_path="logs", + log_path=log_path, dut_interfaces=servers_dut_interfaces.get(ptf_ip) if servers_dut_interfaces else None, ) if 'topo_routes' not in res: diff --git a/tests/layer1/test_port_error.py b/tests/layer1/test_port_error.py index deb92bb3cba..2801127146b 100644 --- a/tests/layer1/test_port_error.py +++ b/tests/layer1/test_port_error.py @@ -2,14 +2,9 @@ import pytest import random import time -import os -import re from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import skip_release -from tests.common.platform.transceiver_utils import parse_sfp_eeprom_infos -from tests.common.mellanox_data import get_supported_available_optical_interfaces -from tests.common.utilities import wait_until pytestmark = [ pytest.mark.disable_loganalyzer, # disable automatic loganalyzer @@ -20,25 +15,9 @@ cmd_sfp_presence = "sudo sfpshow presence" -@pytest.fixture(scope="session") -def collected_ports_num(request): - """ - Fixture to get the number of ports to collect from command line argument - """ - return request.config.getoption("--collected-ports-num") - - class TestMACFault(object): - @pytest.fixture(scope="class", autouse=True) - def is_supported_nvidia_platform_with_sw_control_disabled(self, duthost): - return 'nvidia' in duthost.facts['platform'].lower() and not self.is_sw_control_feature_enabled(duthost) - - @pytest.fixture(scope="class", autouse=True) - def is_supported_nvidia_platform_with_sw_control_enabled(self, duthost): - return 'nvidia' in duthost.facts['platform'].lower() and self.is_sw_control_feature_enabled(duthost) - - @pytest.fixture(scope="class", autouse=True) - def is_supported_platform(self, duthost, tbinfo, is_supported_nvidia_platform_with_sw_control_disabled): + @pytest.fixture(autouse=True) + def is_supported_platform(self, duthost, tbinfo): if 'ptp' not in tbinfo['topo']['name']: pytest.skip("Skipping test: Not applicable for non-PTP topology") @@ -47,9 +26,6 @@ def is_supported_platform(self, duthost, tbinfo, is_supported_nvidia_platform_wi else: pytest.skip("DUT has platform {}, test is not supported".format(duthost.facts['platform'])) - if is_supported_nvidia_platform_with_sw_control_disabled: - pytest.skip("SW control feature is not enabled on Nvidia platform") - @staticmethod def get_mac_fault_count(dut, interface, fault_type): output = dut.show_and_parse("show int errors {}".format(interface)) @@ -68,85 +44,30 @@ def get_mac_fault_count(dut, interface, fault_type): def get_interface_status(dut, interface): return dut.show_and_parse("show interfaces status {}".format(interface))[0].get("oper", "unknown") - @pytest.fixture(scope="class", autouse=True) - def reboot_dut(self, duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname): - from tests.common.reboot import reboot - reboot(duthosts[enum_rand_one_per_hwsku_frontend_hostname], - localhost, safe_reboot=True, check_intf_up_ports=True) - - @pytest.fixture(scope="class") - def get_dut_and_supported_available_optical_interfaces(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - is_supported_nvidia_platform_with_sw_control_enabled): + @pytest.fixture + def select_random_interfaces(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname): dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + interfaces = list(dut.show_and_parse("show interfaces status")) sfp_presence = dut.command(cmd_sfp_presence) parsed_presence = {line.split()[0]: line.split()[1] for line in sfp_presence["stdout_lines"][2:]} - supported_available_optical_interfaces = [] - failed_api_ports = [] - - if is_supported_nvidia_platform_with_sw_control_enabled: - - eeprom_infos = dut.shell("sudo sfputil show eeprom -d")['stdout'] - eeprom_infos = parse_sfp_eeprom_infos(eeprom_infos) - - supported_available_optical_interfaces, failed_api_ports = ( - get_supported_available_optical_interfaces( - eeprom_infos, parsed_presence, return_failed_api_ports=True - ) - ) - pytest_assert(supported_available_optical_interfaces, - "No interfaces with SFP detected. Cannot proceed with tests.") - logging.info("Available Optical interfaces for tests: {}".format(supported_available_optical_interfaces)) - else: - interfaces = list(dut.show_and_parse("show interfaces status")) - supported_available_optical_interfaces = [ - intf["interface"] for intf in interfaces - if parsed_presence.get(intf["interface"]) == "Present" - ] - - pytest_assert(supported_available_optical_interfaces, - "No interfaces with SFP detected. Cannot proceed with tests.") - - return dut, supported_available_optical_interfaces, failed_api_ports - - def is_sw_control_feature_enabled(self, duthost): - """ - Check if SW control feature is enabled. - """ - try: - platform_name = duthost.facts['platform'] - hwsku = duthost.facts.get('hwsku', '') - sai_profile_path = os.path.join('/usr/share/sonic/device', platform_name, hwsku, 'sai.profile') - cmd = duthost.shell('cat {}'.format(sai_profile_path), module_ignore_errors=True) - if cmd['rc'] == 0 and 'SAI_INDEPENDENT_MODULE_MODE' in cmd['stdout']: - sc_enabled = re.search(r"SAI_INDEPENDENT_MODULE_MODE=(\d?)", cmd['stdout']) - if sc_enabled and sc_enabled.group(1) == '1': - return True - except Exception as e: - logging.error("Error checking SW control feature on Nvidia platform: {}".format(e)) - return False - - def shutdown_and_startup_interfaces(self, dut, interface): - dut.command("sudo config interface shutdown {}".format(interface)) - pytest_assert(wait_until(30, 2, 0, lambda: self.get_interface_status(dut, interface) == "down"), - "Interface {} did not go down after shutdown".format(interface)) - - dut.command("sudo config interface startup {}".format(interface)) - pytest_assert(wait_until(30, 2, 0, lambda: self.get_interface_status(dut, interface) == "up"), - "Interface {} did not come up after startup".format(interface)) - - def test_mac_local_fault_increment(self, get_dut_and_supported_available_optical_interfaces, - collected_ports_num): - dut, supported_available_optical_interfaces, failed_api_ports = ( - get_dut_and_supported_available_optical_interfaces() - ) - selected_interfaces = random.sample(supported_available_optical_interfaces, - min(collected_ports_num, len(supported_available_optical_interfaces))) - logging.info("Selected interfaces for tests: {}".format(selected_interfaces)) - - for interface in selected_interfaces: - self.shutdown_and_startup_interfaces(dut, interface) + available_interfaces = [ + intf["interface"] for intf in interfaces + if parsed_presence.get(intf["interface"]) == "Present" + ] + + pytest_assert(available_interfaces, "No interfaces with SFP detected. Cannot proceed with tests.") + + # Select 5 random interfaces (or fewer if not enough available) + selected_interfaces = random.sample(available_interfaces, min(5, len(available_interfaces))) + + return dut, selected_interfaces + + def test_mac_local_fault_increment(self, select_random_interfaces): + dut, interfaces = select_random_interfaces + + for interface in interfaces: pytest_assert(self.get_interface_status(dut, interface) == "up", "Interface {} was not up before disabling/enabling rx-output using sfputil".format(interface)) @@ -156,35 +77,24 @@ def test_mac_local_fault_increment(self, get_dut_and_supported_available_optical dut.shell("sudo sfputil debug rx-output {} disable".format(interface)) time.sleep(5) pytest_assert(self.get_interface_status(dut, interface) == "down", - "Interface {iface} did not go down after 'sudo sfputil debug rx-output {iface} disable'" - .format(iface=interface)) + "Interface {} did not go down after disabling rx-output using sfputil".format(interface)) dut.shell("sudo sfputil debug rx-output {} enable".format(interface)) time.sleep(20) pytest_assert(self.get_interface_status(dut, interface) == "up", - "Interface {iface} did not come up after 'sudo sfputil debug rx-output {iface} enable'" - .format(iface=interface)) + "Interface {} did not come up after enabling rx-output using sfputil".format(interface)) local_fault_after = self.get_mac_fault_count(dut, interface, "mac local fault") logging.info("MAC local fault count after disabling/enabling rx-output using sfputil {}: {}".format( interface, local_fault_after)) pytest_assert(local_fault_after > local_fault_before, - "MAC local fault count did not increment after disabling/enabling rx-output on the device") + "MAC local fault count did not increment after disabling/enabling tx-output on the device") - pytest_assert(len(failed_api_ports) == 0, "Interfaces with failed API ports: {}".format(failed_api_ports)) - - def test_mac_remote_fault_increment(self, get_dut_and_supported_available_optical_interfaces, collected_ports_num): - dut, supported_available_optical_interfaces, failed_api_ports = ( - get_dut_and_supported_available_optical_interfaces() - ) - selected_interfaces = random.sample(supported_available_optical_interfaces, - min(collected_ports_num, len(supported_available_optical_interfaces))) - logging.info("Selected interfaces for tests: {}".format(selected_interfaces)) - - for interface in selected_interfaces: - self.shutdown_and_startup_interfaces(dut, interface) + def test_mac_remote_fault_increment(self, select_random_interfaces): + dut, interfaces = select_random_interfaces + for interface in interfaces: pytest_assert(self.get_interface_status(dut, interface) == "up", "Interface {} was not up before disabling/enabling tx-output using sfputil".format(interface)) @@ -194,21 +104,17 @@ def test_mac_remote_fault_increment(self, get_dut_and_supported_available_optica dut.shell("sudo sfputil debug tx-output {} disable".format(interface)) time.sleep(5) pytest_assert(self.get_interface_status(dut, interface) == "down", - "Interface {iface} did not go down after 'sudo sfputil debug tx-output {iface} disable'" - .format(iface=interface)) + "Interface {} did not go down after disabling rx-output using sfputil".format(interface)) dut.shell("sudo sfputil debug tx-output {} enable".format(interface)) time.sleep(20) pytest_assert(self.get_interface_status(dut, interface) == "up", - "Interface {iface} did not come up after 'sudo sfputil debug tx-output {iface} enable'" - .format(iface=interface)) + "Interface {} did not come up after disabling tx-output using sfputil".format(interface)) remote_fault_after = self.get_mac_fault_count(dut, interface, "mac remote fault") - logging.info("MAC remote fault count after disabling/enabling tx-output using sfputil {}: {}".format( + logging.info("MAC remote fault count after disabling/enabling rx-output using sfputil {}: {}".format( interface, remote_fault_after)) pytest_assert(remote_fault_after > remote_fault_before, "MAC remote fault count did not increment after disabling/enabling tx-output on the device") - - pytest_assert(len(failed_api_ports) == 0, "Interfaces with failed API ports: {}".format(failed_api_ports)) diff --git a/tests/telemetry/events/bgp_events.py b/tests/telemetry/events/bgp_events.py index 2155d09b679..f0c87cf7b0f 100644 --- a/tests/telemetry/events/bgp_events.py +++ b/tests/telemetry/events/bgp_events.py @@ -4,29 +4,30 @@ import time import ipaddress +from tests.common.utilities import is_ipv6_only_topology from run_events_test import run_test logger = logging.getLogger(__name__) tag = "sonic-events-bgp" -def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): +def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang, tbinfo=None): run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, drop_tcp_packets, - "bgp_notification.json", "sonic-events-bgp:notification", tag) + "bgp_notification.json", "sonic-events-bgp:notification", tag, tbinfo=tbinfo) run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, shutdown_bgp_neighbors, - "bgp_state.json", "sonic-events-bgp:bgp-state", tag) + "bgp_state.json", "sonic-events-bgp:bgp-state", tag, tbinfo=tbinfo) -def drop_tcp_packets(duthost): +def drop_tcp_packets(duthost, tbinfo=None): + is_ipv6_only = tbinfo and is_ipv6_only_topology(tbinfo) # Check if DUT management is IPv6-only and select appropriate BGP neighbor dut_facts = duthost.dut_basic_facts()['ansible_facts']['dut_basic_facts'] is_mgmt_ipv6_only = dut_facts.get('is_mgmt_ipv6_only', False) - # Get all BGP neighbors and filter by IP version based on management interface all_bgp_neighbors = duthost.get_bgp_neighbors() bgp_neighbor = None - if is_mgmt_ipv6_only: + if is_ipv6_only: # Find an IPv6 BGP neighbor for neighbor_ip in all_bgp_neighbors.keys(): if ipaddress.ip_address(neighbor_ip).version == 6: @@ -35,10 +36,7 @@ def drop_tcp_packets(duthost): if bgp_neighbor is None: raise Exception("No IPv6 BGP neighbors found for IPv6-only management interface") iptables_cmd = "ip6tables" - logger.info( - "Using IPv6 BGP neighbor %s and ip6tables for IPv6-only DUT management interface", - bgp_neighbor - ) + logger.info("Using IPv6 BGP neighbor {} and ip6tables for IPv6-only DUT management interface".format(bgp_neighbor)) else: # Find an IPv4 BGP neighbor (or just use the first one) for neighbor_ip in all_bgp_neighbors.keys(): @@ -49,7 +47,7 @@ def drop_tcp_packets(duthost): # Fallback to first neighbor if no IPv4 found bgp_neighbor = list(all_bgp_neighbors.keys())[0] iptables_cmd = "iptables" - logger.info("Using IPv4 BGP neighbor {} and iptables for IPv4 DUT management interface".format(bgp_neighbor)) + logger.info("Using IPv4 BGP neighbor {} and iptables for IPv4 topology".format(bgp_neighbor)) holdtime_timer_ms = duthost.get_bgp_neighbor_info(bgp_neighbor)["bgpTimerConfiguredHoldTimeMsecs"] @@ -73,9 +71,10 @@ def drop_tcp_packets(duthost): assert ret["rc"] == 0, "Unable to remove DROP rule from {}".format(iptables_cmd) -def shutdown_bgp_neighbors(duthost): +def shutdown_bgp_neighbors(duthost, tbinfo=None): logger.info("Shutting down bgp neighbors to test bgp-state event") - assert duthost.is_service_running("bgpcfgd", "bgp") is True and duthost.is_bgp_state_idle() is False + is_ipv6_only = tbinfo and is_ipv6_only_topology(tbinfo) + assert duthost.is_service_running("bgpcfgd", "bgp") is True and duthost.is_bgp_state_idle(is_ipv6_only) is False logger.info("Start all bgp sessions") ret = duthost.shell("config bgp startup all") assert ret["rc"] == 0, "Failing to startup" diff --git a/tests/telemetry/events/dhcp-relay_events.py b/tests/telemetry/events/dhcp-relay_events.py index f5072260f9e..e11ef565969 100644 --- a/tests/telemetry/events/dhcp-relay_events.py +++ b/tests/telemetry/events/dhcp-relay_events.py @@ -14,7 +14,7 @@ tag = "sonic-events-dhcp-relay" -def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): +def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang, tbinfo=None): if duthost.dut_basic_facts()['ansible_facts']['dut_basic_facts'].get("is_smartswitch") and \ duthost.facts.get("router_type") == 'leafrouter': pytest.skip("Skipping dhcp_relay events for smartswitch t1 topologies") diff --git a/tests/telemetry/events/host_events.py b/tests/telemetry/events/host_events.py index 88c916da1fa..ca2c53065f0 100644 --- a/tests/telemetry/events/host_events.py +++ b/tests/telemetry/events/host_events.py @@ -12,8 +12,7 @@ logger = logging.getLogger(__name__) tag = "sonic-events-host" - -def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): +def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang, tbinfo=None): logger.info("Beginning to test host events") run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger_kernel_event, "event_kernel.json", "sonic-events-host:event-kernel", tag, False) diff --git a/tests/telemetry/events/run_events_test.py b/tests/telemetry/events/run_events_test.py index ba9c6b4fa09..6a547aca8db 100644 --- a/tests/telemetry/events/run_events_test.py +++ b/tests/telemetry/events/run_events_test.py @@ -11,13 +11,19 @@ def run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger, json_file, - filter_event_regex, tag, heartbeat=False, timeout=30, ptfadapter=None): + filter_event_regex, tag, heartbeat=False, timeout=30, ptfadapter=None, tbinfo=None): op_file = os.path.join(data_dir, json_file) if trigger is not None: # no trigger for heartbeat - if ptfadapter is None: - trigger(duthost) # add events to cache - else: - trigger(duthost, ptfadapter) + try: + if ptfadapter is None: + trigger(duthost, tbinfo=tbinfo) # add events to cache + else: + trigger(duthost, ptfadapter, tbinfo=tbinfo) + except TypeError: + if ptfadapter is None: + trigger(duthost) + else: + trigger(duthost, ptfadapter) listen_for_events(duthost, gnxi_path, ptfhost, filter_event_regex, op_file, timeout) # listen from cache data = {} diff --git a/tests/telemetry/events/swss_events.py b/tests/telemetry/events/swss_events.py index ec05fffacd3..95c82f00012 100644 --- a/tests/telemetry/events/swss_events.py +++ b/tests/telemetry/events/swss_events.py @@ -32,10 +32,10 @@ WAIT_TIME = 3 -def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): +def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang, tbinfo=None): logger.info("Beginning to test swss events") run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, shutdown_interface, - "if_state.json", "sonic-events-swss:if-state", tag) + "if_state.json", "sonic-events-swss:if-state", tag, tbinfo=tbinfo) asic_type = duthost.facts["asic_type"] if asic_type == "mellanox": @@ -47,10 +47,10 @@ def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang) if duthost.facts["hwsku"] not in skip_pfc_hwskus: run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, generate_pfc_storm, - "pfc_storm.json", "sonic-events-swss:pfc-storm", tag) + "pfc_storm.json", "sonic-events-swss:pfc-storm", tag, tbinfo=tbinfo) run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger_crm_threshold_exceeded, - "chk_crm_threshold.json", "sonic-events-swss:chk_crm_threshold", tag) + "chk_crm_threshold.json", "sonic-events-swss:chk_crm_threshold", tag, tbinfo=tbinfo) def shutdown_interface(duthost): diff --git a/tests/telemetry/telemetry_utils.py b/tests/telemetry/telemetry_utils.py index ea93888d9f7..faedbf84708 100644 --- a/tests/telemetry/telemetry_utils.py +++ b/tests/telemetry/telemetry_utils.py @@ -75,10 +75,10 @@ def fetch_json_ptf_output(regex, output, match_no): def listen_for_events(duthost, gnxi_path, ptfhost, filter_event_regex, op_file, timeout, update_count=1, - match_number=0): + match_number=0, tbinfo=None): cmd = generate_client_cli(duthost=duthost, gnxi_path=gnxi_path, method=METHOD_SUBSCRIBE, submode=SUBMODE_ONCHANGE, update_count=update_count, xpath="all[heartbeat=2]", - target="EVENTS", filter_event_regex=filter_event_regex, timeout=timeout) + target="EVENTS", filter_event_regex=filter_event_regex, timeout=timeout, tbinfo=tbinfo) result = ptfhost.shell(cmd) assert result["rc"] == 0, "PTF command failed with non zero return code" output = result["stdout"] @@ -109,7 +109,7 @@ def trigger_logger(duthost, log, process, container="", priority="local0.notice" def generate_client_cli(duthost, gnxi_path, method=METHOD_GET, xpath="COUNTERS/Ethernet0", target="COUNTERS_DB", subscribe_mode=SUBSCRIBE_MODE_STREAM, submode=SUBMODE_SAMPLE, intervalms=0, update_count=3, create_connections=1, filter_event_regex="", namespace=None, - timeout=-1, polling_interval=10, max_sync_count=-1): + timeout=-1, polling_interval=10, max_sync_count=-1, tbinfo=None): """ Generate the py_gnmicli command line based on the given params. This version ensures the command runs from the correct directory and within the activated virtual environment to resolve dependency issues. diff --git a/tests/telemetry/test_events.py b/tests/telemetry/test_events.py index 69790244f9b..cc3a340d08a 100644 --- a/tests/telemetry/test_events.py +++ b/tests/telemetry/test_events.py @@ -38,7 +38,7 @@ def validate_yang(duthost, op_file="", yang_file=""): @pytest.mark.disable_loganalyzer def test_events(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, ptfadapter, setup_streaming_telemetry, gnxi_path, test_eventd_healthy, toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_host_m, # noqa: F811 - setup_standby_ports_on_non_enum_rand_one_per_hwsku_host_m): # noqa: F811 + setup_standby_ports_on_non_enum_rand_one_per_hwsku_host_m, tbinfo): # noqa: F811 """ Run series of events inside duthost and validate that output is correct and conforms to YANG schema""" duthost = duthosts[enum_rand_one_per_hwsku_hostname] @@ -51,7 +51,7 @@ def test_events(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, ptfadapter, if file.endswith("_events.py") and not file.endswith("eventd_events.py"): module = __import__(file[:len(file)-3]) try: - module.test_event(duthost, gnxi_path, ptfhost, ptfadapter, DATA_DIR, validate_yang) + module.test_event(duthost, gnxi_path, ptfhost, ptfadapter, DATA_DIR, validate_yang, tbinfo=tbinfo) except pytest.skip.Exception as e: logger.info("Skipping test file: {} due to {}".format(file, e)) continue