diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index af6339f7b7a..f73efedfd7f 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -124,7 +124,8 @@ connection_plugins = plugins/connection lookup_plugins = plugins/lookup # vars_plugins = /usr/share/ansible_plugins/vars_plugins filter_plugins = plugins/filter -callback_whitelist = profile_tasks +# Disable profile tasks callback to avoid possible deadlock +# callback_whitelist = profile_tasks # by default callbacks are not loaded for /bin/ansible, enable this if you # want, for example, a notification or logging callback to also apply to @@ -190,7 +191,7 @@ become_ask_pass=False # ssh arguments to use # Leaving off ControlPersist will result in poor performance, so use # paramiko on older platforms rather than removing it -ssh_args = -o ControlMaster=auto -o ControlPersist=180s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o ServerAliveCountMax=40 +ssh_args = -o ControlMaster=auto -o ControlPersist=300s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o ServerAliveCountMax=40 -o TCPKeepAlive=yes # The path to use for the ControlPath sockets. This defaults to diff --git a/ansible/config_sonic_basedon_testbed.yml b/ansible/config_sonic_basedon_testbed.yml index c469fb9f1ee..28a2c33c72f 100644 --- a/ansible/config_sonic_basedon_testbed.yml +++ b/ansible/config_sonic_basedon_testbed.yml @@ -40,13 +40,16 @@ set_fact: testbed_file: testbed.yaml when: testbed_file is not defined + tags: always - name: Gathering testbed information test_facts: testbed_name="{{ testbed_name }}" testbed_file="{{ testbed_file }}" delegate_to: localhost + tags: always - fail: msg="The DUT you are trying to run test does not belongs to this testbed" when: inventory_hostname not in testbed_facts['duts'] + tags: always - name: Set default num_asic set_fact: @@ -60,11 +63,13 @@ - name: set testbed_type set_fact: topo: "{{ testbed_facts['topo'] }}" + tags: always - name: set default light mode set_fact: is_light_mode: true when: topo in ["t1-smartswitch-ha","t1-28-lag","smartswitch-t1", "t1-48-lag"] and is_light_mode is not defined + tags: always - name: set ptf image name set_fact: @@ -756,7 +761,7 @@ macsec_profile: "{{ macsec_profile }}" num_asics: "{{ num_asics }}" become: true - when: "('t2' in topo) and (enable_macsec is defined)" + when: "('t2' in topo) and (macsec_profile is defined)" - name: Use minigraph case block: @@ -915,6 +920,7 @@ become: true # t1-28-lag is smartswitch topo only when: topo in ["t1-smartswitch-ha","t1-28-lag","smartswitch-t1", "t1-48-lag"] and is_light_mode|bool == true + tags: [ dpu_config ] - name: Load DPU config in smartswitch load_extra_dpu_config: @@ -924,6 +930,7 @@ become: true # t1-28-lag is smartswitch topo only when: topo in ["t1-smartswitch-ha","t1-28-lag","smartswitch-t1", "t1-48-lag"] and is_light_mode|bool == true + tags: [ dpu_config ] - name: Configure TACACS become: true diff --git a/ansible/group_vars/sonic/variables b/ansible/group_vars/sonic/variables index 257b7c50997..03d72accbbf 100644 --- a/ansible/group_vars/sonic/variables +++ b/ansible/group_vars/sonic/variables @@ -21,7 +21,7 @@ broadcom_th5_hwskus: ['Arista-7060X6-64DE', 'Arista-7060X6-64DE-64x400G', 'Arist broadcom_j2c+_hwskus: ['Nokia-IXR7250E-36x100G', 'Nokia-IXR7250E-36x400G', 'Arista-7280DR3A-36', 'Arista-7280DR3AK-36', 'Arista-7280DR3AK-36S', 'Arista-7280DR3AM-36', 'Arista-7800R3A-36DM2-C36', 'Arista-7800R3A-36DM2-D36', 'Arista-7800R3AK-36DM2-C36', 'Arista-7800R3AK-36DM2-D36', 'Nokia-IXR7250-X3B'] broadcom_jr2_hwskus: ['Arista-7800R3-48CQ2-C48', 'Arista-7800R3-48CQM2-C48'] -broadcom_q3d_hwskus: ['Arista-7280R4-32QF-32DF-64O', 'Arista-7280R4K-32QF-32DF-64O'] +broadcom_q3d_hwskus: ['NH-5010-F-O64', 'NH-5010-F-O32-C32', 'Arista-7280R4-32QF-32DF-64O', 'Arista-7280R4K-32QF-32DF-64O'] mellanox_spc1_hwskus: [ 'ACS-MSN2700', 'ACS-MSN2740', 'ACS-MSN2100', 'ACS-MSN2410', 'ACS-MSN2010', 'Mellanox-SN2700', 'Mellanox-SN2700-A1', 'Mellanox-SN2700-D48C8','Mellanox-SN2700-D40C8S8', 'Mellanox-SN2700-A1-D48C8', 'Mellanox-SN2700-C28D8', 'Mellanox-SN2700-A1-C28D8'] mellanox_spc2_hwskus: [ 'ACS-MSN3700', 'ACS-MSN3700C', 'ACS-MSN3800', 'Mellanox-SN3800-D112C8' , 'ACS-MSN3420'] diff --git a/ansible/library/dut_basic_facts.py b/ansible/library/dut_basic_facts.py index c5acb015162..d9654b0d9c3 100644 --- a/ansible/library/dut_basic_facts.py +++ b/ansible/library/dut_basic_facts.py @@ -51,6 +51,10 @@ def main(): if hasattr(device_info, 'is_chassis'): results['is_chassis'] = device_info.is_chassis() + results['is_chassis_config_absent'] = False + if hasattr(device_info, 'is_chassis_config_absent'): + results['is_chassis_config_absent'] = device_info.is_chassis_config_absent() + if results['is_multi_asic']: results['asic_index_list'] = [] if results['is_chassis']: diff --git a/ansible/roles/eos/templates/t2-single-node-max-64p-core.j2 b/ansible/roles/eos/templates/t2-single-node-max-64p-core.j2 new file mode 120000 index 00000000000..5fdcaec5931 --- /dev/null +++ b/ansible/roles/eos/templates/t2-single-node-max-64p-core.j2 @@ -0,0 +1 @@ +t2-core.j2 \ No newline at end of file diff --git a/ansible/roles/eos/templates/t2-single-node-max-64p-leaf.j2 b/ansible/roles/eos/templates/t2-single-node-max-64p-leaf.j2 new file mode 120000 index 00000000000..2556d9c538c --- /dev/null +++ b/ansible/roles/eos/templates/t2-single-node-max-64p-leaf.j2 @@ -0,0 +1 @@ +t2-leaf.j2 \ No newline at end of file diff --git a/ansible/roles/test/files/ptftests/py3/generic_hash_test.py b/ansible/roles/test/files/ptftests/py3/generic_hash_test.py index 824bfb824c0..a320aacc957 100644 --- a/ansible/roles/test/files/ptftests/py3/generic_hash_test.py +++ b/ansible/roles/test/files/ptftests/py3/generic_hash_test.py @@ -22,8 +22,8 @@ class GenericHashTest(BaseTest): # --------------------------------------------------------------------- # Class variables # --------------------------------------------------------------------- - DEFAULT_BALANCING_RANGE = 0.25 - BALANCING_TEST_TIMES = 625 + DEFAULT_BALANCING_RANGE = 0.3 + BALANCING_TEST_TIMES = 480 VXLAN_PORT = 4789 VXLAN_VNI = 20001 NVGRE_TNI = 20001 @@ -206,12 +206,12 @@ def _get_single_layer_packet(): if self.hash_field == 'ETHERTYPE': pkt['Ether'].type = random.choice(range(self.ethertype_range[0], self.ethertype_range[1])) if not self.is_l2_test: - pkt_summary = f"{self.ipver} packet with src_mac:{src_mac}, dst_mac:{dst_mac}, src_ip:{src_ip}, " \ - f"dst_ip:{dst_ip}, src_port:{src_port}, dst_port: {dst_port}, " \ - f"ip_protocol:{_get_pkt_ip_protocol(pkt)}" + pkt_summary = f"{self.ipver} packet with src_mac: {src_mac}, dst_mac: {dst_mac}, src_ip: {src_ip}, " \ + f"dst_ip: {dst_ip}, src_port: {src_port}, dst_port: {dst_port}, " \ + f"ip_protocol: {_get_pkt_ip_protocol(pkt)}" else: - pkt_summary = f"Ethernet packet with src_mac:{src_mac}, dst_mac:{dst_mac}, " \ - f"ether_type:{hex(pkt['Ether'].type)}, vlan_id:{vlan_id if vlan_id != 0 else 'N/A'}" + pkt_summary = f"Ethernet packet with src_mac: {src_mac}, dst_mac: {dst_mac}, " \ + f"ether_type: {hex(pkt['Ether'].type)}, vlan_id: {vlan_id if vlan_id != 0 else 'N/A'}" if self.hash_field == 'IPV6_FLOW_LABEL': pkt_summary += f", ipv6 flow label: {flow_label} " @@ -244,10 +244,10 @@ def _get_ipinip_packet(): masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "dst") masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "src") masked_expected_pkt.set_do_not_care_packet(scapy.IPv6, "hlim") - pkt_summary = f"{self.ipver} ipinip packet with src_ip:{src_ip}, dst_ip:{dst_ip}, " \ - f"ip_protocol:{_get_pkt_ip_protocol(pkt)}, inner_ipver:{self.inner_ipver}, " \ - f"inner_src_ip:{inner_src_ip}, inner_dst_ip:{inner_dst_ip}, inner_src_port:{src_port}," \ - f" inner_dst_port:{dst_port}, inner_ip_protocol:{_get_pkt_ip_protocol(inner_pkt)}" + pkt_summary = f"{self.ipver} ipinip packet with src_ip: {src_ip}, dst_ip: {dst_ip}, " \ + f"ip_protocol: {_get_pkt_ip_protocol(pkt)}, inner_ipver: {self.inner_ipver}, " \ + f"inner_src_ip: {inner_src_ip}, inner_dst_ip: {inner_dst_ip}, inner_src_port: {src_port}, " \ + f" inner_dst_port: {dst_port}, inner_ip_protocol: {_get_pkt_ip_protocol(inner_pkt)}" return pkt, masked_expected_pkt, pkt_summary def _get_vxlan_packet(): @@ -286,11 +286,12 @@ def _get_vxlan_packet(): masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "dst") masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "src") masked_expected_pkt.set_do_not_care_packet(scapy.IPv6, "hlim") - pkt_summary = f"{self.ipver} vxlan packet with src_ip:{src_ip}, dst_ip:{dst_ip}, " \ - f"src_port:{self.L4_SRC_PORT}, dst_port: {self.vxlan_port}, ip_protocol:{_get_pkt_ip_protocol(pkt)}, " \ - f"inner_ipver:{self.inner_ipver}, inner_src_ip:{inner_src_ip}, inner_dst_ip:{inner_dst_ip}, " \ - f"inner_src_port:{src_port}, inner_dst_port:{dst_port}, " \ - f"inner_ip_protocol:{_get_pkt_ip_protocol(inner_pkt)}" + pkt_summary = f"{self.ipver} vxlan packet with src_ip: {src_ip}, dst_ip: {dst_ip}, " \ + f"src_port: {self.L4_SRC_PORT}, dst_port: {self.vxlan_port}, " \ + f"ip_protocol: {_get_pkt_ip_protocol(pkt)}, " \ + f"inner_ipver: {self.inner_ipver}, inner_src_ip: {inner_src_ip}, inner_dst_ip: {inner_dst_ip}, " \ + f"inner_src_port: {src_port}, inner_dst_port: {dst_port}, " \ + f"inner_ip_protocol: {_get_pkt_ip_protocol(inner_pkt)}" return pkt, masked_expected_pkt, pkt_summary def _get_nvgre_packet(): @@ -325,10 +326,10 @@ def _get_nvgre_packet(): masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "dst") masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "src") masked_expected_pkt.set_do_not_care_packet(scapy.IPv6, "hlim") - pkt_summary = f"{self.ipver} nvgre packet with src_ip:{src_ip}, dst_ip:{dst_ip}, " \ - f"ip_protocol:{_get_pkt_ip_protocol(pkt)}, inner_ipver:{self.inner_ipver}, " \ - f"inner_src_ip:{inner_src_ip}, inner_dst_ip:{inner_dst_ip}, inner_src_port:{src_port}, " \ - f"inner_dst_port:{dst_port}, inner_ip_protocol:{_get_pkt_ip_protocol(inner_pkt)}" + pkt_summary = f"{self.ipver} nvgre packet with src_ip: {src_ip}, dst_ip: {dst_ip}, " \ + f"ip_protocol: {_get_pkt_ip_protocol(pkt)}, inner_ipver: {self.inner_ipver}, " \ + f"inner_src_ip: {inner_src_ip}, inner_dst_ip: {inner_dst_ip}, inner_src_port: {src_port}, " \ + f"inner_dst_port: {dst_port}, inner_ip_protocol: {_get_pkt_ip_protocol(inner_pkt)}" return pkt, masked_expected_pkt, pkt_summary src_mac = _get_src_mac() @@ -396,13 +397,14 @@ def check_ip_route(self, pkt, masked_expected_pkt, sending_port): port_index, received = testutils.verify_packet_any_port( self, masked_expected_pkt, self.expected_port_list, timeout=0.1) except AssertionError: - logging.error("Traffic wasn't sent successfully, trying again") + logging.warning("Packet wasn't received successfully, trying again") logging.info(f"Expected packet: {masked_expected_pkt}") for _ in range(5): testutils.send_packet(self, sending_port, pkt, count=1) time.sleep(0.1) port_index, received = testutils.verify_packet_any_port( self, masked_expected_pkt, self.expected_port_list, timeout=1) + logging.info("Packet was received successfully after retry.") # The port_index is the index of expected_port_list, need to convert it to the ptf port index return self.expected_port_list[port_index], received @@ -511,6 +513,31 @@ def _check_only_lag_hash_balancing(): elif not self.ecmp_hash and self.lag_hash: _check_only_lag_hash_balancing() + def print_result_summary(self, hit_count_map): + total_receiving_ports = len(hit_count_map) + total_received_packets = sum(hit_count_map.values()) + average_packets = total_received_packets / total_receiving_ports + min_received_packets = min(hit_count_map.values()) + max_received_packets = max(hit_count_map.values()) + min_received_ports = [port for port, count in hit_count_map.items() if count == min_received_packets] + max_received_ports = [port for port, count in hit_count_map.items() if count == max_received_packets] + expected_total_hit_cnt = self.balancing_test_times * len(self.expected_port_list) + expected_hit_cnt_per_port = self.balancing_test_times + if self.ecmp_hash and not self.lag_hash: + expected_hit_cnt_per_port = expected_total_hit_cnt / len(self.expected_port_groups) + elif not self.ecmp_hash and self.lag_hash: + expected_hit_cnt_per_port = expected_total_hit_cnt / len(self.expected_port_groups[0]) + max_deviation = max(abs(expected_hit_cnt_per_port - min_received_packets), + abs(expected_hit_cnt_per_port - max_received_packets)) + max_deviation_percentage = str(max_deviation / expected_hit_cnt_per_port * 100) + "%" + logging.info(f"\nTotal number of receiving ports: {total_receiving_ports}\n" + f"Total packet count: {total_received_packets}\n" + f"Average packet count: {average_packets}\n" + f"Min packets on a port: {min_received_packets}(Ports: {min_received_ports})\n" + f"Max packets on a port: {max_received_packets}(Ports: {max_received_ports})\n" + f"Max deviation: {max_deviation}\n" + f"Max deviation percentage: {max_deviation_percentage}\n") + def runTest(self): logging.info("=============Test Start==============") hit_count_map = {} @@ -544,9 +571,9 @@ def runTest(self): logging.info("Received packet at index {}: {}".format( str(matched_port), re.sub(r"(?<=\w)(?=(?:\w\w)+$)", ' ', received.hex()))) - time.sleep(0.02) hit_count_map[matched_port] = hit_count_map.get(matched_port, 0) + 1 logging.info(f"hash_field={self.hash_field}, hit count map: {hit_count_map}") + self.print_result_summary(hit_count_map) # Check if the traffic is properly balanced self.check_balancing(hit_count_map) diff --git a/ansible/templates/golden_config_db_t2.j2 b/ansible/templates/golden_config_db_t2.j2 index 45adcea4235..09096b26bc4 100644 --- a/ansible/templates/golden_config_db_t2.j2 +++ b/ansible/templates/golden_config_db_t2.j2 @@ -23,8 +23,7 @@ {% endif %} {%- endfor -%} {% else %} - { - "MACSEC_PROFILE": { + "MACSEC_PROFILE": { "{{macsec_profile}}": { "priority": "{{priority}}", "cipher_suite": "{{cipher_suite}}", @@ -34,6 +33,5 @@ "send_sci": "{{send_sci}}" } } - }, {%- endif -%} } diff --git a/ansible/vars/topo_t2_single_node_max_64p.yml b/ansible/vars/topo_t2_single_node_max_64p.yml new file mode 100644 index 00000000000..b1410b7a8f6 --- /dev/null +++ b/ansible/vars/topo_t2_single_node_max_64p.yml @@ -0,0 +1,1753 @@ +topology: + VMs: + VM01T3: + vlans: + - 0 + vm_offset: 0 + + VM02T3: + vlans: + - 1 + vm_offset: 1 + + VM03T3: + vlans: + - 2 + vm_offset: 2 + + VM04T3: + vlans: + - 3 + vm_offset: 3 + + VM05T3: + vlans: + - 4 + vm_offset: 4 + + VM06T3: + vlans: + - 5 + vm_offset: 5 + + VM07T3: + vlans: + - 6 + vm_offset: 6 + + VM08T3: + vlans: + - 7 + vm_offset: 7 + + VM09T3: + vlans: + - 8 + vm_offset: 8 + + VM10T3: + vlans: + - 9 + vm_offset: 9 + + VM11T3: + vlans: + - 10 + vm_offset: 10 + + VM12T3: + vlans: + - 11 + vm_offset: 11 + + VM13T3: + vlans: + - 12 + vm_offset: 12 + + VM14T3: + vlans: + - 13 + vm_offset: 13 + + VM15T3: + vlans: + - 14 + vm_offset: 14 + + VM16T3: + vlans: + - 15 + vm_offset: 15 + + VM17T3: + vlans: + - 16 + vm_offset: 16 + + VM18T3: + vlans: + - 17 + vm_offset: 17 + + VM19T3: + vlans: + - 18 + vm_offset: 18 + + VM20T3: + vlans: + - 19 + vm_offset: 19 + + VM21T3: + vlans: + - 20 + vm_offset: 20 + + VM22T3: + vlans: + - 21 + vm_offset: 21 + + VM23T3: + vlans: + - 22 + vm_offset: 22 + + VM24T3: + vlans: + - 23 + vm_offset: 23 + + VM25T3: + vlans: + - 24 + vm_offset: 24 + + VM26T3: + vlans: + - 25 + vm_offset: 25 + + VM27T3: + vlans: + - 26 + vm_offset: 26 + + VM28T3: + vlans: + - 27 + vm_offset: 27 + + VM29T3: + vlans: + - 28 + vm_offset: 28 + + VM30T3: + vlans: + - 29 + vm_offset: 29 + + VM31T3: + vlans: + - 30 + vm_offset: 30 + + VM32T3: + vlans: + - 31 + vm_offset: 31 + + VM01LT2: + vlans: + - 32 + vm_offset: 32 + + VM02LT2: + vlans: + - 33 + vm_offset: 33 + + VM03LT2: + vlans: + - 34 + vm_offset: 34 + + VM04LT2: + vlans: + - 35 + vm_offset: 35 + + VM05LT2: + vlans: + - 36 + vm_offset: 36 + + VM06LT2: + vlans: + - 37 + vm_offset: 37 + + VM07LT2: + vlans: + - 38 + vm_offset: 38 + + VM08LT2: + vlans: + - 39 + vm_offset: 39 + + VM09LT2: + vlans: + - 40 + vm_offset: 40 + + VM10LT2: + vlans: + - 41 + vm_offset: 41 + + VM11LT2: + vlans: + - 42 + vm_offset: 42 + + VM12LT2: + vlans: + - 43 + vm_offset: 43 + + VM13LT2: + vlans: + - 44 + vm_offset: 44 + + VM14LT2: + vlans: + - 45 + vm_offset: 45 + + VM15LT2: + vlans: + - 46 + vm_offset: 46 + + VM16LT2: + vlans: + - 47 + vm_offset: 47 + + VM17LT2: + vlans: + - 48 + vm_offset: 48 + + VM18LT2: + vlans: + - 49 + vm_offset: 49 + + VM19LT2: + vlans: + - 50 + vm_offset: 50 + + VM20LT2: + vlans: + - 51 + vm_offset: 51 + + VM21LT2: + vlans: + - 52 + vm_offset: 52 + + VM22LT2: + vlans: + - 53 + vm_offset: 53 + + VM23LT2: + vlans: + - 54 + vm_offset: 54 + + VM24LT2: + vlans: + - 55 + vm_offset: 55 + + VM25LT2: + vlans: + - 56 + vm_offset: 56 + + VM26LT2: + vlans: + - 57 + vm_offset: 57 + + VM27LT2: + vlans: + - 58 + vm_offset: 58 + + VM28LT2: + vlans: + - 59 + vm_offset: 59 + + VM29LT2: + vlans: + - 60 + vm_offset: 60 + + VM30LT2: + vlans: + - 61 + vm_offset: 61 + + VM31LT2: + vlans: + - 62 + vm_offset: 62 + + VM32LT2: + vlans: + - 63 + vm_offset: 63 + + DUT: + loopback: + ipv4: + - 10.1.0.1/32 + ipv6: + - fc00:10::1/128 + +configuration_properties: + common: + dut_asn: 65100 + dut_type: SpineRouter + podset_number: 400 + tor_number: 16 + tor_subnet_number: 8 + max_tor_subnet_number: 32 + tor_subnet_size: 128 + nhipv4: 10.10.48.254 + nhipv6: fc30::ff + core: + swrole: core + leaf: + swrole: leaf + +configuration: + VM01T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.0 + - fc00::1 + interfaces: + Loopback0: + ipv4: 100.1.0.1/32 + ipv6: 2064:100::1/128 + Port-Channel1: + ipv4: 10.0.0.1/31 + ipv6: fc00::2/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.1/24 + ipv6: fc30::2/64 + + VM02T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.2 + - fc00::5 + interfaces: + Loopback0: + ipv4: 100.1.0.2/32 + ipv6: 2064:100::2/128 + Port-Channel1: + ipv4: 10.0.0.3/31 + ipv6: fc00::6/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.2/24 + ipv6: fc30::3/64 + + VM03T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.4 + - fc00::9 + interfaces: + Loopback0: + ipv4: 100.1.0.3/32 + ipv6: 2064:100::3/128 + Port-Channel1: + ipv4: 10.0.0.5/31 + ipv6: fc00::a/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.3/24 + ipv6: fc30::4/64 + + VM04T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.6 + - fc00::d + interfaces: + Loopback0: + ipv4: 100.1.0.4/32 + ipv6: 2064:100::4/128 + Port-Channel1: + ipv4: 10.0.0.7/31 + ipv6: fc00::e/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.4/24 + ipv6: fc30::5/64 + + VM05T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.8 + - fc00::11 + interfaces: + Loopback0: + ipv4: 100.1.0.5/32 + ipv6: 2064:100::5/128 + Port-Channel1: + ipv4: 10.0.0.9/31 + ipv6: fc00::12/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.5/24 + ipv6: fc30::6/64 + + VM06T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.10 + - fc00::15 + interfaces: + Loopback0: + ipv4: 100.1.0.6/32 + ipv6: 2064:100::6/128 + Port-Channel1: + ipv4: 10.0.0.11/31 + ipv6: fc00::16/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.6/24 + ipv6: fc30::7/64 + + VM07T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.12 + - fc00::19 + interfaces: + Loopback0: + ipv4: 100.1.0.7/32 + ipv6: 2064:100::7/128 + Port-Channel1: + ipv4: 10.0.0.13/31 + ipv6: fc00::1a/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.7/24 + ipv6: fc30::8/64 + + VM08T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.14 + - fc00::1d + interfaces: + Loopback0: + ipv4: 100.1.0.8/32 + ipv6: 2064:100::8/128 + Port-Channel1: + ipv4: 10.0.0.15/31 + ipv6: fc00::1e/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.8/24 + ipv6: fc30::9/64 + + VM09T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.16 + - fc00::21 + interfaces: + Loopback0: + ipv4: 100.1.0.9/32 + ipv6: 2064:100::9/128 + Port-Channel1: + ipv4: 10.0.0.17/31 + ipv6: fc00::22/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.9/24 + ipv6: fc30::a/64 + + VM10T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.18 + - fc00::25 + interfaces: + Loopback0: + ipv4: 100.1.0.10/32 + ipv6: 2064:100::a/128 + Port-Channel1: + ipv4: 10.0.0.19/31 + ipv6: fc00::26/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.10/24 + ipv6: fc30::b/64 + + VM11T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.20 + - fc00::29 + interfaces: + Loopback0: + ipv4: 100.1.0.11/32 + ipv6: 2064:100::b/128 + Port-Channel1: + ipv4: 10.0.0.21/31 + ipv6: fc00::2a/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.11/24 + ipv6: fc30::c/64 + + VM12T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.22 + - fc00::2d + interfaces: + Loopback0: + ipv4: 100.1.0.12/32 + ipv6: 2064:100::c/128 + Port-Channel1: + ipv4: 10.0.0.23/31 + ipv6: fc00::2e/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.12/24 + ipv6: fc30::d/64 + + VM13T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.24 + - fc00::31 + interfaces: + Loopback0: + ipv4: 100.1.0.13/32 + ipv6: 2064:100::d/128 + Port-Channel1: + ipv4: 10.0.0.25/31 + ipv6: fc00::32/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.13/24 + ipv6: fc30::e/64 + + VM14T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.26 + - fc00::35 + interfaces: + Loopback0: + ipv4: 100.1.0.14/32 + ipv6: 2064:100::e/128 + Port-Channel1: + ipv4: 10.0.0.27/31 + ipv6: fc00::36/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.14/24 + ipv6: fc30::f/64 + + VM15T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.28 + - fc00::39 + interfaces: + Loopback0: + ipv4: 100.1.0.15/32 + ipv6: 2064:100::f/128 + Port-Channel1: + ipv4: 10.0.0.29/31 + ipv6: fc00::3a/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.15/24 + ipv6: fc30::10/64 + + VM16T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.30 + - fc00::3d + interfaces: + Loopback0: + ipv4: 100.1.0.16/32 + ipv6: 2064:100::10/128 + Port-Channel1: + ipv4: 10.0.0.31/31 + ipv6: fc00::3e/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.16/24 + ipv6: fc30::11/64 + + VM17T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.32 + - fc00::41 + interfaces: + Loopback0: + ipv4: 100.1.0.17/32 + ipv6: 2064:100::11/128 + Port-Channel1: + ipv4: 10.0.0.33/31 + ipv6: fc00::42/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.17/24 + ipv6: fc30::12/64 + + VM18T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.34 + - fc00::45 + interfaces: + Loopback0: + ipv4: 100.1.0.18/32 + ipv6: 2064:100::12/128 + Port-Channel1: + ipv4: 10.0.0.35/31 + ipv6: fc00::46/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.18/24 + ipv6: fc30::13/64 + + VM19T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.36 + - fc00::49 + interfaces: + Loopback0: + ipv4: 100.1.0.19/32 + ipv6: 2064:100::13/128 + Port-Channel1: + ipv4: 10.0.0.37/31 + ipv6: fc00::4a/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.19/24 + ipv6: fc30::14/64 + + VM20T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.38 + - fc00::4d + interfaces: + Loopback0: + ipv4: 100.1.0.20/32 + ipv6: 2064:100::14/128 + Port-Channel1: + ipv4: 10.0.0.39/31 + ipv6: fc00::4e/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.20/24 + ipv6: fc30::15/64 + + VM21T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.40 + - fc00::51 + interfaces: + Loopback0: + ipv4: 100.1.0.21/32 + ipv6: 2064:100::15/128 + Port-Channel1: + ipv4: 10.0.0.41/31 + ipv6: fc00::52/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.21/24 + ipv6: fc30::16/64 + + VM22T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.42 + - fc00::55 + interfaces: + Loopback0: + ipv4: 100.1.0.22/32 + ipv6: 2064:100::16/128 + Port-Channel1: + ipv4: 10.0.0.43/31 + ipv6: fc00::56/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.22/24 + ipv6: fc30::17/64 + + VM23T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.44 + - fc00::59 + interfaces: + Loopback0: + ipv4: 100.1.0.23/32 + ipv6: 2064:100::17/128 + Port-Channel1: + ipv4: 10.0.0.45/31 + ipv6: fc00::5a/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.23/24 + ipv6: fc30::18/64 + + VM24T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.46 + - fc00::5d + interfaces: + Loopback0: + ipv4: 100.1.0.24/32 + ipv6: 2064:100::18/128 + Port-Channel1: + ipv4: 10.0.0.47/31 + ipv6: fc00::5e/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.24/24 + ipv6: fc30::19/64 + + VM25T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.48 + - fc00::61 + interfaces: + Loopback0: + ipv4: 100.1.0.25/32 + ipv6: 2064:100::19/128 + Port-Channel1: + ipv4: 10.0.0.49/31 + ipv6: fc00::62/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.25/24 + ipv6: fc30::1a/64 + + VM26T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.50 + - fc00::65 + interfaces: + Loopback0: + ipv4: 100.1.0.26/32 + ipv6: 2064:100::1a/128 + Port-Channel1: + ipv4: 10.0.0.51/31 + ipv6: fc00::66/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.26/24 + ipv6: fc30::1b/64 + + VM27T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.52 + - fc00::69 + interfaces: + Loopback0: + ipv4: 100.1.0.27/32 + ipv6: 2064:100::1b/128 + Port-Channel1: + ipv4: 10.0.0.53/31 + ipv6: fc00::6a/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.27/24 + ipv6: fc30::1c/64 + + VM28T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.54 + - fc00::6d + interfaces: + Loopback0: + ipv4: 100.1.0.28/32 + ipv6: 2064:100::1c/128 + Port-Channel1: + ipv4: 10.0.0.55/31 + ipv6: fc00::6e/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.28/24 + ipv6: fc30::1d/64 + + VM29T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.56 + - fc00::71 + interfaces: + Loopback0: + ipv4: 100.1.0.29/32 + ipv6: 2064:100::1d/128 + Port-Channel1: + ipv4: 10.0.0.57/31 + ipv6: fc00::72/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.29/24 + ipv6: fc30::1e/64 + + VM30T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.58 + - fc00::75 + interfaces: + Loopback0: + ipv4: 100.1.0.30/32 + ipv6: 2064:100::1e/128 + Port-Channel1: + ipv4: 10.0.0.59/31 + ipv6: fc00::76/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.30/24 + ipv6: fc30::1f/64 + + VM31T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.60 + - fc00::79 + interfaces: + Loopback0: + ipv4: 100.1.0.31/32 + ipv6: 2064:100::1f/128 + Port-Channel1: + ipv4: 10.0.0.61/31 + ipv6: fc00::7a/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.31/24 + ipv6: fc30::20/64 + + VM32T3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.0.0.62 + - fc00::7d + interfaces: + Loopback0: + ipv4: 100.1.0.32/32 + ipv6: 2064:100::20/128 + Port-Channel1: + ipv4: 10.0.0.63/31 + ipv6: fc00::7e/126 + Ethernet1: + lacp: 1 + bp_interface: + ipv4: 10.10.48.32/24 + ipv6: fc30::21/64 + + VM01LT2: + properties: + - common + - leaf + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.128 + - fc00::101 + interfaces: + Loopback0: + ipv4: 100.1.0.65/32 + ipv6: 2064:100::41/128 + Ethernet1: + ipv4: 10.0.0.129/31 + ipv6: fc00::102/126 + bp_interface: + ipv4: 10.10.48.65/24 + ipv6: fc30::42/64 + + VM02LT2: + properties: + - common + - leaf + bgp: + asn: 64610 + peers: + 65100: + - 10.0.0.130 + - fc00::105 + interfaces: + Loopback0: + ipv4: 100.1.0.66/32 + ipv6: 2064:100::42/128 + Ethernet1: + ipv4: 10.0.0.131/31 + ipv6: fc00::106/126 + bp_interface: + ipv4: 10.10.48.66/24 + ipv6: fc30::43/64 + + VM03LT2: + properties: + - common + - leaf + bgp: + asn: 64620 + peers: + 65100: + - 10.0.0.132 + - fc00::109 + interfaces: + Loopback0: + ipv4: 100.1.0.67/32 + ipv6: 2064:100::43/128 + Ethernet1: + ipv4: 10.0.0.133/31 + ipv6: fc00::10a/126 + bp_interface: + ipv4: 10.10.48.67/24 + ipv6: fc30::44/64 + + VM04LT2: + properties: + - common + - leaf + bgp: + asn: 64630 + peers: + 65100: + - 10.0.0.134 + - fc00::10d + interfaces: + Loopback0: + ipv4: 100.1.0.68/32 + ipv6: 2064:100::44/128 + Ethernet1: + ipv4: 10.0.0.135/31 + ipv6: fc00::10e/126 + bp_interface: + ipv4: 10.10.48.68/24 + ipv6: fc30::45/64 + + VM05LT2: + properties: + - common + - leaf + bgp: + asn: 64640 + peers: + 65100: + - 10.0.0.136 + - fc00::111 + interfaces: + Loopback0: + ipv4: 100.1.0.69/32 + ipv6: 2064:100::45/128 + Ethernet1: + ipv4: 10.0.0.137/31 + ipv6: fc00::112/126 + bp_interface: + ipv4: 10.10.48.69/24 + ipv6: fc30::46/64 + + VM06LT2: + properties: + - common + - leaf + bgp: + asn: 64650 + peers: + 65100: + - 10.0.0.138 + - fc00::115 + interfaces: + Loopback0: + ipv4: 100.1.0.70/32 + ipv6: 2064:100::46/128 + Ethernet1: + ipv4: 10.0.0.139/31 + ipv6: fc00::116/126 + bp_interface: + ipv4: 10.10.48.70/24 + ipv6: fc30::47/64 + + VM07LT2: + properties: + - common + - leaf + bgp: + asn: 64660 + peers: + 65100: + - 10.0.0.140 + - fc00::119 + interfaces: + Loopback0: + ipv4: 100.1.0.71/32 + ipv6: 2064:100::47/128 + Ethernet1: + ipv4: 10.0.0.141/31 + ipv6: fc00::11a/126 + bp_interface: + ipv4: 10.10.48.71/24 + ipv6: fc30::48/64 + + VM08LT2: + properties: + - common + - leaf + bgp: + asn: 64670 + peers: + 65100: + - 10.0.0.142 + - fc00::11d + interfaces: + Loopback0: + ipv4: 100.1.0.72/32 + ipv6: 2064:100::48/128 + Ethernet1: + ipv4: 10.0.0.143/31 + ipv6: fc00::11e/126 + bp_interface: + ipv4: 10.10.48.72/24 + ipv6: fc30::49/64 + + VM09LT2: + properties: + - common + - leaf + bgp: + asn: 64680 + peers: + 65100: + - 10.0.0.144 + - fc00::121 + interfaces: + Loopback0: + ipv4: 100.1.0.73/32 + ipv6: 2064:100::49/128 + Ethernet1: + ipv4: 10.0.0.145/31 + ipv6: fc00::122/126 + bp_interface: + ipv4: 10.10.48.73/24 + ipv6: fc30::4a/64 + + VM10LT2: + properties: + - common + - leaf + bgp: + asn: 64690 + peers: + 65100: + - 10.0.0.146 + - fc00::125 + interfaces: + Loopback0: + ipv4: 100.1.0.74/32 + ipv6: 2064:100::4a/128 + Ethernet1: + ipv4: 10.0.0.147/31 + ipv6: fc00::126/126 + bp_interface: + ipv4: 10.10.48.74/24 + ipv6: fc30::4b/64 + + VM11LT2: + properties: + - common + - leaf + bgp: + asn: 64700 + peers: + 65100: + - 10.0.0.148 + - fc00::129 + interfaces: + Loopback0: + ipv4: 100.1.0.75/32 + ipv6: 2064:100::4b/128 + Ethernet1: + ipv4: 10.0.0.149/31 + ipv6: fc00::12a/126 + bp_interface: + ipv4: 10.10.48.75/24 + ipv6: fc30::4c/64 + + VM12LT2: + properties: + - common + - leaf + bgp: + asn: 64710 + peers: + 65100: + - 10.0.0.150 + - fc00::12d + interfaces: + Loopback0: + ipv4: 100.1.0.76/32 + ipv6: 2064:100::4c/128 + Ethernet1: + ipv4: 10.0.0.151/31 + ipv6: fc00::12e/126 + bp_interface: + ipv4: 10.10.48.76/24 + ipv6: fc30::4d/64 + + VM13LT2: + properties: + - common + - leaf + bgp: + asn: 64720 + peers: + 65100: + - 10.0.0.152 + - fc00::131 + interfaces: + Loopback0: + ipv4: 100.1.0.77/32 + ipv6: 2064:100::4d/128 + Ethernet1: + ipv4: 10.0.0.153/31 + ipv6: fc00::132/126 + bp_interface: + ipv4: 10.10.48.77/24 + ipv6: fc30::4e/64 + + VM14LT2: + properties: + - common + - leaf + bgp: + asn: 64730 + peers: + 65100: + - 10.0.0.154 + - fc00::135 + interfaces: + Loopback0: + ipv4: 100.1.0.78/32 + ipv6: 2064:100::4e/128 + Ethernet1: + ipv4: 10.0.0.155/31 + ipv6: fc00::136/126 + bp_interface: + ipv4: 10.10.48.78/24 + ipv6: fc30::4f/64 + + VM15LT2: + properties: + - common + - leaf + bgp: + asn: 64740 + peers: + 65100: + - 10.0.0.156 + - fc00::139 + interfaces: + Loopback0: + ipv4: 100.1.0.79/32 + ipv6: 2064:100::4f/128 + Ethernet1: + ipv4: 10.0.0.157/31 + ipv6: fc00::13a/126 + bp_interface: + ipv4: 10.10.48.79/24 + ipv6: fc30::50/64 + + VM16LT2: + properties: + - common + - leaf + bgp: + asn: 64750 + peers: + 65100: + - 10.0.0.158 + - fc00::13d + interfaces: + Loopback0: + ipv4: 100.1.0.80/32 + ipv6: 2064:100::50/128 + Ethernet1: + ipv4: 10.0.0.159/31 + ipv6: fc00::13e/126 + bp_interface: + ipv4: 10.10.48.80/24 + ipv6: fc30::51/64 + + VM17LT2: + properties: + - common + - leaf + bgp: + asn: 64760 + peers: + 65100: + - 10.0.0.160 + - fc00::141 + interfaces: + Loopback0: + ipv4: 100.1.0.81/32 + ipv6: 2064:100::51/128 + Ethernet1: + ipv4: 10.0.0.161/31 + ipv6: fc00::142/126 + bp_interface: + ipv4: 10.10.48.81/24 + ipv6: fc30::52/64 + + VM18LT2: + properties: + - common + - leaf + bgp: + asn: 64770 + peers: + 65100: + - 10.0.0.162 + - fc00::145 + interfaces: + Loopback0: + ipv4: 100.1.0.82/32 + ipv6: 2064:100::52/128 + Ethernet1: + ipv4: 10.0.0.163/31 + ipv6: fc00::146/126 + bp_interface: + ipv4: 10.10.48.82/24 + ipv6: fc30::53/64 + + VM19LT2: + properties: + - common + - leaf + bgp: + asn: 64780 + peers: + 65100: + - 10.0.0.164 + - fc00::149 + interfaces: + Loopback0: + ipv4: 100.1.0.83/32 + ipv6: 2064:100::53/128 + Ethernet1: + ipv4: 10.0.0.165/31 + ipv6: fc00::14a/126 + bp_interface: + ipv4: 10.10.48.83/24 + ipv6: fc30::54/64 + + VM20LT2: + properties: + - common + - leaf + bgp: + asn: 64790 + peers: + 65100: + - 10.0.0.166 + - fc00::14d + interfaces: + Loopback0: + ipv4: 100.1.0.84/32 + ipv6: 2064:100::54/128 + Ethernet1: + ipv4: 10.0.0.167/31 + ipv6: fc00::14e/126 + bp_interface: + ipv4: 10.10.48.84/24 + ipv6: fc30::55/64 + + VM21LT2: + properties: + - common + - leaf + bgp: + asn: 64800 + peers: + 65100: + - 10.0.0.168 + - fc00::151 + interfaces: + Loopback0: + ipv4: 100.1.0.85/32 + ipv6: 2064:100::55/128 + Ethernet1: + ipv4: 10.0.0.169/31 + ipv6: fc00::152/126 + bp_interface: + ipv4: 10.10.48.85/24 + ipv6: fc30::56/64 + + VM22LT2: + properties: + - common + - leaf + bgp: + asn: 64810 + peers: + 65100: + - 10.0.0.170 + - fc00::155 + interfaces: + Loopback0: + ipv4: 100.1.0.86/32 + ipv6: 2064:100::56/128 + Ethernet1: + ipv4: 10.0.0.171/31 + ipv6: fc00::156/126 + bp_interface: + ipv4: 10.10.48.86/24 + ipv6: fc30::57/64 + + VM23LT2: + properties: + - common + - leaf + bgp: + asn: 64820 + peers: + 65100: + - 10.0.0.172 + - fc00::159 + interfaces: + Loopback0: + ipv4: 100.1.0.87/32 + ipv6: 2064:100::57/128 + Ethernet1: + ipv4: 10.0.0.173/31 + ipv6: fc00::15a/126 + bp_interface: + ipv4: 10.10.48.87/24 + ipv6: fc30::58/64 + + VM24LT2: + properties: + - common + - leaf + bgp: + asn: 64830 + peers: + 65100: + - 10.0.0.174 + - fc00::15d + interfaces: + Loopback0: + ipv4: 100.1.0.88/32 + ipv6: 2064:100::58/128 + Ethernet1: + ipv4: 10.0.0.175/31 + ipv6: fc00::15e/126 + bp_interface: + ipv4: 10.10.48.88/24 + ipv6: fc30::59/64 + + VM25LT2: + properties: + - common + - leaf + bgp: + asn: 64840 + peers: + 65100: + - 10.0.0.176 + - fc00::161 + interfaces: + Loopback0: + ipv4: 100.1.0.89/32 + ipv6: 2064:100::59/128 + Ethernet1: + ipv4: 10.0.0.177/31 + ipv6: fc00::162/126 + bp_interface: + ipv4: 10.10.48.89/24 + ipv6: fc30::5a/64 + + VM26LT2: + properties: + - common + - leaf + bgp: + asn: 64850 + peers: + 65100: + - 10.0.0.178 + - fc00::165 + interfaces: + Loopback0: + ipv4: 100.1.0.90/32 + ipv6: 2064:100::5a/128 + Ethernet1: + ipv4: 10.0.0.179/31 + ipv6: fc00::166/126 + bp_interface: + ipv4: 10.10.48.90/24 + ipv6: fc30::5b/64 + + VM27LT2: + properties: + - common + - leaf + bgp: + asn: 64860 + peers: + 65100: + - 10.0.0.180 + - fc00::169 + interfaces: + Loopback0: + ipv4: 100.1.0.91/32 + ipv6: 2064:100::5b/128 + Ethernet1: + ipv4: 10.0.0.181/31 + ipv6: fc00::16a/126 + bp_interface: + ipv4: 10.10.48.91/24 + ipv6: fc30::5c/64 + + VM28LT2: + properties: + - common + - leaf + bgp: + asn: 64870 + peers: + 65100: + - 10.0.0.182 + - fc00::16d + interfaces: + Loopback0: + ipv4: 100.1.0.92/32 + ipv6: 2064:100::5c/128 + Ethernet1: + ipv4: 10.0.0.183/31 + ipv6: fc00::16e/126 + bp_interface: + ipv4: 10.10.48.92/24 + ipv6: fc30::5d/64 + + VM29LT2: + properties: + - common + - leaf + bgp: + asn: 64880 + peers: + 65100: + - 10.0.0.184 + - fc00::171 + interfaces: + Loopback0: + ipv4: 100.1.0.93/32 + ipv6: 2064:100::5d/128 + Ethernet1: + ipv4: 10.0.0.185/31 + ipv6: fc00::172/126 + bp_interface: + ipv4: 10.10.48.93/24 + ipv6: fc30::5e/64 + + VM30LT2: + properties: + - common + - leaf + bgp: + asn: 64890 + peers: + 65100: + - 10.0.0.186 + - fc00::175 + interfaces: + Loopback0: + ipv4: 100.1.0.94/32 + ipv6: 2064:100::5e/128 + Ethernet1: + ipv4: 10.0.0.187/31 + ipv6: fc00::176/126 + bp_interface: + ipv4: 10.10.48.94/24 + ipv6: fc30::5f/64 + + VM31LT2: + properties: + - common + - leaf + bgp: + asn: 64900 + peers: + 65100: + - 10.0.0.188 + - fc00::179 + interfaces: + Loopback0: + ipv4: 100.1.0.95/32 + ipv6: 2064:100::5f/128 + Ethernet1: + ipv4: 10.0.0.189/31 + ipv6: fc00::17a/126 + bp_interface: + ipv4: 10.10.48.95/24 + ipv6: fc30::60/64 + + VM32LT2: + properties: + - common + - leaf + bgp: + asn: 64910 + peers: + 65100: + - 10.0.0.190 + - fc00::17d + interfaces: + Loopback0: + ipv4: 100.1.0.96/32 + ipv6: 2064:100::60/128 + Ethernet1: + ipv4: 10.0.0.191/31 + ipv6: fc00::17e/126 + bp_interface: + ipv4: 10.10.48.96/24 + ipv6: fc30::61/64 diff --git a/tests/bgp/conftest.py b/tests/bgp/conftest.py index 92e4d86bbe9..2a3ad6512af 100644 --- a/tests/bgp/conftest.py +++ b/tests/bgp/conftest.py @@ -11,6 +11,7 @@ import socket from jinja2 import Template +from tests.bgp.constants import SHOW_IP_INTERFACE_CMD from tests.common.helpers.assertions import pytest_assert as pt_assert from tests.common.helpers.generators import generate_ips from tests.common.helpers.parallel import parallel_run @@ -827,3 +828,13 @@ def traffic_shift_community(duthost): @pytest.fixture(scope='module') def get_function_completeness_level(pytestconfig): return pytestconfig.getoption("--completeness_level") + + +@pytest.fixture(scope='module') +def ip_version(tbinfo): + return 'v6' if is_ipv6_only_topology(tbinfo) else 'v4' + + +@pytest.fixture(scope='module') +def show_ip_interface_cmd(ip_version): + return SHOW_IP_INTERFACE_CMD[ip_version] diff --git a/tests/bgp/constants.py b/tests/bgp/constants.py index 57a06f5f09d..a23322ef910 100644 --- a/tests/bgp/constants.py +++ b/tests/bgp/constants.py @@ -4,3 +4,4 @@ TS_INCONSISTENT = "System Mode: Not consistent" TS_NO_NEIGHBORS = "System Mode: No external neighbors" TS_UNEXPECTED = "TSC not consistent across asic namespaces" +SHOW_IP_INTERFACE_CMD = {'v4': "show ip interface", 'v6': "show ipv6 interface"} diff --git a/tests/bgp/test_bgp_queue.py b/tests/bgp/test_bgp_queue.py index 4b4c6fe1d14..a87b0470f0a 100644 --- a/tests/bgp/test_bgp_queue.py +++ b/tests/bgp/test_bgp_queue.py @@ -14,19 +14,38 @@ def clear_queue_counters(asichost): asichost.command("sonic-clear queuecounters") -def get_queue_counters(asichost, port, queue): +def get_all_ports_queue_counters(asichost, queue_type_prefix="UC"): """ - Return the counter for a given queue in given port + Fetch queue counters for ALL ports in a single command. + Returns a dict: {port_name: {queue_num: count, ...}, ...} + Example: {'Ethernet0': {0: 0, 1: 0, ...}, 'Ethernet4': {0: 0, 1: 0, ...}} """ - cmd = "show queue counters {}".format(port) + cmd = "show queue counters" output = asichost.command(cmd, new_format=True)['stdout_lines'] - - txq = "UC{}".format(queue) + counters = {} for line in output: fields = line.split() - if fields[1] == txq: - return int(fields[2].replace(',', '')) - return -1 + if len(fields) < 3: + continue + port_name = fields[0] + queue_type = fields[1] + if port_name not in counters: + counters[port_name] = {} + if queue_type.startswith(queue_type_prefix): + try: + queue_num = int(queue_type[len(queue_type_prefix):]) + counters[port_name][queue_num] = int(fields[2].replace(',', '')) + except (ValueError, IndexError): + continue + return counters + + +def assert_queue_counter_zero(queue_counters, port_name, queue_start=0, queue_end=6): + for q in range(queue_start, queue_end + 1): + counter_value = queue_counters.get(q, -1) + assert counter_value == 0, ( + "Queue counter for port '{}' queue {} is not zero. Value: {}" + ).format(port_name, q, counter_value) def test_bgp_queues(duthosts, enum_frontend_dut_hostname, enum_asic_index, tbinfo): @@ -37,6 +56,10 @@ def test_bgp_queues(duthosts, enum_frontend_dut_hostname, enum_asic_index, tbinf bgp_facts = duthost.bgp_facts(instance_id=enum_asic_index)['ansible_facts'] mg_facts = asichost.get_extended_minigraph_facts(tbinfo) + all_ports_queue_counters = get_all_ports_queue_counters(asichost, queue_type_prefix="UC") + if not all_ports_queue_counters: + pytest.skip("No queue counters found on the device.") + arp_dict = {} ndp_dict = {} processed_intfs = set() @@ -58,15 +81,16 @@ def test_bgp_queues(duthosts, enum_frontend_dut_hostname, enum_asic_index, tbinf ndp_dict[ip] = iface for k, v in list(bgp_facts['bgp_neighbors'].items()): + # For "peer group" if it's internal it will be "INTERNAL_PEER_V4" or "INTERNAL_PEER_V6" + # or "VOQ_CHASSIS_PEER_V4" or "VOQ_CHASSIS_PEER_V6" for VOQ_CHASSIS + # If it's external it will be "RH_V4", "RH_V6", "AH_V4", "AH_V6", ... + # Skip internal neighbors for VOQ_CHASSIS until BRCM fixes iBGP traffic in 2024011 + if ("INTERNAL" in v["peer group"] or 'VOQ_CHASSIS' in v["peer group"]): + # Skip iBGP neighbors since we only want to verify eBGP + continue # Only consider established bgp sessions if v['state'] == 'established': - # For "peer group" if it's internal it will be "INTERNAL_PEER_V4" or "INTERNAL_PEER_V6" - # or "VOQ_CHASSIS_PEER_V4" or "VOQ_CHASSIS_PEER_V6" for VOQ_CHASSIS - # If it's external it will be "RH_V4", "RH_V6", "AH_V4", "AH_V6", ... - # Skip internal neighbors for VOQ_CHASSIS until BRCM fixes iBGP traffic in 2024011 - if ("INTERNAL" in v["peer group"] or 'VOQ_CHASSIS' in v["peer group"]): - # Skip iBGP neighbors since we only want to verify eBGP - continue + assert (k in arp_dict.keys() or k in ndp_dict.keys()), ( "BGP neighbor IP '{}' not found in either ARP or NDP tables.\n" "- ARP table: {}\n" @@ -79,22 +103,19 @@ def test_bgp_queues(duthosts, enum_frontend_dut_hostname, enum_asic_index, tbinf ifname = ndp_dict[k].split('.', 1)[0] if ifname in processed_intfs: continue - if (ifname.startswith("PortChannel")): + if ifname.startswith("PortChannel"): for port in mg_facts['minigraph_portchannels'][ifname]['members']: logger.info("PortChannel '{}' : port {}".format(ifname, port)) - for q in range(0, 7): - assert (get_queue_counters(asichost, port, q) == 0), ( - ( - "Queue counter for port '{}' queue {} is not zero after clearing queue counters. " - "Counter value: {}" - ).format(port, q, get_queue_counters(asichost, port, q)) - ) + per_port_queue_counters = all_ports_queue_counters.get(port, {}) + if not per_port_queue_counters: + logger.warning("No queue counters found for port '{}'".format(port)) + else: + assert_queue_counter_zero(per_port_queue_counters, port, 0, 6) else: logger.info(ifname) - for q in range(0, 7): - assert (get_queue_counters(asichost, ifname, q) == 0), ( - "Queue counter for interface '{}' queue {} is not zero after clearing queue counters. " - "Counter value: {}" - ).format(ifname, q, get_queue_counters(asichost, ifname, q)) - + per_iface_queue_counters = all_ports_queue_counters.get(ifname, {}) + if not per_iface_queue_counters: + logger.warning("No queue counters found for interface '{}'".format(ifname)) + else: + assert_queue_counter_zero(per_iface_queue_counters, ifname, 0, 6) processed_intfs.add(ifname) diff --git a/tests/bgp/test_bgp_session.py b/tests/bgp/test_bgp_session.py index f648d99564b..cee9465b3e1 100644 --- a/tests/bgp/test_bgp_session.py +++ b/tests/bgp/test_bgp_session.py @@ -222,6 +222,7 @@ def test_bgp_session_interface_down(duthosts, rand_one_dut_hostname, fanouthosts time.sleep(1) duthost.shell('show ip bgp summary', module_ignore_errors=True) + duthost.shell('show ipv6 bgp summary', module_ignore_errors=True) try: # default keepalive is 60 seconds, timeout 180 seconds. Hence wait for 180 seconds before timeout. diff --git a/tests/bgp/test_bgp_stress_link_flap.py b/tests/bgp/test_bgp_stress_link_flap.py index e1781b386bd..3cc73c58bae 100644 --- a/tests/bgp/test_bgp_stress_link_flap.py +++ b/tests/bgp/test_bgp_stress_link_flap.py @@ -30,7 +30,7 @@ @pytest.fixture(scope='module') -def setup(duthosts, rand_one_dut_hostname, nbrhosts, fanouthosts): +def setup(duthosts, rand_one_dut_hostname, nbrhosts, fanouthosts, show_ip_interface_cmd): duthost = duthosts[rand_one_dut_hostname] config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] @@ -53,7 +53,7 @@ def setup(duthosts, rand_one_dut_hostname, nbrhosts, fanouthosts): pytest_assert(wait_until(30, 5, 0, duthost.check_bgp_session_state, list(bgp_neighbors.keys())), "Not all BGP sessions are established on DUT") - ip_intfs = duthost.show_and_parse('show ip interface') + ip_intfs = duthost.show_and_parse(show_ip_interface_cmd) logger.debug("setup ip_intfs {}".format(ip_intfs)) # Create a mapping of neighbor IP to interfaces and their details diff --git a/tests/bgp/test_bgp_suppress_fib.py b/tests/bgp/test_bgp_suppress_fib.py index a2acfec5874..6ea91074ae3 100644 --- a/tests/bgp/test_bgp_suppress_fib.py +++ b/tests/bgp/test_bgp_suppress_fib.py @@ -63,7 +63,7 @@ FUNCTION = "function" STRESS = "stress" TRAFFIC_WAIT_TIME = 0.1 -BULK_TRAFFIC_WAIT_TIME = 0.004 +BULK_TRAFFIC_WAIT_TIME = 0.01 BGP_ROUTE_FLAP_TIMES = 5 UPDATE_WITHDRAW_THRESHOLD = 5 # consider the switch with low power cpu and a lot of bgp neighbors @@ -135,7 +135,8 @@ def ignore_expected_loganalyzer_errors(duthosts, rand_one_dut_hostname, loganaly "\\(.* minutes\\).*", r".* ERR memory_checker: \[memory_checker\] Failed to get container ID of.*", r".* ERR memory_checker: \[memory_checker\] cgroup memory usage file.*", - r".*ERR teamd#teamsyncd: :- readData: netlink reports an error=.*" + r".*ERR teamd#teamsyncd: :- readData: netlink reports an error=.*", + r".*ERR bgp#fpmsyncd: .*zmq send failed.*zmqerrno: 11:Resource temporarily unavailable.*" ] loganalyzer[duthost.hostname].ignore_regex.extend(ignoreRegex) @@ -538,10 +539,14 @@ def parse_time_stamp(bgp_packets, ipv4_route_list, ipv6_route_list): layer = bgp_updates[i].getlayer(bgp.BGPUpdate, nb=layer_index) if layer.nlri: for route in layer.nlri: + if not hasattr(route, 'prefix'): # skip malformed/segmented routes + continue if route.prefix in ipv4_route_list: update_time_stamp(announce_prefix_time_stamp, route.prefix, bgp_packets[i].time) if layer.withdrawn_routes: for route in layer.withdrawn_routes: + if not hasattr(route, 'prefix'): # skip malformed/segmented routes + continue if route.prefix in ipv4_route_list: update_time_stamp(withdraw_prefix_time_stamp, route.prefix, bgp_packets[i].time) layer_index += 1 @@ -980,7 +985,8 @@ def test_bgp_route_without_suppress(duthost, tbinfo, nbrhosts, ptfadapter, prepa def test_bgp_route_with_suppress_negative_operation(duthost, tbinfo, nbrhosts, ptfadapter, localhost, prepare_param, - restore_bgp_suppress_fib, generate_route_and_traffic_data): + restore_bgp_suppress_fib, generate_route_and_traffic_data, + loganalyzer): is_v6_topo = is_ipv6_only_topology(tbinfo) try: with allure.step("Prepare needed parameters"): diff --git a/tests/bgp/test_bgp_update_replication.py b/tests/bgp/test_bgp_update_replication.py index c4c53a18502..a06adb76f66 100644 --- a/tests/bgp/test_bgp_update_replication.py +++ b/tests/bgp/test_bgp_update_replication.py @@ -12,7 +12,7 @@ from tests.bgp.bgp_helpers import is_neighbor_sessions_established from tests.common.helpers.assertions import pytest_assert -from tests.common.utilities import wait_until +from tests.common.utilities import wait_until, is_ipv6_only_topology logger = logging.getLogger(__name__) @@ -31,12 +31,15 @@ ''' -def generate_routes(num_routes, nexthop): +def generate_routes(num_routes, nexthop, is_ipv6=False): ''' Generator which yields specified amount of dummy routes, in a dict that the route injector can use to announce and withdraw these routes. ''' - SUBNET_TMPL = "10.{first_iter}.{second_iter}.0/24" + if is_ipv6: + SUBNET_TMPL = "2001:db8:{first_iter:x}:{second_iter:x}::/64" + else: + SUBNET_TMPL = "10.{first_iter}.{second_iter}.0/24" loop_iterations = math.floor(num_routes ** 0.5) for first_iter in range(1, loop_iterations + 1): @@ -47,7 +50,7 @@ def generate_routes(num_routes, nexthop): } -def measure_stats(dut): +def measure_stats(dut, is_ipv6=False): ''' Validates that the provided DUT is responsive during test, and that device stats do not exceed specified thresholds, and if so, returns a dictionary containing device statistics @@ -64,7 +67,8 @@ def measure_stats(dut): proc_cpu = dut.shell("show processes cpu | head -n 10", module_ignore_errors=True)['stdout'] time_first_cmd = time.process_time() - bgp_sum = dut.shell("show ip bgp summary | grep memory", module_ignore_errors=True)['stdout'] + bgp_cmd = f"show ip{'v6' if is_ipv6 else ''} bgp summary | grep memory" + bgp_sum = dut.shell(bgp_cmd, module_ignore_errors=True)['stdout'] time_second_cmd = time.process_time() num_cores = dut.shell('cat /proc/cpuinfo | grep "cpu cores" | uniq', module_ignore_errors=True)['stdout'] @@ -187,6 +191,7 @@ def setup_bgp_peers( dut_asn=dut_asn, port=peer_port, neigh_type=neigh_type, + is_ipv6_only=is_ipv6_only_topology(tbinfo), namespace=connection_namespace, is_multihop=is_quagga or is_dualtor, is_passive=False @@ -212,12 +217,14 @@ def setup_bgp_peers( def test_bgp_update_replication( duthost, + tbinfo, setup_bgp_peers, setup_duthost_intervals, ): NUM_ROUTES = 10_000 bgp_peers: list[BGPNeighbor] = setup_bgp_peers duthost_intervals: list[float] = setup_duthost_intervals + is_ipv6 = is_ipv6_only_topology(tbinfo) # Ensure new sessions are ready if not wait_until( @@ -234,7 +241,7 @@ def test_bgp_update_replication( logger.info(f"Route injector: '{route_injector}', route receivers: '{route_receivers}'") - results = [measure_stats(duthost)] + results = [measure_stats(duthost, is_ipv6)] base_rib = int(results[0]["num_rib"]) min_expected_rib = base_rib + NUM_ROUTES max_expected_rib = base_rib + (2 * NUM_ROUTES) @@ -244,12 +251,16 @@ def test_bgp_update_replication( # Repeat 20 times for _ in range(20): # Inject 10000 routes - route_injector.announce_routes_batch(generate_routes(num_routes=NUM_ROUTES, nexthop=route_injector.ip)) - + route_injector.announce_routes_batch( + generate_routes( + num_routes=NUM_ROUTES, nexthop=route_injector.ip, + is_ipv6=is_ipv6 + ) + ) time.sleep(interval) # Measure after injection - results.append(measure_stats(duthost)) + results.append(measure_stats(duthost, is_ipv6)) # Validate all routes have been received curr_num_rib = int(results[-1]["num_rib"]) @@ -263,12 +274,16 @@ def test_bgp_update_replication( ) # Remove routes - route_injector.withdraw_routes_batch(generate_routes(num_routes=NUM_ROUTES, nexthop=route_injector.ip)) - + route_injector.withdraw_routes_batch( + generate_routes( + num_routes=NUM_ROUTES, nexthop=route_injector.ip, + is_ipv6=is_ipv6 + ) + ) time.sleep(interval) # Measure after removal - results.append(measure_stats(duthost)) + results.append(measure_stats(duthost, is_ipv6)) # Validate all routes have been withdrawn curr_num_rib = int(results[-1]["num_rib"]) @@ -281,7 +296,7 @@ def test_bgp_update_replication( f"All announcements have not been withdrawn: current '{curr_num_rib}', expected: '{base_rib}'" ) - results.append(measure_stats(duthost)) + results.append(measure_stats(duthost, is_ipv6)) # Output results as TSV for analysis in other programs results_tsv = tabulate(results, headers="keys", tablefmt="tsv") diff --git a/tests/common/cert_utils.py b/tests/common/cert_utils.py new file mode 100644 index 00000000000..cc50eb1d5bd --- /dev/null +++ b/tests/common/cert_utils.py @@ -0,0 +1,389 @@ +""" +Certificate generation utilities for TLS testing. + +This module provides Python-native certificate generation using the cryptography +library, replacing shell-based openssl commands for better control and reliability. + +Can be used for any TLS-based service testing: gNOI, gNMI, REST API, etc. +""" +import os +import ipaddress +from datetime import datetime, timedelta, timezone +from typing import Optional, Tuple, List + +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa + + +class TlsCertificateGenerator: + """ + Generate TLS certificates for testing. + + This class generates a complete certificate chain (CA, server, client) with + configurable validity periods and Subject Alternative Names. By default, + certificates are backdated by 1 day to handle clock skew between test hosts. + + Can be used for any TLS-based service: gNOI, gNMI, REST API, etc. + + Attributes: + server_ip: IP address to include in server certificate SAN + validity_days: Number of days certificates are valid (default: 825) + backdate_days: Days to backdate not_valid_before (default: 1) + + Example: + generator = TlsCertificateGenerator(server_ip="10.0.0.1") + generator.write_all("/tmp/certs") + # Creates: ca.crt, ca.key, server.crt, server.key, client.crt, client.key + """ + + # Default certificate file names + DEFAULT_CA_CERT = "ca.crt" + DEFAULT_CA_KEY = "ca.key" + DEFAULT_SERVER_CERT = "server.crt" + DEFAULT_SERVER_KEY = "server.key" + DEFAULT_CLIENT_CERT = "client.crt" + DEFAULT_CLIENT_KEY = "client.key" + + # Default certificate subjects + DEFAULT_CA_CN = "test.ca.sonic" + DEFAULT_SERVER_CN = "test.server.sonic" + DEFAULT_CLIENT_CN = "test.client.sonic" + + def __init__( + self, + server_ip: str, + validity_days: int = 825, + backdate_days: int = 1, + dns_names: Optional[List[str]] = None, + key_size: int = 2048, + ca_cn: Optional[str] = None, + server_cn: Optional[str] = None, + client_cn: Optional[str] = None, + ca_cert_name: Optional[str] = None, + ca_key_name: Optional[str] = None, + server_cert_name: Optional[str] = None, + server_key_name: Optional[str] = None, + client_cert_name: Optional[str] = None, + client_key_name: Optional[str] = None, + ): + """ + Initialize the certificate generator. + + Args: + server_ip: IP address to include in server certificate SAN + validity_days: Certificate validity period in days + backdate_days: Days to backdate not_valid_before to handle clock skew + dns_names: List of DNS names to include in server certificate SAN + key_size: RSA key size in bits + ca_cn: Common Name for CA certificate + server_cn: Common Name for server certificate + client_cn: Common Name for client certificate + ca_cert_name: Filename for CA certificate + ca_key_name: Filename for CA private key + server_cert_name: Filename for server certificate + server_key_name: Filename for server private key + client_cert_name: Filename for client certificate + client_key_name: Filename for client private key + """ + self.server_ip = server_ip + self.validity_days = validity_days + self.backdate_days = backdate_days + self.dns_names = dns_names or ["localhost"] + self.key_size = key_size + + # Certificate subject names (configurable) + self.ca_cn = ca_cn or self.DEFAULT_CA_CN + self.server_cn = server_cn or self.DEFAULT_SERVER_CN + self.client_cn = client_cn or self.DEFAULT_CLIENT_CN + + # Certificate file names (configurable) + self.ca_cert_name = ca_cert_name or self.DEFAULT_CA_CERT + self.ca_key_name = ca_key_name or self.DEFAULT_CA_KEY + self.server_cert_name = server_cert_name or self.DEFAULT_SERVER_CERT + self.server_key_name = server_key_name or self.DEFAULT_SERVER_KEY + self.client_cert_name = client_cert_name or self.DEFAULT_CLIENT_CERT + self.client_key_name = client_key_name or self.DEFAULT_CLIENT_KEY + + # Generated keys and certificates (populated by generate_all) + self._ca_key: Optional[rsa.RSAPrivateKey] = None + self._ca_cert: Optional[x509.Certificate] = None + self._server_key: Optional[rsa.RSAPrivateKey] = None + self._server_cert: Optional[x509.Certificate] = None + self._client_key: Optional[rsa.RSAPrivateKey] = None + self._client_cert: Optional[x509.Certificate] = None + + def _generate_key(self) -> rsa.RSAPrivateKey: + """Generate an RSA private key.""" + return rsa.generate_private_key( + public_exponent=65537, + key_size=self.key_size, + ) + + def _get_validity_period(self) -> Tuple[datetime, datetime]: + """Get certificate validity period with backdating for clock skew tolerance.""" + now = datetime.now(timezone.utc) + not_valid_before = now - timedelta(days=self.backdate_days) + not_valid_after = now + timedelta(days=self.validity_days) + return not_valid_before, not_valid_after + + def _generate_ca(self) -> Tuple[rsa.RSAPrivateKey, x509.Certificate]: + """Generate CA certificate and key.""" + key = self._generate_key() + not_valid_before, not_valid_after = self._get_validity_period() + + subject = issuer = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, self.ca_cn), + ]) + + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(not_valid_before) + .not_valid_after(not_valid_after) + .add_extension( + x509.BasicConstraints(ca=True, path_length=None), + critical=True, + ) + .add_extension( + x509.KeyUsage( + digital_signature=True, + key_cert_sign=True, + crl_sign=True, + key_encipherment=False, + content_commitment=False, + data_encipherment=False, + key_agreement=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .sign(key, hashes.SHA256()) + ) + + return key, cert + + def _generate_server( + self, ca_key: rsa.RSAPrivateKey, ca_cert: x509.Certificate + ) -> Tuple[rsa.RSAPrivateKey, x509.Certificate]: + """Generate server certificate signed by CA.""" + key = self._generate_key() + not_valid_before, not_valid_after = self._get_validity_period() + + subject = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, self.server_cn), + ]) + + # Build SAN with DNS names and server IP address + san_entries = [x509.DNSName(name) for name in self.dns_names] + san_entries.append(x509.IPAddress(ipaddress.ip_address(self.server_ip))) + + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(ca_cert.subject) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(not_valid_before) + .not_valid_after(not_valid_after) + .add_extension( + x509.SubjectAlternativeName(san_entries), + critical=False, + ) + .add_extension( + x509.BasicConstraints(ca=False, path_length=None), + critical=True, + ) + .add_extension( + x509.KeyUsage( + digital_signature=True, + key_encipherment=True, + key_cert_sign=False, + crl_sign=False, + content_commitment=False, + data_encipherment=False, + key_agreement=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .add_extension( + x509.ExtendedKeyUsage([x509.oid.ExtendedKeyUsageOID.SERVER_AUTH]), + critical=False, + ) + .sign(ca_key, hashes.SHA256()) + ) + + return key, cert + + def _generate_client( + self, ca_key: rsa.RSAPrivateKey, ca_cert: x509.Certificate + ) -> Tuple[rsa.RSAPrivateKey, x509.Certificate]: + """Generate client certificate signed by CA.""" + key = self._generate_key() + not_valid_before, not_valid_after = self._get_validity_period() + + subject = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, self.client_cn), + ]) + + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(ca_cert.subject) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(not_valid_before) + .not_valid_after(not_valid_after) + .add_extension( + x509.BasicConstraints(ca=False, path_length=None), + critical=True, + ) + .add_extension( + x509.KeyUsage( + digital_signature=True, + key_encipherment=True, + key_cert_sign=False, + crl_sign=False, + content_commitment=False, + data_encipherment=False, + key_agreement=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .add_extension( + x509.ExtendedKeyUsage([x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH]), + critical=False, + ) + .sign(ca_key, hashes.SHA256()) + ) + + return key, cert + + def generate_all(self) -> None: + """Generate complete certificate chain (CA, server, client).""" + self._ca_key, self._ca_cert = self._generate_ca() + self._server_key, self._server_cert = self._generate_server( + self._ca_key, self._ca_cert + ) + self._client_key, self._client_cert = self._generate_client( + self._ca_key, self._ca_cert + ) + + def _serialize_cert(self, cert: x509.Certificate) -> bytes: + """Serialize certificate to PEM format.""" + return cert.public_bytes(serialization.Encoding.PEM) + + def _serialize_key(self, key: rsa.RSAPrivateKey) -> bytes: + """Serialize private key to PEM format (unencrypted).""" + return key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + + def write_all(self, output_dir: str) -> None: + """ + Generate and write all certificates to the specified directory. + + Args: + output_dir: Directory to write certificate files + + Creates files based on configured names (defaults shown): + - ca.crt, ca.key (CA certificate and key) + - server.crt, server.key (Server certificate and key) + - client.crt, client.key (Client certificate and key) + """ + os.makedirs(output_dir, exist_ok=True) + + # Generate all certificates + self.generate_all() + + # Write CA files + with open(os.path.join(output_dir, self.ca_cert_name), "wb") as f: + f.write(self._serialize_cert(self._ca_cert)) + with open(os.path.join(output_dir, self.ca_key_name), "wb") as f: + f.write(self._serialize_key(self._ca_key)) + + # Write server files + with open(os.path.join(output_dir, self.server_cert_name), "wb") as f: + f.write(self._serialize_cert(self._server_cert)) + with open(os.path.join(output_dir, self.server_key_name), "wb") as f: + f.write(self._serialize_key(self._server_key)) + + # Write client files + with open(os.path.join(output_dir, self.client_cert_name), "wb") as f: + f.write(self._serialize_cert(self._client_cert)) + with open(os.path.join(output_dir, self.client_key_name), "wb") as f: + f.write(self._serialize_key(self._client_key)) + + def get_client_cn(self) -> str: + """Return the client certificate Common Name (e.g., for GNMI_CLIENT_CERT config).""" + return self.client_cn + + def get_cert_bytes(self) -> dict: + """ + Get all certificates and keys as bytes (for in-memory usage). + + Returns: + Dict with keys: ca_cert, ca_key, server_cert, server_key, + client_cert, client_key + """ + if self._ca_cert is None: + self.generate_all() + + return { + "ca_cert": self._serialize_cert(self._ca_cert), + "ca_key": self._serialize_key(self._ca_key), + "server_cert": self._serialize_cert(self._server_cert), + "server_key": self._serialize_key(self._server_key), + "client_cert": self._serialize_cert(self._client_cert), + "client_key": self._serialize_key(self._client_key), + } + + +def create_gnmi_cert_generator(server_ip: str, **kwargs) -> TlsCertificateGenerator: + """ + Factory function to create a certificate generator with gNMI/gNOI naming conventions. + + This preserves backward compatibility with existing file naming (gnmiCA.cer, + gnmiserver.cer, etc.) while using the generic TlsCertificateGenerator. + + Args: + server_ip: IP address of the server (DUT) to include in server cert SAN + **kwargs: Additional arguments passed to TlsCertificateGenerator + + Returns: + TlsCertificateGenerator configured with gNMI naming conventions + + Example: + generator = create_gnmi_cert_generator(server_ip="10.0.0.1") + generator.write_all("/tmp/gnoi_certs") + # Creates: gnmiCA.cer, gnmiCA.key, gnmiserver.cer, gnmiserver.key, + # gnmiclient.cer, gnmiclient.key + """ + gnmi_defaults = { + "ca_cn": "test.gnmi.sonic", + "server_cn": "test.server.gnmi.sonic", + "client_cn": "test.client.gnmi.sonic", + "ca_cert_name": "gnmiCA.cer", + "ca_key_name": "gnmiCA.key", + "server_cert_name": "gnmiserver.cer", + "server_key_name": "gnmiserver.key", + "client_cert_name": "gnmiclient.cer", + "client_key_name": "gnmiclient.key", + "dns_names": ["hostname.com"], + } + + # User-provided kwargs override defaults + gnmi_defaults.update(kwargs) + + return TlsCertificateGenerator(server_ip=server_ip, **gnmi_defaults) diff --git a/tests/common/devices/base.py b/tests/common/devices/base.py index 9db40b94d7a..5644d0e0724 100644 --- a/tests/common/devices/base.py +++ b/tests/common/devices/base.py @@ -55,21 +55,20 @@ def __init__(self, ansible_adhoc, hostname, *args, **kwargs): def __getattr__(self, module_name): if self.host.has_module(module_name): - self.module_name = module_name - self.module = getattr(self.host, module_name) - - return self._run + def _run_wrapper(*module_args, **kwargs): + return self._run(module_name, *module_args, **kwargs) + return _run_wrapper raise AttributeError( "'%s' object has no attribute '%s'" % (self.__class__, module_name) ) - def _run(self, *module_args, **complex_args): + def _run(self, module_name, *module_args, **complex_args): previous_frame = inspect.currentframe().f_back filename, line_number, function_name, lines, index = inspect.getframeinfo(previous_frame) verbose = complex_args.pop('verbose', True) - + module = getattr(self.host, module_name) if verbose: logger.debug( "{}::{}#{}: [{}] AnsibleModule::{}, args={}, kwargs={}".format( @@ -77,7 +76,7 @@ def _run(self, *module_args, **complex_args): function_name, line_number, self.hostname, - self.module_name, + module_name, json.dumps(module_args, cls=AnsibleHostBase.CustomEncoder), json.dumps(complex_args, cls=AnsibleHostBase.CustomEncoder) ) @@ -89,7 +88,7 @@ def _run(self, *module_args, **complex_args): function_name, line_number, self.hostname, - self.module_name + module_name ) ) @@ -98,7 +97,7 @@ def _run(self, *module_args, **complex_args): if module_async: def run_module(module_args, complex_args): - return self.module(*module_args, **complex_args)[self.hostname] + return module(*module_args, **complex_args)[self.hostname] pool = ThreadPool() result = pool.apply_async(run_module, (module_args, complex_args)) return pool, result @@ -106,9 +105,9 @@ def run_module(module_args, complex_args): module_args = json.loads(json.dumps(module_args, cls=AnsibleHostBase.CustomEncoder)) complex_args = json.loads(json.dumps(complex_args, cls=AnsibleHostBase.CustomEncoder)) - adhoc_res: AdHocResult = self.module(*module_args, **complex_args) + adhoc_res: AdHocResult = module(*module_args, **complex_args) - if self.module_name == "meta": + if module_name == "meta": # The meta module is special in Ansible - it doesn't execute on remote hosts, it controls Ansible's behavior # There are no per-host ModuleResults contained within it return @@ -123,7 +122,7 @@ def run_module(module_args, complex_args): function_name, line_number, self.hostname, - self.module_name, json.dumps(hostname_res, cls=AnsibleHostBase.CustomEncoder) + module_name, json.dumps(hostname_res, cls=AnsibleHostBase.CustomEncoder) ) ) else: @@ -133,14 +132,14 @@ def run_module(module_args, complex_args): function_name, line_number, self.hostname, - self.module_name, + module_name, hostname_res.is_failed, hostname_res.get('rc', None) ) ) if (hostname_res.is_failed or 'exception' in hostname_res) and not module_ignore_errors: - raise RunAnsibleModuleFail("run module {} failed".format(self.module_name), hostname_res) + raise RunAnsibleModuleFail("run module {} failed".format(module_name), hostname_res) return hostname_res diff --git a/tests/common/devices/duthosts.py b/tests/common/devices/duthosts.py index a1165950dd1..b4a7ff03abf 100644 --- a/tests/common/devices/duthosts.py +++ b/tests/common/devices/duthosts.py @@ -19,9 +19,9 @@ class DutHosts(object): """ class _Nodes(list): """ Internal class representing a list of MultiAsicSonicHosts """ - def _run_on_nodes(self, *module_args, **complex_args): + def _run_on_nodes(self, module, *module_args, **complex_args): """ Delegate the call to each of the nodes, return the results in a dict.""" - return {node.hostname: getattr(node, self.attr)(*module_args, **complex_args) for node in self} + return {node.hostname: getattr(node, module)(*module_args, **complex_args) for node in self} def __getattr__(self, attr): """ To support calling ansible modules on a list of MultiAsicSonicHost @@ -32,8 +32,9 @@ def __getattr__(self, attr): a dictionary with key being the MultiAsicSonicHost's hostname, and value being the output of ansible module on that MultiAsicSonicHost """ - self.attr = attr - return self._run_on_nodes + def _run_on_nodes_wrapper(*module_args, **complex_args): + return self._run_on_nodes(attr, *module_args, **complex_args) + return _run_on_nodes_wrapper def __eq__(self, o): """ To support eq operator on the DUTs (nodes) in the testbed """ @@ -62,6 +63,12 @@ def __init__(self, ansible_adhoc, tbinfo, request, duts, target_hostname=None, i self.request = request self.duts = duts self.is_parallel_run = target_hostname is not None + # Initialize _nodes to None to avoid recursion in __getattr__ + self._nodes = None + self._nodes_for_parallel = None + self._supervisor_nodes = None + self._frontend_nodes = None + # TODO: Initialize the nodes in parallel using multi-threads? if self.is_parallel_run: self.parallel_run_stage = NON_INITIAL_CHECKS_STAGE diff --git a/tests/common/devices/multi_asic.py b/tests/common/devices/multi_asic.py index 9350f68640a..a72100086f3 100644 --- a/tests/common/devices/multi_asic.py +++ b/tests/common/devices/multi_asic.py @@ -124,10 +124,11 @@ def critical_services_tracking_list(self): def get_default_critical_services_list(self): return self._DEFAULT_SERVICES - def _run_on_asics(self, *module_args, **complex_args): + def _run_on_asics(self, multi_asic_attr, *module_args, **complex_args): """ Run an asible module on asics based on 'asic_index' keyword in complex_args Args: + multi_asic_attr: name of the ansible module to run module_args: other ansible module args passed from the caller complex_args: other ansible keyword args @@ -148,7 +149,7 @@ def _run_on_asics(self, *module_args, **complex_args): """ if "asic_index" not in complex_args: # Default ASIC/namespace - return getattr(self.sonichost, self.multi_asic_attr)(*module_args, **complex_args) + return getattr(self.sonichost, multi_asic_attr)(*module_args, **complex_args) else: asic_complex_args = copy.deepcopy(complex_args) asic_index = asic_complex_args.pop("asic_index") @@ -157,11 +158,11 @@ def _run_on_asics(self, *module_args, **complex_args): if self.sonichost.facts['num_asic'] == 1: if asic_index != 0: raise ValueError("Trying to run module '{}' against asic_index '{}' on a single asic dut '{}'" - .format(self.multi_asic_attr, asic_index, self.sonichost.hostname)) - return getattr(self.asic_instance(asic_index), self.multi_asic_attr)(*module_args, **asic_complex_args) + .format(multi_asic_attr, asic_index, self.sonichost.hostname)) + return getattr(self.asic_instance(asic_index), multi_asic_attr)(*module_args, **asic_complex_args) elif type(asic_index) == str and asic_index.lower() == "all": # All ASICs/namespace - return [getattr(asic, self.multi_asic_attr)(*module_args, **asic_complex_args) for asic in self.asics] + return [getattr(asic, multi_asic_attr)(*module_args, **asic_complex_args) for asic in self.asics] else: raise ValueError("Argument 'asic_index' must be an int or string 'all'.") @@ -358,8 +359,9 @@ def __getattr__(self, attr): """ sonic_asic_attr = getattr(SonicAsic, attr, None) if not attr.startswith("_") and sonic_asic_attr and callable(sonic_asic_attr): - self.multi_asic_attr = attr - return self._run_on_asics + def _run_on_asics_wrapper(*module_args, **complex_args): + return self._run_on_asics(attr, *module_args, **complex_args) + return _run_on_asics_wrapper else: return getattr(self.sonichost, attr) # For backward compatibility diff --git a/tests/common/devices/sonic.py b/tests/common/devices/sonic.py index d9e5e48ccbc..cd80ccb207a 100644 --- a/tests/common/devices/sonic.py +++ b/tests/common/devices/sonic.py @@ -3009,10 +3009,10 @@ def _get_serial_device_prefix(self) -> str: Get the serial device prefix for the platform. Returns: - str: The device prefix (e.g., "/dev/C0-", "/dev/ttyUSB-") + str: The device prefix (e.g., "/dev/C0-", "/dev/ttyUSB") """ # Reads udevprefix.conf from the platform directory to determine the correct device prefix - # Falls back to /dev/ttyUSB- if the config file doesn't exist + # Falls back to /dev/ttyUSB if the config file doesn't exist script = ''' from sonic_py_common import device_info import os @@ -3032,8 +3032,8 @@ def _get_serial_device_prefix(self) -> str: res: ShellResult = self.shell(cmd, module_ignore_errors=True) if res['rc'] != 0 or not res['stdout'].strip(): - logging.warning("Failed to get serial device prefix, using default /dev/ttyUSB-") - device_prefix = "/dev/ttyUSB-" + logging.warning("Failed to get serial device prefix, using default /dev/ttyUSB") + device_prefix = "/dev/ttyUSB" else: device_prefix = res['stdout'].strip() @@ -3047,7 +3047,7 @@ def _get_serial_device_path(self, port: int) -> str: port: Port number (e.g., 1, 2) Returns: - str: The full device path (e.g., "/dev/C0-1", "/dev/ttyUSB-1") + str: The full device path (e.g., "/dev/C0-1", "/dev/ttyUSB1") """ device_prefix = self._get_serial_device_prefix() return f"{device_prefix}{port}" diff --git a/tests/common/fixtures/fib_utils.py b/tests/common/fixtures/fib_utils.py index 3a46cd580f6..a43720dae22 100644 --- a/tests/common/fixtures/fib_utils.py +++ b/tests/common/fixtures/fib_utils.py @@ -44,16 +44,19 @@ def get_t2_fib_info(duthosts, duts_cfg_facts, duts_mg_facts, testname=None): dut_inband_intfs = {} dut_port_channels = {} switch_type = '' + is_chassis = False for duthost in duthosts.frontend_nodes: cfg_facts = duts_cfg_facts[duthost.hostname] for asic_cfg_facts in cfg_facts: if duthost.facts['switch_type'] == "voq": + is_chassis = \ + not duthost.dut_basic_facts()['ansible_facts']['dut_basic_facts'].get("is_chassis_config_absent") switch_type = "voq" if 'VOQ_INBAND_INTERFACE' in asic_cfg_facts[1]: dut_inband_intfs.setdefault(duthost.hostname, []).extend(asic_cfg_facts[1]['VOQ_INBAND_INTERFACE']) dut_port_channels.setdefault(duthost.hostname, {}).update(asic_cfg_facts[1].get('PORTCHANNEL_MEMBER', {})) sys_neigh = {} - if switch_type == "voq": + if switch_type == "voq" and is_chassis: if len(duthosts) == 1: voq_db = VoqDbCli(duthosts.frontend_nodes[0]) else: diff --git a/tests/common/fixtures/grpc_fixtures.py b/tests/common/fixtures/grpc_fixtures.py index 10f9c7a4d5b..8a8adcb97fb 100644 --- a/tests/common/fixtures/grpc_fixtures.py +++ b/tests/common/fixtures/grpc_fixtures.py @@ -4,6 +4,7 @@ This module provides pytest fixtures for easy access to gRPC clients with automatic configuration discovery, making it simple to write gRPC-based tests. """ +import os import pytest import logging from tests.common.grpc_config import grpc_config @@ -176,7 +177,7 @@ def test_gnmi_get(ptf_gnmi): @pytest.fixture(scope="module") -def setup_gnoi_tls_server(duthost, localhost, ptfhost): +def setup_gnoi_tls_server(duthost, ptfhost): """ Set up gNOI server with TLS certificates and configuration. @@ -185,7 +186,7 @@ def setup_gnoi_tls_server(duthost, localhost, ptfhost): The fixture: 1. Creates a configuration checkpoint for rollback - 2. Generates TLS certificates with proper SAN for DUT IP + 2. Generates TLS certificates with proper SAN for DUT IP (backdated to handle clock skew) 3. Distributes certificates to DUT and PTF container 4. Configures CONFIG_DB for TLS mode (port 50052) 5. Restarts the gNOI server process @@ -194,7 +195,6 @@ def setup_gnoi_tls_server(duthost, localhost, ptfhost): Args: duthost: DUT host instance to configure - localhost: Localhost instance for certificate generation ptfhost: PTF host instance for client certificates Usage: @@ -207,10 +207,14 @@ def test_gnoi_with_tls(ptf_gnoi): Note: Client fixtures (ptf_grpc, ptf_gnoi) automatically adapt to TLS mode when this fixture is active through GNMIEnvironment detection. + + Certificates are backdated by 1 day to handle clock skew between + the test host, DUT, and PTF container. """ from tests.common.gu_utils import create_checkpoint, rollback checkpoint_name = "gnoi_tls_setup" + cert_dir = "/tmp/gnoi_certs" logger.info("Setting up gNOI TLS server environment") @@ -218,8 +222,8 @@ def test_gnoi_with_tls(ptf_gnoi): create_checkpoint(duthost, checkpoint_name) try: - # 2. Generate certificates - _create_gnoi_certs(duthost, localhost, ptfhost) + # 2. Generate and distribute certificates + _create_gnoi_certs(duthost, ptfhost, cert_dir) # 3. Configure server for TLS mode _configure_gnoi_tls_server(duthost) @@ -243,59 +247,32 @@ def test_gnoi_with_tls(ptf_gnoi): logger.error(f"Failed to rollback configuration: {e}") try: - _delete_gnoi_certs(localhost) + _delete_gnoi_certs(cert_dir) logger.info("Certificate cleanup completed") except Exception as e: logger.error(f"Failed to cleanup certificates: {e}") -def _create_gnoi_certs(duthost, localhost, ptfhost): - """Generate gNOI TLS certificates with proper SAN for DUT IP.""" - logger.info("Generating gNOI TLS certificates") - - # Create all certificate files in /tmp to avoid polluting working directory - cert_dir = "/tmp/gnoi_certs" - localhost.shell(f"mkdir -p {cert_dir}") - localhost.shell(f"cd {cert_dir}") - - # Create Root key - localhost.shell(f"cd {cert_dir} && openssl genrsa -out gnmiCA.key 2048") - - # Create Root cert - localhost.shell(f"""cd {cert_dir} && openssl req -x509 -new -nodes -key gnmiCA.key -sha256 -days 1825 \ - -subj '/CN=test.gnmi.sonic' -out gnmiCA.cer""") - - # Create server key - localhost.shell(f"cd {cert_dir} && openssl genrsa -out gnmiserver.key 2048") - - # Create server CSR - localhost.shell(f"""cd {cert_dir} && openssl req -new -key gnmiserver.key \ - -subj '/CN=test.server.gnmi.sonic' -out gnmiserver.csr""") - - # Create extension file with DUT IP SAN - ext_conf_content = f"""[ req_ext ] -subjectAltName = @alt_names -[alt_names] -DNS.1 = hostname.com -IP = {duthost.mgmt_ip}""" +def _create_gnoi_certs(duthost, ptfhost, cert_dir): + """ + Generate and distribute gNOI TLS certificates. - localhost.shell(f"cd {cert_dir} && echo '{ext_conf_content}' > extfile.cnf") + Certificates are backdated by 1 day to handle clock skew between hosts. - # Sign server certificate with SAN extension - localhost.shell(f"""cd {cert_dir} && openssl x509 -req -in gnmiserver.csr -CA gnmiCA.cer -CAkey gnmiCA.key \ - -CAcreateserial -out gnmiserver.cer -days 825 -sha256 \ - -extensions req_ext -extfile extfile.cnf""") + Args: + duthost: DUT host instance (for IP and copying server certs) + ptfhost: PTF host instance (for copying client certs) + cert_dir: Local directory to store generated certificates + """ + from tests.common.cert_utils import create_gnmi_cert_generator - # Create client key - localhost.shell(f"cd {cert_dir} && openssl genrsa -out gnmiclient.key 2048") + logger.info("Generating gNOI TLS certificates") - # Create client CSR - localhost.shell(f"""cd {cert_dir} && openssl req -new -key gnmiclient.key \ - -subj '/CN=test.client.gnmi.sonic' -out gnmiclient.csr""") + # Generate certificates with 1-day backdating to handle clock skew + generator = create_gnmi_cert_generator(server_ip=duthost.mgmt_ip) + generator.write_all(cert_dir) - # Sign client certificate - localhost.shell(f"""cd {cert_dir} && openssl x509 -req -in gnmiclient.csr -CA gnmiCA.cer -CAkey gnmiCA.key \ - -CAcreateserial -out gnmiclient.cer -days 825 -sha256""") + logger.info(f"Certificates generated in {cert_dir}") # Get certificate copy destinations from centralized config copy_destinations = grpc_config.get_cert_copy_destinations() @@ -398,10 +375,12 @@ def _verify_gnoi_tls_connectivity(duthost, ptfhost): logger.info("TLS connectivity verification completed successfully") -def _delete_gnoi_certs(localhost): +def _delete_gnoi_certs(cert_dir): """Clean up generated certificate files.""" + import shutil + logger.info("Cleaning up certificate files") - # Remove the entire certificate directory in /tmp - cert_dir = "/tmp/gnoi_certs" - localhost.shell(f"rm -rf {cert_dir}", module_ignore_errors=True) + # Remove the entire certificate directory + if os.path.exists(cert_dir): + shutil.rmtree(cert_dir, ignore_errors=True) diff --git a/tests/common/helpers/bgp.py b/tests/common/helpers/bgp.py index e1b7ee1962a..49a4eedab89 100644 --- a/tests/common/helpers/bgp.py +++ b/tests/common/helpers/bgp.py @@ -130,7 +130,8 @@ def __init__(self, duthost, ptfhost, name, neighbor_ip, neighbor_asn, dut_ip, dut_asn, port, neigh_type=None, namespace=None, is_multihop=False, is_passive=False, debug=False, - confed_asn=None, use_vtysh=False): + is_ipv6_only=False, router_id=None, confed_asn=None, use_vtysh=False): + self.duthost = duthost self.ptfhost = ptfhost self.ptfip = ptfhost.mgmt_ip @@ -145,6 +146,14 @@ def __init__(self, duthost, ptfhost, name, self.is_passive = is_passive self.is_multihop = not is_passive and is_multihop self.debug = debug + self.is_ipv6_neighbor = is_ipv6_only + if not self.is_ipv6_neighbor: + self.router_id = router_id or self.ip + else: + # Generate router ID by combining 20.0.0.0 base with last 3 bytes of IPv6 addr + router_id_base = ipaddress.IPv4Address("20.0.0.0") + ipv6_addr = ipaddress.IPv6Address(self.ip) + self.router_id = router_id or str(ipaddress.IPv4Address(int(router_id_base) | int(ipv6_addr) & 0xFFFFFF)) self.use_vtysh = use_vtysh self.confed_asn = confed_asn @@ -185,19 +194,11 @@ def start_session(self): peer_name=self.name ) - if ipaddress.ip_address(self.ip).version == 4: - router_id = self.ip - else: - # Generate router ID by combining 20.0.0.0 base with last 3 bytes of IPv6 addr - router_id_base = ipaddress.IPv4Address("20.0.0.0") - ipv6_addr = ipaddress.IPv6Address(self.ip) - router_id = str(ipaddress.IPv4Address(int(router_id_base) | int(ipv6_addr) & 0xFFFFFF)) - self.ptfhost.exabgp( name=self.name, - state="started", + state="restarted" if self.is_ipv6_neighbor else "started", local_ip=self.ip, - router_id=router_id, + router_id=self.router_id, peer_ip=self.peer_ip, local_asn=self.asn, peer_asn=self.confed_asn if self.confed_asn is not None else self.peer_asn, diff --git a/tests/common/helpers/dut_utils.py b/tests/common/helpers/dut_utils.py index 538cf622490..67e16ca9dc7 100644 --- a/tests/common/helpers/dut_utils.py +++ b/tests/common/helpers/dut_utils.py @@ -126,6 +126,64 @@ def clear_failed_flag_and_restart(duthost, container_name): pytest_assert(restarted, "Failed to restart container '{}' after reset-failed was cleared".format(container_name)) +def restart_service_with_startlimit_guard(duthost, service_name, backoff_seconds=30, verify_timeout=180): + """ + Restart a systemd-managed service with StartLimitHit guard. + + Strategy: + 0) Pre-detect StartLimitHit and, if present, skip a failing restart + 1) When not rate-limited, reset-failed to clear stale counters and try restart + 2) If restart fails, rate-limit is detected, or container isn't running: + - 'systemctl reset-failed .service' + - fixed backoff (default 30s when rate-limited, 1s otherwise) + - 'systemctl start .service' + - wait until container is running + + Returns: True when the service is (re)started and running; asserts on failure. + """ + + # 0) Pre-detect StartLimitHit so we can optionally skip a failing restart + pre_rate_limited = is_hitting_start_limit(duthost, service_name) + + if not pre_rate_limited: + # 1) Proactively clear stale failure counters and try a normal restart + duthost.shell( + f"sudo systemctl reset-failed {service_name}.service", + module_ignore_errors=True + ) + ret = duthost.shell( + f"sudo systemctl restart {service_name}.service", + module_ignore_errors=True + ) + rate_limited = is_hitting_start_limit(duthost, service_name) + else: + logger.info( + f"StartLimitHit pre-detected for {service_name}, applying reset-failed and " + f"fixed backoff {backoff_seconds}s before start" + ) + # Force the recovery path below without attempting an immediate restart. + ret = {"rc": 1} + rate_limited = True + + # 2/3) Recovery path: reset-failed + backoff + start if needed + if ret.get("rc", 1) != 0 or rate_limited or not is_container_running(duthost, service_name): + duthost.shell( + f"sudo systemctl reset-failed {service_name}.service", + module_ignore_errors=True + ) + time.sleep(backoff_seconds if rate_limited else 1) + duthost.shell( + f"sudo systemctl start {service_name}.service", + module_ignore_errors=True + ) + pytest_assert( + wait_until(verify_timeout, 1, 0, check_container_state, duthost, service_name, True), + f"{service_name} container did not become running after recovery start" + ) + + return True + + def get_group_program_info(duthost, container_name, group_name): """Gets program names, running status and their pids by analyzing the command output of "docker exec supervisorctl status". Program name diff --git a/tests/common/macsec/__init__.py b/tests/common/macsec/__init__.py index bfcfb9763ab..d26ea8a4f8a 100644 --- a/tests/common/macsec/__init__.py +++ b/tests/common/macsec/__init__.py @@ -122,7 +122,27 @@ def macsec_setup(self, startup_macsec, shutdown_macsec, macsec_feature): @pytest.fixture(scope="module", autouse=True) def load_macsec_info(self, request, macsec_duthost, ctrl_links, macsec_profile, tbinfo): + """Pre-load MACsec session info for all control links. + + If MACsec is enabled and configured for this DUT/profile, wait for + MKA establishment (APP/STATE DB populated with SC/SA, including SAK) + before calling ``load_all_macsec_info``. This avoids races where + ``get_macsec_attr`` hits APP_DB before the egress SA row (and ``sak``) + has been written by wpa_supplicant. + """ + if get_macsec_enable_status(macsec_duthost) and get_macsec_profile(macsec_duthost): + # Ensure MKA sessions are established (SC/SA present in DB) if the + # test environment provides the wait_mka_establish fixture + # (defined in tests/macsec/conftest.py). For environments that do + # not define it, fall back to the original behaviour. + try: + request.getfixturevalue('wait_mka_establish') + except pytest.FixtureLookupError: + # Some environments do not define wait_mka_establish; fall back + # to the original behaviour when the fixture is missing. + pass + if is_macsec_configured(macsec_duthost, macsec_profile, ctrl_links): load_all_macsec_info(macsec_duthost, ctrl_links, tbinfo) else: diff --git a/tests/common/macsec/macsec_config_helper.py b/tests/common/macsec/macsec_config_helper.py index 57c23730077..aba968c4464 100644 --- a/tests/common/macsec/macsec_config_helper.py +++ b/tests/common/macsec/macsec_config_helper.py @@ -1,6 +1,5 @@ import logging import time - from tests.common.macsec.macsec_helper import get_mka_session, getns_prefix, wait_all_complete, \ submit_async_task, load_all_macsec_info from tests.common.macsec.macsec_platform_helper import global_cmd, find_portchannel_from_member, get_portchannel @@ -17,7 +16,8 @@ 'enable_macsec_port', 'disable_macsec_port', 'get_macsec_enable_status', - 'get_macsec_profile' + 'get_macsec_profile', + 'wait_for_macsec_cleanup' ] logger = logging.getLogger(__name__) @@ -219,9 +219,21 @@ def cleanup_macsec_configuration(duthost, ctrl_links, profile_name): submit_async_task(delete_macsec_profile, (d, None, profile_name)) wait_all_complete(timeout=300) + logger.info("Cleanup macsec configuration step3: wait for automatic cleanup") + + # Extract DUT interface names from ctrl_links and wait for automatic + # MACsec cleanup on the DUT side. + interfaces = list(ctrl_links.keys()) + wait_for_macsec_cleanup(duthost, interfaces) + + # Also wait for neighbor devices to complete automatic cleanup for their + # corresponding ports. + for dut_port, nbr in list(ctrl_links.items()): + wait_for_macsec_cleanup(nbr["host"], [nbr["port"]]) + logger.info("Cleanup macsec configuration finished") - # Waiting for all mka session were cleared in all devices + # Waiting for all MKA sessions to be cleared on neighbor devices. for d in devices: if isinstance(d, EosHost): continue @@ -268,3 +280,93 @@ def setup_macsec_configuration(duthost, ctrl_links, profile_name, default_priori # Load the MACSEC_INFO, to have data of all macsec sessions load_all_macsec_info(duthost, ctrl_links, tbinfo) + + +def wait_for_macsec_cleanup(host, interfaces, timeout=90): + """Wait for MACsec daemon to automatically clean up all MACsec entries. + + This function implements proper synchronization to wait for the automatic + cleanup process to complete, preserving the intended MACsec cleanup behavior. + + Args: + host: SONiC DUT or neighbor host object + interfaces: List of interface names to check + timeout: Maximum time to wait in seconds for MACsec cleanup to finish (default: 90). + + Returns: + bool: True if cleanup completed, False if timeout + """ + if isinstance(host, EosHost): + # EOS hosts don't use Redis databases + logger.info("EOS host detected, skipping Redis cleanup verification") + return True + + logger.info(f"Waiting for automatic MACsec cleanup (timeout: {timeout}s)") + + start_time = time.time() + # Poll at most ~10 times over the full timeout, capped at 10 seconds between checks. + poll_interval = min(10, max(1, timeout / 10.0)) + + # We only care about APPL_DB and STATE_DB for MACsec tables. Instead of + # trying to reverse-engineer numeric DB IDs from CONFIG_DB, rely on + # sonic-db-cli with logical DB names and the same namespace logic used + # elsewhere in MACsec helpers. + + while time.time() - start_time < timeout: + all_clean = True + remaining_entries = {} + + for interface in interfaces: + ns_prefix = getns_prefix(host, interface) + + for db_name, sep in (("APPL_DB", ":"), ("STATE_DB", "|")): + pattern = f"MACSEC_*{sep}{interface}*" + cmd = f"sonic-db-cli {ns_prefix} {db_name} KEYS '{pattern}'" + + try: + result = host.command(cmd, verbose=False) + out_lines = result.get("stdout_lines", []) + except Exception as e: + logger.warning( + "Failed to query MACsec keys on host %s, DB %s, interface %s: %r", + getattr(host, 'hostname', host), + db_name, + interface, + e, + ) + # If we cannot query Redis for this DB/interface, be + # conservative and assume cleanup is not complete yet. + all_clean = False + continue + + keys = [k.strip() for k in out_lines if k.strip()] + if keys: + all_clean = False + remaining_entries.setdefault((db_name, interface), []).extend(keys) + + elapsed = time.time() - start_time + + if all_clean: + logger.info( + f"Automatic MACsec cleanup completed successfully in {elapsed:.1f}s" + ) + return True + + # Log progress every 30 seconds to reduce verbosity + if int(elapsed) % 30 == 0 and elapsed > 0: + logger.info(f"Still waiting for cleanup... ({elapsed:.0f}s elapsed)") + + time.sleep(poll_interval) + + # Timeout reached + elapsed = time.time() - start_time + logger.warning(f"Automatic MACsec cleanup timeout after {elapsed:.1f}s") + + # Log summary of remaining entries + total_remaining = sum(len(entries) for entries in remaining_entries.values()) + if total_remaining > 0: + logger.warning( + f" {total_remaining} MACsec entries still remain after timeout" + ) + + return False diff --git a/tests/common/platform/processes_utils.py b/tests/common/platform/processes_utils.py index 38f5e0c0073..0dd638984c9 100644 --- a/tests/common/platform/processes_utils.py +++ b/tests/common/platform/processes_utils.py @@ -5,6 +5,7 @@ """ import logging import time +import re from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until, get_plt_reboot_ctrl @@ -12,6 +13,25 @@ logger = logging.getLogger(__name__) +def check_pmon_uptime_minutes(duthost, minimal_runtime=6): + """ + @summary: This function checks if pmon uptime is at least the minimal_runtime + @return: True pmon has been running at least the minimal_runtime, False for otherwise + """ + result = duthost.command("docker ps | grep pmon", _uses_shell=True) + if result["stdout"]: + match = re.search(r'Up (\d+) (minutes|hours)', result["stdout"]) + if match: + if match.group(2) == "hours": + return int(match.group(1))*60 >= minimal_runtime + else: + return int(match.group(1)) >= minimal_runtime + match = re.search(r'Up About an hour', result["stdout"]) + if match: + return 60 >= minimal_runtime + return False + + def reset_timeout(duthost): """ return: if timeout is specified in inventory file for this dut, return new timeout diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml index 54618f1f07f..6e3f8f20943 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml @@ -1771,12 +1771,6 @@ everflow/test_everflow_per_interface.py::test_everflow_per_interface[ipv6-erspan reason: "Skip everflow per interface IPv6 test on unsupported platforms x86_64-nvidia_sn5640-r0." conditions: - "platform in ['x86_64-nvidia_sn5640-r0']" - xfail: - reason: "xfail for IPv6-only topologies, need support for IPv6 bgp. Or test case has issue on the t0-isolated-d256u256s2 topo." - conditions_logical_operator: or - conditions: - - "https://github.com/sonic-net/sonic-mgmt/issues/19096 and '-v6-' in topo_name" - - "'t0-isolated-d256u256s2' in topo_name and platform in ['x86_64-nvidia_sn5640-r0']" everflow/test_everflow_per_interface.py::test_everflow_per_interface[ipv6-m0_l3_scenario]: skip: @@ -3577,10 +3571,6 @@ pc/test_po_update.py::test_po_update: reason: "Skip test due to there is no portchannel or no portchannel member exists in current topology." conditions: - "len(minigraph_portchannels) == 0 or len(minigraph_portchannels[list(minigraph_portchannels.keys())[0]]['members']) == 0" - xfail: - reason: "xfail for IPv6-only topologies, need to add support for IPv6-only - https://github.com/sonic-net/sonic-mgmt/issues/20729" - conditions: - - "https://github.com/sonic-net/sonic-mgmt/issues/20729 and '-v6-' in topo_name" pc/test_po_update.py::test_po_update_io_no_loss: skip: @@ -4337,20 +4327,12 @@ route/test_static_route.py: - "platform in ['x86_64-8122_64eh_o-r0', 'x86_64-8122_64ehf_o-r0']" route/test_static_route.py::test_static_route[: - skip: - reason: "Skip for IPv6-only topologies" - conditions: - - "'-v6-' in topo_name" xfail: reason: "xfail for scale topology, issue https://github.com/sonic-net/sonic-buildimage/issues/24537" conditions: - "https://github.com/sonic-net/sonic-buildimage/issues/24537 and 't0-isolated-d256u256s2' in topo_name" route/test_static_route.py::test_static_route_ecmp[: - skip: - reason: "Skip for IPv6-only topologies" - conditions: - - "'-v6-' in topo_name" xfail: reason: "xfail for scale topology, issue https://github.com/sonic-net/sonic-buildimage/issues/24537" conditions: diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions_voq.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions_voq.yaml new file mode 100644 index 00000000000..b682a0af4f0 --- /dev/null +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions_voq.yaml @@ -0,0 +1,65 @@ +####################################### +#####test_po_voq.py ##### +####################################### +pc/test_po_voq.py: + skip: + reason: "Skipped as the test applies to chassis only" + conditions: + - "is_chassis_config_absent==True" + +################################################# +#####test_voq_chassis_app_db_consistency.py ##### +################################################# +voq/test_voq_chassis_app_db_consistency.py: + skip: + reason: "Skipped as the test applies to chassis only" + conditions: + - "is_chassis_config_absent==True" + +################################################# +#####test_voq_disrupts.py ##### +################################################# +voq/test_voq_disrupts.py: + skip: + reason: "Skipped as the test applies to chassis only" + conditions: + - "is_chassis_config_absent==True" + +####################################### +#####test_voq_init.py ##### +####################################### +voq/test_voq_init.py::test_voq_inband_port_create: + skip: + reason: "Skipped as the test applies to chassis only" + conditions: + - "is_chassis_config_absent==True" + +voq/test_voq_init.py::test_voq_interface_create: + skip: + reason: "Skipped as the test applies to chassis only" + conditions: + - "is_chassis_config_absent==True" + +voq/test_voq_init.py::test_voq_neighbor_create: + skip: + reason: "Skipped as the test applies to chassis only" + conditions: + - "is_chassis_config_absent==True" + +################################################# +#####test_voq_ipfwd.py ##### +################################################# +voq/test_voq_ipfwd.py: + skip: + reason: "Skipped as the test applies to chassis only" + conditions: + - "is_chassis_config_absent==True" + +################################################# +#####test_voq_nbr.py ##### +################################################# +voq/test_voq_nbr.py: + skip: + reason: "Skipped as the test applies to chassis only" + conditions: + - "is_chassis_config_absent==True" diff --git a/tests/common/plugins/parallel_fixture/README.md b/tests/common/plugins/parallel_fixture/README.md new file mode 100644 index 00000000000..60219b003dd --- /dev/null +++ b/tests/common/plugins/parallel_fixture/README.md @@ -0,0 +1,230 @@ +# Parallel Fixture Manager Design Document + +## 1. Overview + +The **Parallel Fixture Manager** is a pytest plugin designed to optimize test execution time by parallelizing the setup and teardown of fixtures. The sonic-mgmt fixture setup/teardowns often involves blocking I/O operations such as device configuration, service restarts, or waiting for convergence. By offloading these tasks to a thread pool, the manager allows multiple same-level fixtures to setup/teardown concurrently, which could reduce the overall test execution time. + +![test execution](images/pytest.jpg) + +## 2. Requirements + +The Parallel Fixture Manager is designed to address specific challenges in the SONiC testing infrastructure. The key requirements are: + +* **Test Fixture Setup/Teardown Parallelization** +* **Scope-Based Synchronization** + * The system must strictly enforce pytest scoping rules: + 1. All background tasks associated with a specific scope (Session, Module, Class, Function) in setup must complete successfully before the test runner proceeds to a narrower scope or executes the test function. + 2. All background tasks associated with a specific scope (Session, Module, Class, Function) in teardown must complete successfully before the test runner proceeds to a broader scope or finish the test execution. +* **Fail-Fast Reliability** + * The system must immediately detect the exception and abort the ongoing test setup to prevent cascading failures, resource wastage, and misleading test results. +* **Non-Intrusive Integration** + * The system must expose a minimal and intuitive API. Existing fixtures should be able to adopt parallel execution patterns with minimal code changes, preserving the standard pytest fixture structure. +* **Safe Termination & Cleanup** + * The system must handle interruptions and timeouts gracefully. It must ensure that background threads are properly terminated and resources are cleaned up, even in the event of a test failure or user interruption. + +## 3. Architecture + +### 3.1 Core Components + +* **`ParallelFixtureManager`**: The central thread pool controller exposed as a session-scoped fixture (`parallel_manager`). + * **Executor**: Uses `concurrent.futures.ThreadPoolExecutor` to execute tasks. + * **Monitor Thread**: A daemon thread (`_monitor_workers`) that polls active futures to log task execution status and any exception in worker thread. + * **Task Queues**: Maintains separate lists of futures for setup and teardown tasks, categorized by scope. +* **`TaskScope`**: Enum defining the lifecycle scopes: `SESSION`, `MODULE`, `CLASS`, and `FUNCTION`. +* **`Barriers`**: Autouse fixtures that enforce synchronization. They block the main thread until all background tasks for a specific scope are complete. + * Setup Barriers: + * `setup_barrier_session` + * `setup_barrier_module` + * `setup_barrier_class` + * `setup_barrier_function` + * Teardown Barriers: + * `teardown_barrier_session` + * `teardown_barrier_module` + * `teardown_barrier_class` + * `teardown_barrier_function` + +### 3.2 Execution Lifecycle + +The manager hooks into the pytest lifecycle to coordinate parallel execution: + +```mermaid +sequenceDiagram + participant Pytest + participant ParallelManager + participant ThreadPool + participant MonitorThread + participant Fixture + participant Barrier + + Note over Pytest,Barrier: Session Setup Phase + Pytest->>ParallelManager: Create (session scope) + ParallelManager->>ThreadPool: Initialize worker threads + ParallelManager->>MonitorThread: Start monitoring + + Pytest->>Fixture: Execute fixture (session scope) + Fixture->>ParallelManager: submit_setup_task(SESSION, func) + ParallelManager->>ThreadPool: Submit task to thread pool + ThreadPool-->>ParallelManager: Return future + Fixture-->>Pytest: Yield immediately + + Pytest->>Barrier: setup_barrier_session + Barrier->>ParallelManager: wait_for_setup_tasks(SESSION) + ParallelManager->>ThreadPool: Wait for all session tasks + ThreadPool-->>ParallelManager: Tasks complete + + Note over Pytest,Barrier: Module/Class/Function Scopes + Pytest->>Fixture: Execute fixture (module scope) + Fixture->>ParallelManager: submit_setup_task(MODULE, func) + ParallelManager->>ThreadPool: Submit to pool + Fixture-->>Pytest: Yield + + Pytest->>Barrier: setup_barrier_module + Barrier->>ParallelManager: wait_for_setup_tasks(MODULE) + ParallelManager->>ThreadPool: Wait for module tasks + + Note over Pytest,Barrier: Test Execution + Pytest->>ParallelManager: pytest_runtest_call hook + ParallelManager->>ThreadPool: Ensure all tasks complete + ParallelManager->>ThreadPool: Terminate executor + Pytest->>Pytest: Run test function + + Note over Pytest,Barrier: Teardown Phase + Pytest->>ParallelManager: pytest_runtest_teardown hook + ParallelManager->>ThreadPool: Reset and create new executor + ParallelManager->>MonitorThread: Restart monitoring + + Pytest->>Fixture: Teardown fixture + Fixture->>ParallelManager: submit_teardown_task(scope, func) + ParallelManager->>ThreadPool: Submit teardown task + Fixture-->>Pytest: Return + + Pytest->>Barrier: teardown_barrier_function + Barrier->>ParallelManager: wait_for_teardown_tasks(FUNCTION) + ParallelManager->>ThreadPool: Wait for function teardowns + + Pytest->>ParallelManager: pytest_runtest_logreport hook + ParallelManager->>ThreadPool: Terminate executor + ParallelManager->>MonitorThread: Stop monitoring + +``` + +#### Setup Phase + +1. **Submission**: Fixtures submit setup functions using `parallel_manager.submit_setup_task(scope, func, *args, **kwargs)`. +2. **Non-Blocking Return/Yield**: The fixture yields/Returns immediately, allowing pytest to proceed to the next fixture. +3. **Barrier Enforcement**: At the end of a scope (e.g., after all module-scoped fixtures have run), a barrier fixture waits for all submitted tasks of that scope to complete. + +#### Test Execution Phase + +1. **Pre-Test Wait**: Before the test function runs, the manager ensures all setup tasks are finished. +2. **Termination**: The setup executor is terminated to ensure a stable environment during the test. + +#### Teardown Phase + +1. **Restart**: The manager is restarted to handle teardown tasks. +2. **Submission**: Fixtures submit teardown functions using `parallel_manager.submit_teardown_task(scope, func, *args, **kwargs)`. +3. **Barrier Enforcement**: Teardown barriers wait for tasks to complete before moving to the next scope. + +## 4. Exception Handling Strategy + +The system implements a **Fail-Fast** strategy to detect exceptions in the background threads and fail the main Pytest thread timely, which helps prevent cascading failures and wasted execution time. + +* **Background Exception Logging**: The monitor thread detects and logs exceptions in worker threads as they happen. +* **Checkpoints**: + * **`pytest_fixture_setup`**: Before starting *any* fixture, the manager checks if a background task has failed. If so, it raises `ParallelTaskRuntimeError` immediately, aborting the test setup immediately. + * **Barriers**: When waiting at a barrier, exceptions from failed tasks are re-raised in the main thread. +* **Forced Termination**: In cases of interrupts or critical failures, `ctypes` is used to inject exceptions into worker threads to force immediate termination. + +## 5. Pytest Hooks Integration + +The plugin relies on several pytest hooks to orchestrate the flow: + +* **`pytest_runtest_setup`**: Dynamically reorders fixtures to ensure that barrier fixtures always execute **after** all other fixtures of the same scope. +* **`pytest_fixture_setup`**: Performs as exception handling checkpoint to interrupt the test execution in case of any exceptions are detected in the background threads. +* **`pytest_runtest_call`**: Acts as a final gate before the test runs, ensuring all setup tasks are done and terminating the setup executor. +* **`pytest_exception_interact`**: Handles exceptions during setup/call to terminate the manager gracefully. +* **`pytest_runtest_teardown`**: Restarts the parallel manager to prepare for the teardown phase. +* **`pytest_runtest_logreport`**: Terminates the parallel manager gracefully after teardown is complete. + +## 6. Deadlock Handling + +The Parallel Fixture Manager introduces multi-threading to the test execution environment. When combined with multi-processing (e.g., Ansible execution, `multiprocessing.Process`), this creates a risk of deadlocks, particularly involving logging locks. + +A common scenario is: +1. Thread A (main thread) calls a logging function and acquires the logging lock. +2. Thread B (parallel fixture manager worker) forks a new process (e.g., to run an Ansible task). +3. The child process inherits the memory state, including the held logging lock. +4. Since Thread A does not exist in the child process, the lock remains held indefinitely. +5. If the child process tries to log something, it attempts to acquire the lock and deadlocks. + +To prevent this, the framework (in `tests/common/helpers/parallel.py`) leverages `os.register_at_fork` hooks to: +* Acquire logging locks before forking. +* Release logging locks after forking (in both parent and child). +* Handle Ansible display locks similarly. + +This ensures that **locks are always in a released state within the child process immediately after forking**. + +```mermaid +sequenceDiagram + participant Main as Main Thread + participant Worker as Parallel Fixture Manager Worker + participant Logger as Logging Module + participant Handler as Log Handler + participant Fork as Fork Operation + participant Child as Child Process + + Note over Main,Child: Before Fix (Deadlock Scenario) + Worker->>Logger: Write log (acquires handler lock) + activate Handler + Main->>Fork: fork() called + Note over Fork: Lock state copied to child + Fork->>Child: Child process created + Note over Child: Child inherits locked handler + Child->>Handler: Attempt to log + Note over Child: DEADLOCK: Lock already held by Parallel Fixture Manager Worker + deactivate Handler + + Note over Main,Child: After Fix (Safe Fork) + Main->>Logger: _fix_logging_handler_fork_lock() + Logger->>Handler: Register at_fork handlers + Note over Handler: before=lock.acquire
after_in_parent=lock.release
after_in_child=lock.release + Main->>Fork: fork() called + Fork->>Handler: Execute before fork (acquire lock) + activate Handler + Fork->>Child: Child process created + Fork->>Handler: Execute after_in_parent (release lock) + deactivate Handler + Fork->>Child: Execute after_in_child (release lock) + Child->>Handler: Attempt to log + Note over Child: SUCCESS: Lock is free + Handler-->>Child: Log written successfully +``` + +## 7. Usage Example + +Fixtures interact with the parallel manager via the `parallel_manager` fixture. + +```python +import pytest +from tests.common.plugins.parallel_fixture import TaskScope + +@pytest.fixture(scope="module") +def heavy_initialization(parallel_manager): + def setup_logic(): + # Configure DUT + ... + + def teardown_logic(): + # Cleanup resources + ... + + # Submit setup task to run in background + future = parallel_manager.submit_setup_task(TaskScope.MODULE, setup_logic) + + # Yield immediately to let other fixtures start + yield + + # Submit teardown only if setup completed successfully + if parallel_manager.is_task_finished(future): + parallel_manager.submit_teardown_task(TaskScope.MODULE, teardown_logic) +``` diff --git a/tests/common/plugins/parallel_fixture/__init__.py b/tests/common/plugins/parallel_fixture/__init__.py new file mode 100644 index 00000000000..07610f8590f --- /dev/null +++ b/tests/common/plugins/parallel_fixture/__init__.py @@ -0,0 +1,674 @@ +import bisect +import contextlib +import ctypes +import enum +import functools +import logging +import pytest +import threading +import time +import traceback +import sys + +from concurrent.futures import CancelledError +from concurrent.futures import FIRST_EXCEPTION +from concurrent.futures import ALL_COMPLETED +from concurrent.futures import Future +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import TimeoutError +from concurrent.futures import wait + + +class TaskScope(enum.Enum): + """Defines the lifecycle scopes for parallel task.""" + SESSION = 0 + MODULE = 1 + CLASS = 2 + FUNCTION = 3 + + +class ParallelTaskRuntimeError(BaseException): + pass + + +class ParallelTaskTerminatedError(BaseException): + pass + + +def raise_async_exception(tid, exc_type): + """Injects an exception into the specified thread.""" + if not isinstance(tid, int): + raise TypeError("Thread ID must be an integer") + + ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(tid), + ctypes.py_object(exc_type)) + + +_log_context = threading.local() +_original_log_factory = logging.getLogRecordFactory() + + +def _prefixed_log_factory(*args, **kwargs): + record = _original_log_factory(*args, **kwargs) + # Check if we are inside a parallel task wrapper + prefix = getattr(_log_context, "prefix", None) + if prefix: + # Prepend the prefix to the log message + # This handles standard logging.info("msg") calls + record.msg = f"{prefix} {record.msg}" + return record + + +# Apply the factory globally +logging.setLogRecordFactory(_prefixed_log_factory) + + +class ParallelFixtureManager(object): + + DEFAULT_WAIT_TIMEOUT = 180 + THREAD_POOL_POLLING_INTERVAL = 0.1 + + TASK_SCOPE_SESSION = TaskScope.SESSION + TASK_SCOPE_MODULE = TaskScope.MODULE + TASK_SCOPE_CLASS = TaskScope.CLASS + TASK_SCOPE_FUNCTION = TaskScope.FUNCTION + + class ParallelTaskFuture(Future): + """A Future subclass that supports timeout handling with thread interruption.""" + + @property + def default_result(self): + if hasattr(self, '_default_result'): + return self._default_result + return None + + @default_result.setter + def default_result(self, value): + self._default_result = value + + @property + def timeout(self): + if hasattr(self, '_timeout'): + return self._timeout + return None + + @timeout.setter + def timeout(self, value): + self._timeout = value + + def result(self, timeout=None, interrupt_when_timeout=False, + return_default_on_timeout=False): + try: + return super().result(timeout=timeout) + except TimeoutError: + task_name = self.task_name + if self.cancel(): + logging.warning("[Parallel Fixture] Task %s timed out and was cancelled.", task_name) + elif self.running() and interrupt_when_timeout: + task_context = getattr(self, 'task_context', None) + if task_context and hasattr(task_context, 'tid'): + tid = task_context.tid + if tid: + logging.warning( + "[Parallel Fixture] Task %s timed out. Interrupting thread %s.", + task_name, tid + ) + raise_async_exception(tid, ParallelTaskTerminatedError) + else: + logging.warning("[Parallel Fixture] Task %s timed out but TID not found.", task_name) + if return_default_on_timeout: + logging.info("[Parallel Fixture] Task %s returning default result on timeout: %s", + task_name, self.default_result) + return self.default_result + raise + + class ParallelTaskContext(object): + """Context information for a parallel task.""" + def __init__(self, tid=None, start_time=None, end_time=None, task_name=None): + self.tid = tid + self.start_time = start_time + self.end_time = end_time + self.task_name = task_name + + def __init__(self, worker_count): + self.terminated = False + self.worker_count = worker_count + self.executor = ThreadPoolExecutor(max_workers=worker_count) + + # Initialize buckets for all defined scopes + self.setup_futures = {scope: [] for scope in TaskScope} + self.teardown_futures = {scope: [] for scope in TaskScope} + self.current_scope = None + + # Start the background monitor thread + self.monitor_lock = threading.Lock() + self.active_futures = set() + self.done_futures = set() + self.is_monitor_running = True + self.monitor_thread = threading.Thread(target=self._monitor_workers, daemon=True) + self.monitor_thread.start() + + def _monitor_workers(self): + """Monitor thread pool tasks.""" + i = 0 + while True: + future_threads = {} + with self.monitor_lock: + done_futures = set() + for f in self.active_futures: + tid = f.task_context.tid + if tid is not None: + future_threads[tid] = f + if f.done(): + done_futures.add(f) + if f.exception(): + logging.info("[Parallel Fixture] Detect exception from task %s: %s", + f.task_name, f.exception()) + else: + logging.info("[Parallel Fixture] Detect task %s is done", f.task_name) + self.active_futures -= done_futures + self.done_futures |= done_futures + + if i % 100 == 0: + # Log the running task of each thread pool worker + # every 10 seconds + log_msg = ["[Parallel Fixture] Current worker threads status:"] + current_time = time.time() + if self.executor._threads: + current_threads = list(self.executor._threads) + current_threads.sort(key=lambda t: (len(t.name), t.name)) + for thread in current_threads: + if thread.is_alive(): + if thread.ident in future_threads: + start_time = future_threads[thread.ident].task_context.start_time + log_msg.append(f"Thread {thread.name}: " + f"{future_threads[thread.ident].task_name}, " + f"{current_time - start_time}s") + else: + log_msg.append(f"Thread {thread.name}: idle") + else: + log_msg.append(f"Thread {thread.name}: terminated") + else: + log_msg.append("No alive worker thread found.") + logging.info("\n".join(log_msg)) + + if not self.is_monitor_running: + break + + time.sleep(ParallelFixtureManager.THREAD_POOL_POLLING_INTERVAL) + i += 1 + + def _resolve_scope(self, scope): + """Ensure scope is a TaskScope Enum member.""" + if isinstance(scope, TaskScope): + return scope + try: + return TaskScope(scope) + except ValueError: + raise ValueError(f"Invalid scope '{scope}'. " + f"Must be one of {[e.value for e in TaskScope]}") + + def _cancel_futures(self, futures): + for future in futures: + future.cancel() + + def _wait_for_futures(self, futures, timeout, + wait_strategy=FIRST_EXCEPTION, reraise=True, + raise_timeout_error=True): + if not futures: + return + + # Wait for all futures to complete + done, not_done = wait(futures, timeout=timeout, return_when=wait_strategy) + + # Check for exceptions in completed tasks + for future in done: + if future.exception(): + # If any exception is raised, cancel the rest + self._cancel_futures(not_done) + if reraise: + raise ParallelTaskRuntimeError from future.exception() + + # Wait timeout, cancel the rest + if not_done: + # Attempt cancel to cleanup + self._cancel_futures(not_done) + if raise_timeout_error: + raise TimeoutError( + f"Parallel Tasks Timed Out! " + f"{len(not_done)} tasks failed to complete within {timeout}s: " + f"{[f.task_name for f in not_done]}" + ) + + def _format_task_name(self, func, *args, **kwargs): + task_name = f"{func.__name__}" + if args: + task_name += f"({args}" + if kwargs: + task_name += f", {kwargs}" + if args or kwargs: + task_name += ")" + return task_name + + def _wrap_task(self, func, task_context): + + @functools.wraps(func) + def wrapper(*args, **kwargs): + tid = threading.get_ident() + task_context.tid = tid + task_context.start_time = time.time() + current_thread = threading.current_thread().name + + prefix = f"[Parallel Fixture][{current_thread}][{task_context.task_name}]" + # Set thread-local context for logging module + _log_context.prefix = prefix + try: + return func(*args, **kwargs) + except Exception: + _, exc_value, exc_traceback = sys.exc_info() + logging.error("[Parallel Fixture] Task %s exception:\n%s", + task_context.task_name, + traceback.format_exc()) + raise exc_value.with_traceback(exc_traceback) + finally: + _log_context.prefix = None + task_context.end_time = time.time() + logging.debug("[Parallel Fixture] Task %s finished in %.2f seconds", + task_context.task_name, task_context.end_time - task_context.start_time) + + return wrapper + + def wait_for_tasks_completion(self, futures, timeout=DEFAULT_WAIT_TIMEOUT, + wait_strategy=ALL_COMPLETED, reraise=True): + """Block until all given tasks are done.""" + logging.debug("[Parallel Fixture] Waiting for tasks to finish, timeout: %s", timeout) + self._wait_for_futures(futures, timeout, wait_strategy, reraise) + + def submit_setup_task(self, scope, func, *args, **kwargs): + """Submit a setup task to the parallel fixture manager.""" + scope = self._resolve_scope(scope) + task_name = self._format_task_name(func, *args, **kwargs) + logging.info("[Parallel Fixture] Submit setup task (%s): %s", scope, task_name) + task_context = ParallelFixtureManager.ParallelTaskContext(task_name=task_name) + wrapped_func = self._wrap_task(func, task_context) + future = self.executor.submit(wrapped_func, *args, **kwargs) + future.__class__ = ParallelFixtureManager.ParallelTaskFuture + future.task_name = task_name + future.task_context = task_context + self.setup_futures[scope].append(future) + with self.monitor_lock: + self.active_futures.add(future) + return future + + def submit_teardown_task(self, scope, func, *args, **kwargs): + """Submit a teardown task to the parallel fixture manager.""" + scope = self._resolve_scope(scope) + task_name = self._format_task_name(func, *args, **kwargs) + logging.info("[Parallel Fixture] Submit teardown task (%s): %s", scope, task_name) + task_context = ParallelFixtureManager.ParallelTaskContext(task_name=task_name) + wrapped_func = self._wrap_task(func, task_context) + future = self.executor.submit(wrapped_func, *args, **kwargs) + future.__class__ = ParallelFixtureManager.ParallelTaskFuture + future.task_name = task_name + future.task_context = task_context + self.teardown_futures[scope].append(future) + with self.monitor_lock: + self.active_futures.add(future) + return future + + def wait_for_setup_tasks(self, scope, + timeout=DEFAULT_WAIT_TIMEOUT, + wait_strategy=FIRST_EXCEPTION, reraise=True): + """Block until all setup tasks in a specific scope are done.""" + logging.debug("[Parallel Fixture] Waiting for setup tasks to finish, scope: %s, timeout: %s", scope, timeout) + scope = self._resolve_scope(scope) + futures = self.setup_futures.get(scope, []) + self._wait_for_futures(futures, timeout, wait_strategy, reraise) + + def wait_for_teardown_tasks(self, scope, + timeout=DEFAULT_WAIT_TIMEOUT, + wait_strategy=FIRST_EXCEPTION, reraise=True): + """Block until all teardown tasks in a specific scope are done.""" + logging.debug("[Parallel Fixture] Waiting for teardown tasks to finish, scope: %s, timeout: %s", scope, timeout) + scope = self._resolve_scope(scope) + futures = self.teardown_futures.get(scope, []) + self._wait_for_futures(futures, timeout, wait_strategy, reraise) + + def terminate(self): + """Terminate the parallel fixture manager.""" + + if self.terminated: + return + + logging.info("[Parallel Fixture] Terminating parallel fixture manager") + + self.terminated = True + + # Stop the monitor + self.is_monitor_running = False + self.monitor_thread.join() + + # Cancel any pending futures + for future in self.active_futures: + future.cancel() + + # Force terminate the thread pool workers that are still running + running_futures = [future for future in self.active_futures if not future.done()] + logging.debug("[Parallel Fixture] Running tasks to be terminated: %s", [_.task_name for _ in running_futures]) + if running_futures: + logging.debug("[Parallel Fixture] Force interrupt thread pool workers") + running_futures_tids = [future.task_context.tid for future in running_futures + if future.task_context.tid is not None] + for thread in self.executor._threads: + if thread.is_alive() and thread.ident in running_futures_tids: + raise_async_exception(thread.ident, ParallelTaskTerminatedError) + + logging.debug("[Parallel Fixture] Current worker threads: %s", + [thread.is_alive() for thread in self.executor._threads]) + # Wait for all threads to terminate + self.executor.shutdown(wait=True) + logging.debug("[Parallel Fixture] Current worker threads: %s", + [thread.is_alive() for thread in self.executor._threads]) + + cancel_futures = [] + stopped_futures = [] + pending_futures = [] + done_futures = self.done_futures + for future in self.active_futures: + try: + exc = future.exception(0.1) + if isinstance(exc, ParallelTaskTerminatedError): + stopped_futures.append(future) + except CancelledError: + cancel_futures.append(future) + except TimeoutError: + # NOTE: should never hit this as all futures are either + # cancelled or stopped with ParallelTaskTerminatedError + pending_futures.append(future) + + logging.debug(f"[Parallel Fixture] The fixture manager is terminated:\n" + f"stopped tasks {[_.task_name for _ in stopped_futures]},\n" + f"canceled tasks {[_.task_name for _ in cancel_futures]},\n" + f"pending tasks {[_.task_name for _ in pending_futures]},\n" + f"done tasks {[(_.task_name, _.exception()) for _ in done_futures]}.") + + def reset(self): + """Reset the parallel fixture manager.""" + if not self.terminated: + raise RuntimeError("Cannot reset a running parallel fixture manager.") + + # Reinitialize buckets for all defined scopes + self.setup_futures = {scope: [] for scope in TaskScope} + self.teardown_futures = {scope: [] for scope in TaskScope} + self.current_scope = None + + self.active_futures.clear() + self.done_futures.clear() + self.executor = ThreadPoolExecutor(max_workers=self.worker_count) + self.is_monitor_running = True + self.monitor_thread = threading.Thread(target=self._monitor_workers, daemon=True) + self.monitor_thread.start() + self.terminated = False + + def check_for_exception(self): + """Check done futures and re-raise any exception.""" + with self.monitor_lock: + for future in self.done_futures: + if future.exception(): + raise ParallelTaskRuntimeError from future.exception() + + def is_task_finished(self, future): + return future.done() and future.exception() is None + + def __del__(self): + self.terminate() + + +@contextlib.contextmanager +def log_function_call_duration(func_name): + start = time.time() + logging.debug("[Parallel Fixture] Start %s", func_name) + yield + logging.debug("[Parallel Fixture] End %s, duration %s", func_name, time.time() - start) + + +# ----------------------------------------------------------------- +# the parallel manager fixture +# ----------------------------------------------------------------- + + +_PARALLEL_MANAGER = None + + +@pytest.fixture(scope="session", autouse=True) +def parallel_manager(tbinfo): + dut_count = len(tbinfo.get("duts", [])) + worker_count = max(dut_count * 8, 16) + global _PARALLEL_MANAGER + _PARALLEL_MANAGER = ParallelFixtureManager(worker_count=worker_count) + _PARALLEL_MANAGER.current_scope = TaskScope.SESSION + return _PARALLEL_MANAGER + + +# ----------------------------------------------------------------- +# the setup barrier fixtures +# ----------------------------------------------------------------- + + +@pytest.fixture(scope="session", autouse=True) +def setup_barrier_session(parallel_manager): + """Barrier to wait for all session level setup tasks to finish.""" + with log_function_call_duration("setup_barrier_session"): + parallel_manager.wait_for_setup_tasks(TaskScope.SESSION) + parallel_manager.current_scope = TaskScope.MODULE + yield + return + + +@pytest.fixture(scope="module", autouse=True) +def setup_barrier_module(parallel_manager): + """Barrier to wait for all module level setup tasks to finish.""" + with log_function_call_duration("setup_barrier_module"): + parallel_manager.wait_for_setup_tasks(TaskScope.MODULE) + parallel_manager.current_scope = TaskScope.CLASS + yield + return + + +@pytest.fixture(scope="class", autouse=True) +def setup_barrier_class(parallel_manager): + """Barrier to wait for all class level setup tasks to finish.""" + with log_function_call_duration("setup_barrier_class"): + parallel_manager.wait_for_setup_tasks(TaskScope.CLASS) + parallel_manager.current_scope = TaskScope.FUNCTION + yield + return + + +@pytest.fixture(scope="function", autouse=True) +def setup_barrier_function(parallel_manager): + """Barrier to wait for all function level setup tasks to finish.""" + with log_function_call_duration("setup_barrier_function"): + parallel_manager.wait_for_setup_tasks(TaskScope.FUNCTION) + parallel_manager.current_scope = None + yield + return + + +# ----------------------------------------------------------------- +# the teardown barrier fixtures +# ----------------------------------------------------------------- + + +@pytest.fixture(scope="session", autouse=True) +def teardown_barrier_session(parallel_manager): + """Barrier to wait for all session level teardown tasks to finish.""" + yield + with log_function_call_duration("teardown_barrier_session"): + parallel_manager.wait_for_teardown_tasks(TaskScope.SESSION) + parallel_manager.current_scope = None + + +@pytest.fixture(scope="module", autouse=True) +def teardown_barrier_module(parallel_manager): + """Barrier to wait for all module level teardown tasks to finish.""" + yield + with log_function_call_duration("teardown_barrier_module"): + parallel_manager.wait_for_teardown_tasks(TaskScope.MODULE) + parallel_manager.current_scope = TaskScope.SESSION + + +@pytest.fixture(scope="class", autouse=True) +def teardown_barrier_class(parallel_manager): + """Barrier to wait for all class level teardown tasks to finish.""" + yield + with log_function_call_duration("teardown_barrier_class"): + parallel_manager.wait_for_teardown_tasks(TaskScope.CLASS) + parallel_manager.current_scope = TaskScope.MODULE + + +@pytest.fixture(scope="function", autouse=True) +def teardown_barrier_function(parallel_manager): + """Barrier to wait for all function level teardown tasks to finish.""" + yield + with log_function_call_duration("teardown_barrier_function"): + parallel_manager.wait_for_teardown_tasks(TaskScope.FUNCTION) + parallel_manager.current_scope = TaskScope.CLASS + + +# ----------------------------------------------------------------- +# pytest hooks +# ----------------------------------------------------------------- + + +@pytest.hookimpl(wrapper=True) +def pytest_runtest_setup(item): + """ + HOOK: Runs once BEFORE every fixture setup. + Reorder the setup/teardown barriers to ensure barriers should run + after ALL fixtures of the same-scope. + """ + logging.debug("[Parallel Fixture] Setup barrier fixtures") + + barriers = { + TaskScope.SESSION.value: ["teardown_barrier_session", + "setup_barrier_session"], + TaskScope.MODULE.value: ["teardown_barrier_module", + "setup_barrier_module"], + TaskScope.CLASS.value: ["teardown_barrier_class", + "setup_barrier_class"], + TaskScope.FUNCTION.value: ["teardown_barrier_function", + "setup_barrier_function"] + } + fixtureinfo = item._fixtureinfo + current_fixture_names = fixtureinfo.names_closure[:] + + logging.debug("[Parallel Fixture] Fixture order before:\n%s", current_fixture_names) + + for fixtures in barriers.values(): + for fixture in fixtures: + current_fixture_names.remove(fixture) + current_fixture_scopes = [] + for fixture in current_fixture_names: + fixture_defs = fixtureinfo.name2fixturedefs.get(fixture, []) + if not fixture_defs: + fixture_scope = current_fixture_scopes[-1] + else: + try: + fixture_scope = TaskScope[fixture_defs[0].scope.upper()].value + except Exception: + logging.debug("[Parallel Fixture] Unknown fixture scope for %r," + "default to previous scope", fixture_defs) + fixture_scope = current_fixture_scopes[-1] + current_fixture_scopes.append(fixture_scope) + + # NOTE: Inject the barriers to ensure they are running last + # in the fixtures of the same scope. + for scope, fixtures in barriers.items(): + for fixture in fixtures: + if fixture.startswith("setup"): + insert_pos = bisect.bisect_right(current_fixture_scopes, scope) + current_fixture_names.insert(insert_pos, fixture) + current_fixture_scopes.insert(insert_pos, scope) + if fixture.startswith("teardown"): + insert_pos = bisect.bisect_left(current_fixture_scopes, scope) + current_fixture_names.insert(insert_pos, fixture) + current_fixture_scopes.insert(insert_pos, scope) + + logging.debug("[Parallel Fixture] Fixture order after:\n%s", current_fixture_names) + fixtureinfo.names_closure[:] = current_fixture_names + + yield + return + + +@pytest.hookimpl(tryfirst=True) +def pytest_fixture_setup(fixturedef, request): + """ + HOOK: Runs BEFORE every fixture setup. + If a background task failed while the PREVIOUS fixture was running, + we catch it here and stop the next fixture from starting. + """ + if _PARALLEL_MANAGER: + logging.debug("[Parallel Fixture] Check for fixture exceptions before running %r", fixturedef) + _PARALLEL_MANAGER.check_for_exception() + + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_call(item): + """ + HOOK: Runs BEFORE the test function starts. + Happy path to terminate the parallel fixture manager. + All tasks should be done as those barrier fixtures should catch them + all. + """ + logging.debug("[Parallel Fixture] Wait for tasks to finish before test function") + parallel_manager = _PARALLEL_MANAGER + if parallel_manager: + try: + for scope in TaskScope: + parallel_manager.wait_for_setup_tasks(scope) + finally: + parallel_manager.terminate() + + +def pytest_exception_interact(call, report): + """ + HOOK: Runs WHEN an exception occurs. + Sad path to terminate the parallel fixture manager. + When a ParallelTaskRuntimeError is detected, tries to poll + the rest running tasks and terminate the parallel manager. + """ + parallel_manager = _PARALLEL_MANAGER + if parallel_manager and report.when == "setup": + reraise = not isinstance(call.excinfo.value, ParallelTaskRuntimeError) + logging.debug("[Parallel Fixture] Wait for tasks to finish after exception occurred in setup %s", + call.excinfo.value) + try: + for scope in TaskScope: + parallel_manager.wait_for_setup_tasks(scope, wait_strategy=ALL_COMPLETED, reraise=reraise) + finally: + parallel_manager.terminate() + + +def pytest_runtest_teardown(item, nextitem): + """ + HOOK: Runs once BEFORE all fixture teardown. + Reset the parallel manager. + """ + logging.debug("[Parallel Fixture] Reset parallel manager before teardown") + parallel_manager = _PARALLEL_MANAGER + if parallel_manager: + parallel_manager.reset() + parallel_manager.current_scope = TaskScope.FUNCTION + + +def pytest_runtest_logreport(report): + """ + HOOK: Runs once AFTER all fixture teardown. + Terminate the parallel manager. + """ + logging.debug("[Parallel Fixture] Terminate parallel manager after teardown") + parallel_manager = _PARALLEL_MANAGER + if parallel_manager: + parallel_manager.terminate() diff --git a/tests/common/plugins/parallel_fixture/images/pytest.jpg b/tests/common/plugins/parallel_fixture/images/pytest.jpg new file mode 100644 index 00000000000..f066d793e6d Binary files /dev/null and b/tests/common/plugins/parallel_fixture/images/pytest.jpg differ diff --git a/tests/conftest.py b/tests/conftest.py index dd4801144fe..724ece0e258 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -115,7 +115,8 @@ 'tests.common.plugins.conditional_mark', 'tests.common.plugins.random_seed', 'tests.common.plugins.memory_utilization', - 'tests.common.fixtures.duthost_utils') + 'tests.common.fixtures.duthost_utils', + 'tests.common.plugins.parallel_fixture') patch_ansible_worker_process() @@ -793,8 +794,8 @@ def ptfhosts(enhance_inventory, ansible_adhoc, tbinfo, duthost, request): # when no ptf defined in testbed.csv # try to parse it from inventory ptf_host = duthost.host.options["inventory_manager"].get_host(duthost.hostname).get_vars()["ptf_host"] - _hosts.apend(PTFHost(ansible_adhoc, ptf_host, duthost, tbinfo, - macsec_enabled=request.config.option.enable_macsec)) + _hosts.append(PTFHost(ansible_adhoc, ptf_host, duthost, tbinfo, + macsec_enabled=request.config.option.enable_macsec)) return _hosts diff --git a/tests/container_upgrade/container_upgrade_helper.py b/tests/container_upgrade/container_upgrade_helper.py index e49eca9683e..dcfea0081dd 100644 --- a/tests/container_upgrade/container_upgrade_helper.py +++ b/tests/container_upgrade/container_upgrade_helper.py @@ -28,6 +28,9 @@ "docker-gnmi-watchdog": "gnmi_watchdog", "docker-sonic-bmp": "bmp", "docker-bmp-watchdog": "bmp_watchdog", + "docker-sonic-restapi": "restapi", + "docker-restapi-watchdog": "restapi_watchdog", + "docker-restapi-sidecar": "restapi_sidecar", } diff --git a/tests/container_upgrade/parameters.json b/tests/container_upgrade/parameters.json index bbca808f852..e90439696c6 100644 --- a/tests/container_upgrade/parameters.json +++ b/tests/container_upgrade/parameters.json @@ -13,5 +13,14 @@ }, "docker-bmp-watchdog": { "parameters": "--pid=host --net=host -v /etc/localtime:/etc/localtime:ro -v /etc/sonic:/etc/sonic:ro" + }, + "docker-sonic-restapi": { + "parameters": "--net=host -v /var/run/redis/redis.sock:/var/run/redis/redis.sock -v /etc/sonic/credentials:/etc/sonic/credentials:ro -v /etc/localtime:/etc/localtime:ro" + }, + "docker-restapi-watchdog": { + "parameters": "--net=host -v /etc/localtime:/etc/localtime:ro" + }, + "docker-restapi-sidecar": { + "parameters": "--privileged --pid=host --net=host --uts=host --ipc=host -v /etc/sonic:/etc/sonic:ro -v /usr/bin/docker:/usr/bin/docker:ro -v /var/run/docker.sock:/var/run/docker.sock -v /:/hostroot:ro -e DOCKER_BIN=/usr/bin/docker" } } diff --git a/tests/container_upgrade/restapi_testcases.json b/tests/container_upgrade/restapi_testcases.json new file mode 100644 index 00000000000..6f50fa0091c --- /dev/null +++ b/tests/container_upgrade/restapi_testcases.json @@ -0,0 +1,4 @@ +{ + "restapi/test_restapi_client_cert_auth.py": 0, + "restapi/test_restapi_vxlan_ecmp.py": 0 +} diff --git a/tests/crm/test_crm.py b/tests/crm/test_crm.py index 00b01eac01f..de91c73aa16 100755 --- a/tests/crm/test_crm.py +++ b/tests/crm/test_crm.py @@ -28,6 +28,11 @@ logger = logging.getLogger(__name__) SONIC_RES_UPDATE_TIME = 50 +SONIC_RES_CLEANUP_UPDATE_TIME = 20 +CONFIG_UPDATE_TIME = 5 +FDB_CLEAR_TIMEOUT = 20 +ROUTE_COUNTER_POLL_TIMEOUT = 15 +CRM_COUNTER_TOLERANCE = 2 ACL_TABLE_NAME = "DATAACL" RESTORE_CMDS = {"test_crm_route": [], @@ -126,7 +131,9 @@ def apply_acl_config(duthost, asichost, test_name, collector, entry_num=1): pytest.skip("DATAACL does not exist") # Make sure CRM counters updated - time.sleep(CRM_UPDATE_TIME) + # ACL configuration usually propagates in 3-5 seconds + logger.info("Waiting for ACL configuration to propagate...") + time.sleep(CONFIG_UPDATE_TIME) collector["acl_tbl_key"] = get_acl_tbl_key(asichost) @@ -206,7 +213,9 @@ def apply_fdb_config(duthost, test_name, vlan_id, iface, entry_num): duthost.command(cmd) # Make sure CRM counters updated - time.sleep(CRM_UPDATE_TIME) + # FDB entries typically propagate in 3-5 seconds + logger.info("Waiting for FDB entries to propagate...") + time.sleep(CONFIG_UPDATE_TIME) def get_acl_tbl_key(asichost): @@ -315,6 +324,104 @@ def check_crm_stats(cmd, duthost, origin_crm_stats_used, origin_crm_stats_availa return False +def wait_for_crm_counter_update(cmd, duthost, expected_used, oper_used=">=", timeout=15, interval=1): + """ + Wait for CRM used counter to update by polling. + Raises pytest.fail() if timeout is reached. + """ + last_values = {'used': None, 'avail': None} + + def check_counters(): + try: + crm_used, crm_avail = get_crm_stats(cmd, duthost) + last_values['used'] = crm_used + last_values['avail'] = crm_avail + used_ok = eval(f"{crm_used} {oper_used} {expected_used}") + + if used_ok: + logger.info(f"CRM counter updated: used={crm_used} (expected {oper_used} {expected_used})") + return True + else: + logger.debug(f"Waiting for CRM update: used={crm_used} (expected {oper_used} {expected_used})") + return False + except Exception as e: + logger.debug(f"Error checking CRM stats: {e}") + return False + + pytest_assert(wait_until(timeout, interval, 0, check_counters), + f"CRM counter did not reach expected value within {timeout} seconds. " + f"Expected: used {oper_used} {expected_used}, " + f"Actual: used={last_values['used']}, available={last_values['avail']}") + + +def wait_for_resource_stabilization(get_stats_func, duthost, asic_index, resource_key, + min_expected_used=None, tolerance_percent=5, + timeout=60, interval=5): + """ + Wait for large resource configurations to stabilize by polling. + Raises pytest.fail() if timeout is reached. + """ + logger.info("Waiting for {} resources to stabilize (expecting at least {} used)...".format( + resource_key, min_expected_used if min_expected_used else "N/A")) + + stable_count = 0 + prev_used = None + start_time = time.time() + + def check_stabilized(): + nonlocal stable_count, prev_used + try: + stats = get_stats_func(duthost, asic_index) + current_used = stats[resource_key]['used'] + current_avail = stats[resource_key]['available'] + + logger.debug("{} used: {}, available: {}".format(resource_key, current_used, current_avail)) + + # Check if we've reached minimum expected usage + if min_expected_used and current_used < min_expected_used * (1 - tolerance_percent / 100): + logger.debug("Still adding resources: {} < {} (min expected)".format( + current_used, min_expected_used)) + prev_used = current_used + stable_count = 0 + return False + + # Check if counter is stable (not changing significantly) + if prev_used is not None: + change = abs(current_used - prev_used) + if change <= max(1, current_used * tolerance_percent / 100): + stable_count += 1 + if stable_count >= 2: # Stable for 2 consecutive checks + logger.info("{} resources stabilized at used={}, available={}".format( + resource_key, current_used, current_avail)) + return True + else: + stable_count = 0 + + prev_used = current_used + return False + except Exception as e: + logger.debug("Error checking resource stats: {}".format(e)) + return False + + if wait_until(timeout, interval, 0, check_stabilized): + elapsed = time.time() - start_time + logger.info("{} stabilization took {:.1f} seconds".format(resource_key, elapsed)) + else: + # final stats for error message + try: + final_stats = get_stats_func(duthost, asic_index) + final_used = final_stats[resource_key]['used'] + final_avail = final_stats[resource_key]['available'] + except Exception: + final_used = prev_used + final_avail = "unknown" + + pytest.fail("{} resources did not stabilize within {} seconds. " + "Expected min: {}, Actual: used={}, available={}".format( + resource_key, timeout, min_expected_used if min_expected_used else "N/A", + final_used, final_avail)) + + def generate_neighbors(amount, ip_ver): """ Generate list of IPv4 or IPv6 addresses """ if ip_ver == "4": @@ -453,7 +560,9 @@ def configure_neighbors(amount, interface, ip_ver, asichost, test_name): iface=interface, namespace=asichost.namespace)) # Make sure CRM counters updated - time.sleep(CRM_UPDATE_TIME) + # Neighbor additions typically propagate in 3-5 seconds + logger.info("Waiting for neighbor entries to propagate...") + time.sleep(CONFIG_UPDATE_TIME) def get_entries_num(used, available): @@ -562,8 +671,22 @@ def test_crm_route(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro if duthost.facts['asic_type'] == 'broadcom': check_available_counters = False - # Make sure CRM counters updated - time.sleep(CRM_UPDATE_TIME) + # Helper function to get current route used counter + def get_route_used(): + stats = get_crm_resources_fdb_and_ip_route(duthost, enum_frontend_asic_index) + return stats[f'ipv{ip_ver}_route']['used'] + + # Make sure CRM counters updated - use polling to wait for route counter to update + logger.info(f"Waiting for route counters to update after adding {total_routes} routes...") + expected_min_used = crm_stats_route_used + total_routes - CRM_COUNTER_TOLERANCE + + def check_route_added(): + return get_route_used() >= expected_min_used + + pytest_assert(wait_until(ROUTE_COUNTER_POLL_TIMEOUT, CRM_POLLING_INTERVAL, 0, check_route_added), + f"Route counter did not update after adding {total_routes} routes " + f"within {ROUTE_COUNTER_POLL_TIMEOUT} seconds. " + f"Expected: used >= {expected_min_used}, Actual: used={get_route_used()}") # Get new ipv[4/6]_route/fdb_entry used and available counter value crm_stats = get_crm_resources_fdb_and_ip_route(duthost, enum_frontend_asic_index) @@ -596,8 +719,17 @@ def test_crm_route(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro for i in range(total_routes): duthost.command(route_del_cmd.format(asichost.ip_cmd, i, nh_ip)) - # Make sure CRM counters updated - time.sleep(CRM_UPDATE_TIME) + # Make sure CRM counters updated - use polling to wait for route counter to update + logger.info(f"Waiting for route counters to update after deleting {total_routes} routes...") + expected_max_used = crm_stats_route_used - total_routes + CRM_COUNTER_TOLERANCE + + def check_route_deleted(): + return get_route_used() <= expected_max_used + + pytest_assert(wait_until(ROUTE_COUNTER_POLL_TIMEOUT, CRM_POLLING_INTERVAL, 0, check_route_deleted), + f"Route counter did not update after deleting {total_routes} routes " + f"within {ROUTE_COUNTER_POLL_TIMEOUT} seconds. " + f"Expected: used <= {expected_max_used}, Actual: used={get_route_used()}") # Get new ipv[4/6]_route/fdb_entry used and available counter value crm_stats = get_crm_resources_fdb_and_ip_route(duthost, enum_frontend_asic_index) @@ -644,11 +776,14 @@ def test_crm_route(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro duthost.shell(add_routes_template.render(routes_list=routes_list, interface=crm_interface[0], namespace=asichost.namespace)) - logger.info("Waiting {} seconds for SONiC to update resources...".format(SONIC_RES_UPDATE_TIME)) - # Make sure SONIC configure expected entries - time.sleep(SONIC_RES_UPDATE_TIME) + # Wait for resources to stabilize using adaptive polling + expected_routes = new_crm_stats_route_used + routes_num + logger.info("Waiting for {} route resources to stabilize".format(routes_num)) + wait_for_resource_stabilization(get_crm_resources_fdb_and_ip_route, duthost, + enum_frontend_asic_index, 'ipv{}_route'.format(ip_ver), + min_expected_used=expected_routes, timeout=60, interval=5) - RESTORE_CMDS["wait"] = SONIC_RES_UPDATE_TIME + RESTORE_CMDS["wait"] = SONIC_RES_CLEANUP_UPDATE_TIME # Verify thresholds for "IPv[4/6] route" CRM resource # Get "crm_stats_ipv[4/6]_route" used and available counter value @@ -770,11 +905,14 @@ def test_crm_nexthop(duthosts, enum_rand_one_per_hwsku_frontend_hostname, configure_neighbors(amount=neighbours_num, interface=crm_interface[0], ip_ver=ip_ver, asichost=asichost, test_name="test_crm_nexthop") - logger.info("Waiting {} seconds for SONiC to update resources...".format(SONIC_RES_UPDATE_TIME)) - # Make sure SONIC configure expected entries - time.sleep(SONIC_RES_UPDATE_TIME) + # Wait for nexthop resources to stabilize using polling + expected_nexthop_used = new_crm_stats_nexthop_used + neighbours_num - CRM_COUNTER_TOLERANCE + logger.info("Waiting for {} nexthop resources to stabilize (expecting ~{} total used)...".format( + neighbours_num, expected_nexthop_used)) + wait_for_crm_counter_update(get_nexthop_stats, duthost, expected_used=expected_nexthop_used, + oper_used=">=", timeout=60, interval=5) - RESTORE_CMDS["wait"] = SONIC_RES_UPDATE_TIME + RESTORE_CMDS["wait"] = SONIC_RES_CLEANUP_UPDATE_TIME # Verify thresholds for "IPv[4/6] nexthop" CRM resource verify_thresholds(duthost, asichost, crm_cli_res="ipv{ip_ver} nexthop".format(ip_ver=ip_ver), @@ -853,11 +991,13 @@ def test_crm_neighbor(duthosts, enum_rand_one_per_hwsku_frontend_hostname, configure_neighbors(amount=neighbours_num, interface=crm_interface[0], ip_ver=ip_ver, asichost=asichost, test_name="test_crm_neighbor") - logger.info("Waiting {} seconds for SONiC to update resources...".format(SONIC_RES_UPDATE_TIME)) - # Make sure SONIC configure expected entries - time.sleep(SONIC_RES_UPDATE_TIME) + # Wait for neighbor resources to stabilize using polling + expected_neighbor_used = new_crm_stats_neighbor_used + neighbours_num - CRM_COUNTER_TOLERANCE + logger.info("Waiting for {} neighbor resources to stabilize".format(neighbours_num)) + wait_for_crm_counter_update(get_neighbor_stats, duthost, expected_used=expected_neighbor_used, + oper_used=">=", timeout=60, interval=5) - RESTORE_CMDS["wait"] = SONIC_RES_UPDATE_TIME + RESTORE_CMDS["wait"] = SONIC_RES_CLEANUP_UPDATE_TIME # Verify thresholds for "IPv[4/6] neighbor" CRM resource verify_thresholds(duthost, asichost, crm_cli_res="ipv{ip_ver} neighbor".format(ip_ver=ip_ver), @@ -976,11 +1116,13 @@ def test_crm_nexthop_group(duthosts, enum_rand_one_per_hwsku_frontend_hostname, asichost=asichost, test_name="test_crm_nexthop_group", chunk_size=chunk_size) - logger.info("Waiting {} seconds for SONiC to update resources...".format(SONIC_RES_UPDATE_TIME)) - # Make sure SONIC configure expected entries - time.sleep(SONIC_RES_UPDATE_TIME) + # Wait for nexthop group resources to stabilize using polling + expected_nhg_used = new_nexthop_group_used + nexthop_group_num - CRM_COUNTER_TOLERANCE + logger.info("Waiting for {} nexthop group resources to stabilize".format(nexthop_group_num)) + wait_for_crm_counter_update(get_nexthop_group_stats, duthost, expected_used=expected_nhg_used, + oper_used=">=", timeout=60, interval=5) - RESTORE_CMDS["wait"] = SONIC_RES_UPDATE_TIME + RESTORE_CMDS["wait"] = SONIC_RES_CLEANUP_UPDATE_TIME verify_thresholds(duthost, asichost, crm_cli_res=redis_threshold, crm_cmd=get_nexthop_group_stats) @@ -1058,8 +1200,11 @@ def verify_acl_crm_stats(duthost, asichost, enum_rand_one_per_hwsku_frontend_hos logger.info(f"Next hop group number: {nexthop_group_num}") apply_acl_config(duthost, asichost, "test_acl_entry", asic_collector, nexthop_group_num) - # Make sure SONIC configure expected entries - time.sleep(SONIC_RES_UPDATE_TIME) + # Wait for ACL entry resources to stabilize using polling + expected_acl_used = new_crm_stats_acl_entry_used + nexthop_group_num - CRM_COUNTER_TOLERANCE + logger.info("Waiting for {} ACL entry resources to stabilize".format(nexthop_group_num)) + wait_for_crm_counter_update(get_acl_entry_stats, duthost, expected_used=expected_acl_used, + oper_used=">=", timeout=60, interval=5) # Verify thresholds for "ACL entry" CRM resource verify_thresholds(duthost, asichost, crm_cli_res="acl group entry", crm_cmd=get_acl_entry_stats) @@ -1179,12 +1324,14 @@ def test_acl_counter(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_f crm_stats_acl_entry_available".format(db_cli=asichost.sonic_db_cli, acl_tbl_key=acl_tbl_key) _, available_acl_entry_num = get_crm_stats(get_acl_entry_stats, duthost) # The number we can applied is limited to available_acl_entry_num - apply_acl_config(duthost, asichost, "test_acl_counter", asic_collector, - min(needed_acl_counter_num, available_acl_entry_num)) + actual_acl_count = min(needed_acl_counter_num, available_acl_entry_num) + apply_acl_config(duthost, asichost, "test_acl_counter", asic_collector, actual_acl_count) - logger.info("Waiting {} seconds for SONiC to update resources...".format(SONIC_RES_UPDATE_TIME)) - # Make sure SONIC configure expected entries - time.sleep(SONIC_RES_UPDATE_TIME) + # Wait for ACL counter resources to stabilize using polling + expected_acl_counter_used = new_crm_stats_acl_counter_used + actual_acl_count - CRM_COUNTER_TOLERANCE + logger.info("Waiting for {} ACL counter resources to stabilize".format(actual_acl_count)) + wait_for_crm_counter_update(get_acl_counter_stats, duthost, expected_used=expected_acl_counter_used, + oper_used=">=", timeout=60, interval=5) new_crm_stats_acl_counter_used, new_crm_stats_acl_counter_available = \ get_crm_stats(get_acl_counter_stats, duthost) @@ -1298,11 +1445,14 @@ def test_crm_fdb_entry(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum fdb_entries_num = get_entries_num(new_crm_stats_fdb_entry_used, new_crm_stats_fdb_entry_available) # Generate FDB json file with 'fdb_entries_num' entries and apply it on DUT apply_fdb_config(duthost, "test_crm_fdb_entry", vlan_id, iface, fdb_entries_num) - logger.info("Waiting {} seconds for SONiC to update resources...".format(SONIC_RES_UPDATE_TIME)) - # Make sure SONIC configure expected entries - time.sleep(SONIC_RES_UPDATE_TIME) - RESTORE_CMDS["wait"] = SONIC_RES_UPDATE_TIME + # Wait for FDB entry resources to stabilize using polling + expected_fdb_used = new_crm_stats_fdb_entry_used + fdb_entries_num - CRM_COUNTER_TOLERANCE + logger.info("Waiting for {} FDB entry resources to stabilize".format(fdb_entries_num)) + wait_for_crm_counter_update(get_fdb_stats, duthost, expected_used=expected_fdb_used, + oper_used=">=", timeout=60, interval=5) + + RESTORE_CMDS["wait"] = SONIC_RES_CLEANUP_UPDATE_TIME # Verify thresholds for "FDB entry" CRM resource verify_thresholds(duthost, asichost, crm_cli_res="fdb", crm_cmd=get_fdb_stats) @@ -1311,16 +1461,19 @@ def test_crm_fdb_entry(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum cmd = "fdbclear" duthost.command(cmd) - # Make sure CRM counters updated - time.sleep(CRM_UPDATE_TIME) - # Timeout for asyc fdb clear - FDB_CLEAR_TIMEOUT = 10 - while FDB_CLEAR_TIMEOUT > 0: + # Make sure CRM counters updated - use polling + # Wait for FDB clear to complete with async polling + logger.info("Waiting for FDB clear to complete...") + fdb_clear_timeout = FDB_CLEAR_TIMEOUT + new_crm_stats_fdb_entry_used = None + new_crm_stats_fdb_entry_available = None + while fdb_clear_timeout > 0: # Get new "crm_stats_fdb_entry" used and available counter value new_crm_stats_fdb_entry_used, new_crm_stats_fdb_entry_available = get_crm_stats(get_fdb_stats, duthost) if new_crm_stats_fdb_entry_used == 0: + logger.debug("FDB cleared successfully") break - FDB_CLEAR_TIMEOUT -= CRM_POLLING_INTERVAL + fdb_clear_timeout -= CRM_POLLING_INTERVAL time.sleep(CRM_POLLING_INTERVAL) # Verify "crm_stats_fdb_entry_used" counter was decremented diff --git a/tests/dualtor/test_ipinip.py b/tests/dualtor/test_ipinip.py index 3a383b799e4..32066eada46 100644 --- a/tests/dualtor/test_ipinip.py +++ b/tests/dualtor/test_ipinip.py @@ -16,6 +16,7 @@ from ptf import mask from ptf import testutils from scapy.all import Ether, IP +from tests.common import config_reload from tests.common.dualtor.dual_tor_mock import * # noqa: F403 from tests.common.dualtor.dual_tor_utils import get_t1_ptf_ports from tests.common.dualtor.dual_tor_utils import rand_selected_interface # noqa: F401 @@ -240,25 +241,8 @@ def setup_uplink(rand_selected_dut, tbinfo, enable_feature_autorestart): yield mg_facts['minigraph_ptf_indices'][up_member] - # Startup the uplinks that were shutdown - for pc in portchannels: - if pc != up_portchannel: - cmd = "config interface startup {}".format(pc) - rand_selected_dut.shell(cmd) - # Restore the LAG - if len(pc_members) > 1: - cmds = [ - # Update min_links - "sonic-db-cli CONFIG_DB hset 'PORTCHANNEL|{}' 'min_links' 2".format(up_portchannel), - # Add back portchannel member - "config portchannel member add {} {}".format(up_portchannel, pc_members[1]), - # Unmask the service - "systemctl unmask teamd", - # Resart teamd - "systemctl restart teamd" - ] - rand_selected_dut.shell_cmds(cmds=cmds) - _wait_portchannel_up(rand_selected_dut, up_portchannel) + # Teardown: Restore the original config + config_reload(rand_selected_dut, config_source="running_golden_config", safe_reload=True) @pytest.fixture diff --git a/tests/generic_config_updater/conftest.py b/tests/generic_config_updater/conftest.py index 2481fad3879..534f5dc3518 100644 --- a/tests/generic_config_updater/conftest.py +++ b/tests/generic_config_updater/conftest.py @@ -108,6 +108,7 @@ def ignore_expected_loganalyzer_exceptions(request, duthosts, loganalyzer): ".*ERR kernel.*Reset adapter.*", # Valid test_portchannel_interface replace mtu ".*ERR swss[0-9]*#orchagent: :- getPortOperSpeed.*", # Valid test_portchannel_interface replace mtu ".*ERR systemd.*Failed to start Host core file uploader daemon.*", # Valid test_syslog + r".*ERR monit\[\d+\]: 'routeCheck' status failed \(255\) -- Failure results:.*", # sonic-swss/orchagent/crmorch.cpp ".*ERR swss[0-9]*#orchagent.*getResAvailableCounters.*", # test_monitor_config diff --git a/tests/generic_config_updater/test_dns_nameserver.py b/tests/generic_config_updater/test_dns_nameserver.py new file mode 100644 index 00000000000..8cd594e41fa --- /dev/null +++ b/tests/generic_config_updater/test_dns_nameserver.py @@ -0,0 +1,156 @@ +import logging +import pytest +import re + +from tests.common.helpers.assertions import pytest_assert +from tests.common.gu_utils import apply_patch, expect_op_success +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import format_json_patch_for_multiasic +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('any'), + pytest.mark.device_type('vs') +] + +DNS_SERVER_RE = "nameserver {}" + + +@pytest.fixture(autouse=True) +def setup_env(duthosts, rand_one_dut_hostname): # noqa: F811 + """ + Setup/teardown fixture for dns nameserver config + Args: + duthosts: list of DUTs. + rand_selected_dut: The fixture returns a randomly selected DuT. + """ + duthost = duthosts[rand_one_dut_hostname] + create_checkpoint(duthost) + + init_dns_nameservers = current_dns_nameservers(duthost) + + yield + + try: + logger.info("Rolled back to original checkpoint") + rollback_or_reload(duthost) + + cur_dns_nameservers = current_dns_nameservers(duthost) + pytest_assert(cur_dns_nameservers == init_dns_nameservers, + "DNS nameservers {} do not match {}.".format( + cur_dns_nameservers, init_dns_nameservers)) + finally: + delete_checkpoint(duthost) + + +def current_dns_nameservers(duthost): + """ Ger running dns nameservers + """ + config_facts = duthost.config_facts(host=duthost.hostname, + source="running")['ansible_facts'] + dns_nameservers = config_facts.get('DNS_NAMESERVER', {}) + return dns_nameservers + + +def dns_nameserver_test_setup(duthost): + """ Clean up dns nameservers before test + """ + dns_nameservers = current_dns_nameservers(duthost) + for dns_nameserver in dns_nameservers: + duthost.command("config dns nameserver del %s" % dns_nameserver) + + +def server_exist_in_conf(duthost, server_pattern): + """ Check if dns nameserver take effect in resolv.conf + """ + content = duthost.command("cat /etc/resolv.conf") + for line in content['stdout_lines']: + if re.search(server_pattern, line): + return True + return False + + +def add_dns_nameserver(duthost, dns_nameserver): + json_patch = [ + { + "op": "add", + "path": f"/DNS_NAMESERVER/{dns_nameserver}", + "value": {} + } + ] + json_patch = format_json_patch_for_multiasic(duthost=duthost, + json_data=json_patch, + is_host_specific=True) + + json_patch_bc = [ + { + "op": "remove", + "path": f"/DNS_NAMESERVER/{dns_nameserver}", + "value": {} + } + ] + json_patch_bc = format_json_patch_for_multiasic(duthost=duthost, + json_data=json_patch_bc, + is_host_specific=True) + + tmpfile = generate_tmpfile(duthost) + logger.info("tmpfile {}".format(tmpfile)) + + try: + output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) + if output['rc'] != 0: + logger.error(f"Failed to apply patch, rolling back: {output['stdout']}") + apply_patch(duthost, json_data=json_patch_bc, dest_file=tmpfile) + expect_op_success(duthost, output) + + pytest_assert( + server_exist_in_conf(duthost, DNS_SERVER_RE.format(dns_nameserver)), + "Failed to add {} in /etc/resolv.conf".format(dns_nameserver) + ) + + finally: + delete_tmpfile(duthost, tmpfile) + + +def remove_dns_nameserver(duthost, dns_nameserver): + json_patch = [ + { + "op": "remove", + "path": f"/DNS_NAMESERVER/{dns_nameserver}", + "value": {} + } + ] + json_patch = format_json_patch_for_multiasic(duthost=duthost, + json_data=json_patch, + is_host_specific=True) + + tmpfile = generate_tmpfile(duthost) + logger.info("tmpfile {}".format(tmpfile)) + + try: + output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) + expect_op_success(duthost, output) + + pytest_assert( + not server_exist_in_conf(duthost, DNS_SERVER_RE.format(dns_nameserver)), + "Failed to remove {} from /etc/resolv.conf".format(dns_nameserver) + ) + + finally: + delete_tmpfile(duthost, tmpfile) + + +def test_dns_server_add_and_remove(rand_selected_dut): + """ Test suite for dns nameserver + """ + + dns_nameserver_test_setup(rand_selected_dut) + add_dns_nameserver(rand_selected_dut, "10.6.3.1") + add_dns_nameserver(rand_selected_dut, "10.6.3.2") + remove_dns_nameserver(rand_selected_dut, "10.6.3.1") + + # Removing the last DNS server isn't supported under GCU, so let config + # rollback take care of it + # remove_dns_nameserver(rand_selected_dut, "10.6.3.2") diff --git a/tests/generic_config_updater/test_eth_interface.py b/tests/generic_config_updater/test_eth_interface.py index df4691820f4..3ef4d699e13 100644 --- a/tests/generic_config_updater/test_eth_interface.py +++ b/tests/generic_config_updater/test_eth_interface.py @@ -20,17 +20,17 @@ @pytest.fixture(autouse=True) -def ensure_dut_readiness(duthosts, rand_one_dut_hostname): +def ensure_dut_readiness(duthosts, rand_one_dut_front_end_hostname): """ Setup/teardown fixture for each ethernet test rollback to check if it goes back to starting config Args: duthosts: list of DUTs - rand_one_dut_hostname: The fixture returns a randomly selected DUT hostname + rand_one_dut_front_end_hostname: The fixture returns a randomly selected frontend DUT hostname """ - duthost = duthosts[rand_one_dut_hostname] + duthost = duthosts[rand_one_dut_front_end_hostname] create_checkpoint(duthost) yield @@ -112,6 +112,27 @@ def check_interface_status(duthost, field, interface='Ethernet0'): return status +def remove_port_from_portchannel(duthost, port, portchannel, namespace=None): + """ + Removes a port from its PortChannel membership + + Args: + duthost: DUT host object under test + port: Port name to remove + portchannel: PortChannel name + namespace: DUT asic namespace + """ + namespace_prefix = '' if namespace is None else '-n ' + namespace + cmd = 'config portchannel {} member del {} {}'.format(namespace_prefix, portchannel, port) + logger.info("Removing {} from {} in namespace {}".format( + port, portchannel, namespace or 'default')) + output = duthost.shell(cmd) + pytest_assert( + output['rc'] == 0, + "Failed to remove {} from {}: {}".format(port, portchannel, output.get('stderr', ''))) + return True + + def get_ethernet_port_not_in_portchannel(duthost, namespace=None): """ Returns the name of an ethernet port which is not a member of a port channel @@ -147,6 +168,62 @@ def get_ethernet_port_not_in_portchannel(duthost, namespace=None): return port_name +def get_test_port(duthost, namespace=None, remove_from_portchannel=True): + """ + Returns an available ethernet port for testing. + If no free ports exist and remove_from_portchannel=True, removes a port from a PortChannel. + The port will be restored by the ensure_dut_readiness fixture's rollback mechanism. + + Args: + duthost: DUT host object under test + namespace: DUT asic namespace + remove_from_portchannel: If True, remove a port from PortChannel if no free ports available + + Returns: + Port name string, or empty string if no suitable port found + """ + # First try to get a port not in a PortChannel + port = get_ethernet_port_not_in_portchannel(duthost, namespace=namespace) + if port: + logger.info("Found available port: {}".format(port)) + return port + + if not remove_from_portchannel: + logger.warning("No available ports and remove_from_portchannel=False") + return "" + + # If no free port, find one in a PortChannel and remove it + logger.info("No free ports available, attempting to remove a port from PortChannel") + config_facts = duthost.config_facts( + host=duthost.hostname, + source="running", + verbose=False, + namespace=namespace + )['ansible_facts'] + + if 'PORTCHANNEL_MEMBER' not in config_facts or 'PORT' not in config_facts: + logger.warning("No PortChannel members or ports found") + return "" + + port_channel_member_facts = config_facts['PORTCHANNEL_MEMBER'] + + # Find a suitable port to remove (prefer Ext role ports) + for portchannel in list(port_channel_member_facts.keys()): + for member in list(port_channel_member_facts[portchannel].keys()): + port_role = config_facts['PORT'].get(member, {}).get('role') + if port_role and port_role != 'Ext': + continue # Skip internal/fabric ports + + # Found a candidate - remove it from the PortChannel + logger.info("Removing {} from {} for testing (will be restored by rollback)".format( + member, portchannel)) + remove_port_from_portchannel(duthost, member, portchannel, namespace=namespace) + return member + + logger.warning("No suitable ports found even in PortChannels") + return "" + + def get_port_speeds_for_test(duthost, port): """ Get the speeds parameters for case test_update_speed, including 2 valid speeds and 1 invalid speed @@ -195,7 +272,8 @@ def test_remove_lanes(duthosts, rand_one_dut_front_end_hostname, duthost = duthosts[rand_one_dut_front_end_hostname] asic_namespace = None if enum_rand_one_frontend_asic_index is None else \ 'asic{}'.format(enum_rand_one_frontend_asic_index) - port = get_ethernet_port_not_in_portchannel(duthost, namespace=asic_namespace) + port = get_test_port(duthost, namespace=asic_namespace) + pytest_assert(port, "No available ethernet ports on this ASIC") json_patch = [ { "op": "remove", @@ -221,7 +299,8 @@ def test_replace_lanes(duthosts, rand_one_dut_front_end_hostname, ensure_dut_rea duthost = duthosts[rand_one_dut_front_end_hostname] asic_namespace = None if enum_rand_one_frontend_asic_index is None else \ 'asic{}'.format(enum_rand_one_frontend_asic_index) - port = get_ethernet_port_not_in_portchannel(duthost, namespace=asic_namespace) + port = get_test_port(duthost, namespace=asic_namespace) + pytest_assert(port, "No available ethernet ports on this ASIC") cur_lanes = check_interface_status(duthost, "Lanes", port) cur_lanes = cur_lanes.split(",") cur_lanes.sort() @@ -253,10 +332,14 @@ def test_replace_mtu(duthosts, rand_one_dut_front_end_hostname, ensure_dut_readi duthost = duthosts[rand_one_dut_front_end_hostname] asic_namespace = None if enum_rand_one_frontend_asic_index is None else \ 'asic{}'.format(enum_rand_one_frontend_asic_index) - # Can't directly change mtu of the port channel member - # So find a ethernet port that are not in a port channel - port = get_ethernet_port_not_in_portchannel(duthost, namespace=asic_namespace) - pytest_assert(port, "No available ethernet ports, all ports are in port channels.") + + # Get a test port - check without removing from PortChannel to avoid routing issues + port = get_test_port(duthost, namespace=asic_namespace, remove_from_portchannel=False) + + if not port: + # MTU changes on ports removed from PortChannel can cause routing convergence issues + # Skip this test to avoid teardown failures + pytest.skip("No free ports available. Skipping MTU test to avoid routing issues from PortChannel changes.") target_mtu = "1514" json_patch = [ { @@ -287,7 +370,8 @@ def test_toggle_pfc_asym(duthosts, rand_one_dut_front_end_hostname, ensure_dut_r duthost = duthosts[rand_one_dut_front_end_hostname] asic_namespace = None if enum_rand_one_frontend_asic_index is None else \ 'asic{}'.format(enum_rand_one_frontend_asic_index) - port = get_ethernet_port_not_in_portchannel(duthost, namespace=asic_namespace) + port = get_test_port(duthost, namespace=asic_namespace) + pytest_assert(port, "No available ethernet ports on this ASIC") json_patch = [ { "op": "replace", @@ -320,7 +404,8 @@ def test_replace_fec(duthosts, rand_one_dut_front_end_hostname, ensure_dut_readi 'asic{}'.format(enum_rand_one_frontend_asic_index) namespace_prefix = '' if asic_namespace is None else '-n ' + asic_namespace intf_init_status = duthost.get_interfaces_status() - port = get_ethernet_port_not_in_portchannel(duthost, namespace=asic_namespace) + port = get_test_port(duthost, namespace=asic_namespace) + pytest_assert(port, "No available ethernet ports on this ASIC") intf_init_fec_oper = get_fec_oper(duthost, port) json_patch = [ { @@ -363,7 +448,8 @@ def test_update_invalid_index(duthosts, rand_one_dut_front_end_hostname, ensure_ duthost = duthosts[rand_one_dut_front_end_hostname] asic_namespace = None if enum_rand_one_frontend_asic_index is None else \ 'asic{}'.format(enum_rand_one_frontend_asic_index) - port = get_ethernet_port_not_in_portchannel(duthost, namespace=asic_namespace) + port = get_test_port(duthost, namespace=asic_namespace) + pytest_assert(port, "No available ethernet ports on this ASIC") json_patch = [ { "op": "replace", @@ -433,7 +519,8 @@ def test_update_speed(duthosts, rand_one_dut_front_end_hostname, ensure_dut_read duthost = duthosts[rand_one_dut_front_end_hostname] asic_namespace = None if enum_rand_one_frontend_asic_index is None else \ 'asic{}'.format(enum_rand_one_frontend_asic_index) - port = get_ethernet_port_not_in_portchannel(duthost, namespace=asic_namespace) + port = get_test_port(duthost, namespace=asic_namespace) + pytest_assert(port, "No available ethernet ports on this ASIC") speed_params = get_port_speeds_for_test(duthost, port) for speed, is_valid in speed_params: json_patch = [ @@ -468,7 +555,8 @@ def test_update_description(duthosts, rand_one_dut_front_end_hostname, ensure_du duthost = duthosts[rand_one_dut_front_end_hostname] asic_namespace = None if enum_rand_one_frontend_asic_index is None else \ 'asic{}'.format(enum_rand_one_frontend_asic_index) - port = get_ethernet_port_not_in_portchannel(duthost, namespace=asic_namespace) + port = get_test_port(duthost, namespace=asic_namespace) + pytest_assert(port, "No available ethernet ports on this ASIC") json_patch = [ { "op": "replace", @@ -495,7 +583,8 @@ def test_eth_interface_admin_change(duthosts, rand_one_dut_front_end_hostname, a duthost = duthosts[rand_one_dut_front_end_hostname] asic_namespace = None if enum_rand_one_frontend_asic_index is None else \ 'asic{}'.format(enum_rand_one_frontend_asic_index) - port = get_ethernet_port_not_in_portchannel(duthost, namespace=asic_namespace) + port = get_test_port(duthost, namespace=asic_namespace) + pytest_assert(port, "No available ethernet ports on this ASIC") json_patch = [ { "op": "add", diff --git a/tests/generic_config_updater/test_ntp.py b/tests/generic_config_updater/test_ntp.py index 83baad34361..103cc0e0232 100644 --- a/tests/generic_config_updater/test_ntp.py +++ b/tests/generic_config_updater/test_ntp.py @@ -297,3 +297,53 @@ def test_ntp_server_tc1_suite(rand_selected_dut): ntp_server_tc1_xfail(rand_selected_dut) ntp_server_tc1_replace(rand_selected_dut, ntp_service) ntp_server_tc1_remove(rand_selected_dut, ntp_service) + + +def ntp_server_set_intf(duthost, ntp_service, src_intf): + """ Test to set NTP source interface + """ + json_patch = [ + { + "op": "add", + "path": "/NTP", + "value": { + "global": { + "src_intf": src_intf + } + } + } + ] + json_patch = format_json_patch_for_multiasic(duthost=duthost, json_data=json_patch, is_host_specific=True) + + tmpfile = generate_tmpfile(duthost) + logger.info("tmpfile {}".format(tmpfile)) + + ntp_daemon = get_ntp_daemon_in_use(duthost) + + try: + start_time = int(duthost.command("date +%s")['stdout'].strip()) + output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) + expect_op_success(duthost, output) + + pytest_assert( + ntp_service_restarted(duthost, ntp_service, start_time), + f"{ntp_service} is not restarted after change" + ) + + if ntp_daemon == NtpDaemon.CHRONY: + pytest_assert( + server_exist_in_conf(duthost, f"bindacqdevice {src_intf}"), + f"Failed to set source interface to {src_intf}" + ) + + finally: + delete_tmpfile(duthost, tmpfile) + + +def test_ntp_server_change_source_intf(rand_selected_dut): + """ Test changing the source interface via GCU + """ + ntp_service = get_ntp_service_name(rand_selected_dut) + + ntp_server_set_intf(rand_selected_dut, ntp_service, "Loopback0") + ntp_server_set_intf(rand_selected_dut, ntp_service, "eth0") diff --git a/tests/hash/generic_hash_helper.py b/tests/hash/generic_hash_helper.py index a8f6e8ee77b..7211e04df3d 100644 --- a/tests/hash/generic_hash_helper.py +++ b/tests/hash/generic_hash_helper.py @@ -4,6 +4,7 @@ import logging import pytest import ipaddress +import re from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until @@ -67,8 +68,8 @@ l2_ports = set() vlans_to_remove = [] interfaces_to_startup = [] -balancing_test_times = 480 -balancing_range = 0.25 +base_balancing_test_times = 480 +base_balancing_range = 0.3 balancing_range_in_port = 0.8 vxlan_ecmp_utils = VxLAN_Ecmp_Utils() vxlan_port_list = [13330, 4789] @@ -94,6 +95,33 @@ def skip_vs_setups(rand_selected_dut): pytest.skip("Generic hash test only runs on physical setups.") +@pytest.fixture(scope="function", autouse=True) +def skip_lag_tests_on_no_lag_topos(request, rand_selected_dut): + if "lag" in request.node.name and \ + "PORTCHANNEL" not in rand_selected_dut.get_running_config_facts(): + pytest.skip("The topology doesn't have portchannels, skip the lag test cases.") + + +@pytest.fixture(scope="function", autouse=True) +def skip_tests_on_isolated_topos(request, tbinfo): + if 'isolated' in tbinfo['topo']['name']: + uplink_count = re.search(r'u(\d+)', tbinfo['topo']['name']) + downlink_count = re.search(r'd(\d+)', tbinfo['topo']['name']) + if uplink_count: + uplink_count = int(uplink_count.group(1)) + else: + pytest.skip("Isolated topologies with no uplinks is not supported by the test.") + if downlink_count: + downlink_count = int(downlink_count.group(1)) + else: + pytest.skip("Isolated topologies with no downlinks is not supported by the test.") + if uplink_count > 32 and "IP_PROTOCOL" in request.node.name: + pytest.skip("IP_PROTOCOL hash field is not supported on topos with more than 32 uplinks.") + if downlink_count / uplink_count < 2 and "IN_PORT" in request.node.name: + pytest.skip("At least twice the number of downlinks compared" + " to uplinks is required for IN_PORT hash test.") + + @pytest.fixture(scope="module") def mg_facts(rand_selected_dut, tbinfo): """ Fixture to get the extended minigraph facts """ @@ -300,6 +328,27 @@ def check_default_route(duthost, expected_nexthops): return set(nexthops) == set(expected_nexthops) +def check_default_route_asic_db(duthost): + """ + Check the default route exists in the asic db. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + Returns: + True if the default route and nexthop id exist in the asic db. + """ + logger.info("Check if the default route exists in the asic db.") + route_entry = duthost.shell( + 'redis-cli -n 1 keys "*ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY:*0.0.0.0/0*"', + module_ignore_errors=True)["stdout"] + if not route_entry: + return False + route_entry_content = duthost.shell(f"redis-cli -n 1 hgetall '{route_entry}'")["stdout"] + if "SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID" in route_entry_content: + return True + else: + return False + + def get_ptf_port_indices(mg_facts, downlink_interfaces, uplink_interfaces): """ Get the ptf port indices for the interfaces under test. @@ -317,10 +366,10 @@ def get_ptf_port_indices(mg_facts, downlink_interfaces, uplink_interfaces): for interface in downlink_interfaces: sending_ports.append(mg_facts['minigraph_ptf_indices'][interface]) expected_port_groups = [] - for index, portchannel in enumerate(uplink_interfaces.keys()): + for index, interface in enumerate(uplink_interfaces.keys()): expected_port_groups.append([]) - for interface in uplink_interfaces[portchannel]: - expected_port_groups[index].append(mg_facts['minigraph_ptf_indices'][interface]) + for port in uplink_interfaces[interface]: + expected_port_groups[index].append(mg_facts['minigraph_ptf_indices'][port]) expected_port_groups[index].sort() return sending_ports, expected_port_groups @@ -441,6 +490,33 @@ def get_interfaces_for_test(duthost, mg_facts, hash_field): return uplink_interfaces, downlink_interfaces +def get_updated_balancing_test_times_and_range(uplink_interfaces): + """ + Update the balancing test times and range based on the number of uplink interfaces. + When the number of egress ports gets larger, the variations in some of the fields + may be not enough to get a perfect balancing result. + For example the IP_PROTOCOL field has only 256 values and VLAN_ID field has only 4096 values. + So we relax the threshold based on the number of egress ports. + Args: + uplink_interfaces: a dictionary of the uplink interfaces + Returns: + the new balancing test times and range + """ + uplink_physicalport_count = sum(len(members) for members in uplink_interfaces.values()) + balancing_test_times = base_balancing_test_times + balancing_range = base_balancing_range + if uplink_physicalport_count >= 32 and uplink_physicalport_count < 64: + balancing_test_times = int(base_balancing_test_times * 0.75) + balancing_range = round(balancing_range + 0.05, 2) + elif uplink_physicalport_count >= 64 and uplink_physicalport_count < 128: + balancing_test_times = int(base_balancing_test_times * 0.5) + balancing_range = round(balancing_range + 0.1, 2) + elif uplink_physicalport_count >= 128: + balancing_test_times = int(base_balancing_test_times * 0.25) + balancing_range = round(balancing_range + 0.15, 2) + return balancing_test_times, balancing_range + + def get_asic_type(request): metadata = get_testbed_metadata(request) if metadata is None: @@ -706,6 +782,11 @@ def generate_test_params(duthost, tbinfo, mg_facts, hash_field, ipver, inner_ipv dest_mac = get_vlan_intf_mac(duthost) else: dest_mac = duthost.facts['router_mac'] + # Update the balancing_test_times and balancing_range based on the number of uplink interfaces + # The test will run too long time if the balancing_test_times is big when + # the number of uplink interfaces is large + # Meanwhile relax the balancing_range to make sure the test is still stable. + balancing_test_times, balancing_range = get_updated_balancing_test_times_and_range(uplink_interfaces) ptf_params = {"router_mac": dest_mac, "sending_ports": ptf_sending_ports, "expected_port_groups": ptf_expected_port_groups, diff --git a/tests/hash/test_generic_hash.py b/tests/hash/test_generic_hash.py index bd82286266a..e4e679048ba 100644 --- a/tests/hash/test_generic_hash.py +++ b/tests/hash/test_generic_hash.py @@ -8,10 +8,12 @@ get_reboot_type_from_option, HASH_CAPABILITIES, check_global_hash_config, startup_interface, \ get_interfaces_for_test, get_ptf_port_indices, check_default_route, generate_test_params, flap_interfaces, \ PTF_QLEN, remove_ip_interface_and_config_vlan, config_custom_vxlan_port, shutdown_interface, \ - remove_add_portchannel_member, get_hash_algorithm_from_option, check_global_hash_algorithm, get_diff_hash_algorithm + remove_add_portchannel_member, get_hash_algorithm_from_option, check_global_hash_algorithm, \ + get_diff_hash_algorithm, check_default_route_asic_db from generic_hash_helper import restore_configuration, reload, global_hash_capabilities, restore_interfaces # noqa:F401 from generic_hash_helper import mg_facts, restore_init_hash_config, restore_vxlan_port, \ - get_supported_hash_algorithms, toggle_all_simulator_ports_to_upper_tor # noqa:F401 + get_supported_hash_algorithms, toggle_all_simulator_ports_to_upper_tor, skip_lag_tests_on_no_lag_topos # noqa:F401 +from generic_hash_helper import skip_tests_on_isolated_topos # noqa:F401 from tests.common.utilities import wait_until from tests.ptf_runner import ptf_runner from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa: F401 @@ -708,6 +710,8 @@ def test_reboot(rand_selected_dut, tbinfo, ptfhost, localhost, fine_params, mg_f with allure.step('Check the route is established'): pytest_assert(wait_until(60, 10, 0, check_default_route, rand_selected_dut, uplink_interfaces.keys()), "The default route is not established after the cold reboot.") + pytest_assert(wait_until(120, 10, 0, check_default_route_asic_db, rand_selected_dut), + 'The default route are not installed to the asic db.') with allure.step('Start the ptf test, send traffic and check the balancing'): ptf_runner( ptfhost, diff --git a/tests/ipfwd/test_nhop_group.py b/tests/ipfwd/test_nhop_group.py index 34177bbb32f..b3ef163b6ab 100644 --- a/tests/ipfwd/test_nhop_group.py +++ b/tests/ipfwd/test_nhop_group.py @@ -23,7 +23,7 @@ PTF_QUEUE_LEN = 100000 pytestmark = [ - pytest.mark.topology('t1', 't2', 'm1') + pytest.mark.topology('t1', 't2', 'm1', "lt2", "ft2") ] logger = logging.getLogger(__name__) diff --git a/tests/lldp/test_lldp_syncd.py b/tests/lldp/test_lldp_syncd.py index 64209639103..5212308eaf6 100644 --- a/tests/lldp/test_lldp_syncd.py +++ b/tests/lldp/test_lldp_syncd.py @@ -40,7 +40,8 @@ def db_instance(duthosts, enum_rand_one_per_hwsku_frontend_hostname): appl_db.append(SonicDbCli(asic, APPL_DB)) duthost.facts['switch_type'] == "voq" is_chassis = duthost.get_facts().get("modular_chassis") - if duthost.facts['switch_type'] == "voq" and not is_chassis: + # For single ASIC fixed system, APPL_DB is already added above. so skip here + if duthost.facts['switch_type'] == "voq" and (not is_chassis and len(duthost.asics) > 1): appl_db.append(SonicDbCli(duthost, APPL_DB)) # Cleanup code here return appl_db diff --git a/tests/macsec/test_docker_restart.py b/tests/macsec/test_docker_restart.py index d7535ab6108..0574dc61da1 100644 --- a/tests/macsec/test_docker_restart.py +++ b/tests/macsec/test_docker_restart.py @@ -3,6 +3,8 @@ from tests.common.utilities import wait_until from tests.common.macsec.macsec_helper import check_appl_db +from tests.common.helpers.dut_utils import restart_service_with_startlimit_guard + logger = logging.getLogger(__name__) @@ -17,6 +19,6 @@ def test_restart_macsec_docker(duthosts, ctrl_links, policy, cipher_suite, send_ duthost = duthosts[enum_rand_one_per_hwsku_macsec_frontend_hostname] logger.info(duthost.shell(cmd="docker ps", module_ignore_errors=True)['stdout']) - duthost.restart_service("macsec") + restart_service_with_startlimit_guard(duthost, "macsec", backoff_seconds=35, verify_timeout=180) logger.info(duthost.shell(cmd="docker ps", module_ignore_errors=True)['stdout']) assert wait_until(300, 6, 12, check_appl_db, duthost, ctrl_links, policy, cipher_suite, send_sci) diff --git a/tests/pc/test_po_update.py b/tests/pc/test_po_update.py index 63a4061c529..750da2e9370 100644 --- a/tests/pc/test_po_update.py +++ b/tests/pc/test_po_update.py @@ -73,7 +73,9 @@ def _wait_until_pc_members_removed(asichost, pc_names): pytest.fail("Portchannel members are not removed from {}".format(pc_names)) -def has_bgp_neighbors(duthost, portchannel): +def has_bgp_neighbors(duthost, portchannel, is_ipv6=False): + if is_ipv6: + return duthost.shell("show ipv6 int | grep {} | awk '{{print $4}}'".format(portchannel))['stdout'] != 'N/A' return duthost.shell("show ip int | grep {} | awk '{{print $4}}'".format(portchannel))['stdout'] != 'N/A' @@ -96,17 +98,37 @@ def test_po_update(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro portchannel = None portchannel_members = None + is_ipv6 = False for portchannel in port_channels_data: logging.info('Trying to get PortChannel: {} for test'.format(portchannel)) if int_facts['ansible_interface_facts'][portchannel].get('ipv4'): portchannel_members = port_channels_data[portchannel] break + elif int_facts['ansible_interface_facts'][portchannel].get('ipv6'): + # Check for non-link-local IPv6 address + for ipv6_info in int_facts['ansible_interface_facts'][portchannel]['ipv6']: + if not ipaddress.ip_address(ipv6_info['address']).is_link_local: + portchannel_members = port_channels_data[portchannel] + is_ipv6 = True + break + if portchannel_members: + break pytest_assert(portchannel and portchannel_members, 'Can not get PortChannel interface for test') tmp_portchannel = "PortChannel999" - # Initialize portchannel_ip and portchannel_members - portchannel_ip = int_facts['ansible_interface_facts'][portchannel]['ipv4']['address'] + # Initialize portchannel_ip and prefix_len based on IP version + if is_ipv6: + # Find non-link-local IPv6 address + for ipv6_info in int_facts['ansible_interface_facts'][portchannel]['ipv6']: + if not ipaddress.ip_address(ipv6_info['address']).is_link_local: + portchannel_ip = ipv6_info['address'] + prefix_len = str(ipv6_info['prefix']) + break + else: + portchannel_ip = int_facts['ansible_interface_facts'][portchannel]['ipv4']['address'] + prefix_len = "31" + bgp_state_key = 'ipv6_idle' if is_ipv6 else 'ipv4_idle' # Initialize flags remove_portchannel_members = False @@ -118,6 +140,7 @@ def test_po_update(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro logging.info("portchannel=%s" % portchannel) logging.info("portchannel_ip=%s" % portchannel_ip) logging.info("portchannel_members=%s" % portchannel_members) + logging.info("is_ipv6=%s" % is_ipv6) try: # Step 1: Remove portchannel members from portchannel @@ -126,15 +149,15 @@ def test_po_update(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro remove_portchannel_members = True # Step 2: Remove portchannel ip from portchannel - asichost.config_ip_intf(portchannel, portchannel_ip + "/31", "remove") + asichost.config_ip_intf(portchannel, portchannel_ip + "/" + prefix_len, "remove") remove_portchannel_ip = True time.sleep(30) int_facts = asichost.interface_facts()['ansible_facts'] pytest_assert(not int_facts['ansible_interface_facts'][portchannel]['link']) pytest_assert( - has_bgp_neighbors(duthost, portchannel) and - wait_until(120, 10, 0, asichost.check_bgp_statistic, 'ipv4_idle', 1) + has_bgp_neighbors(duthost, portchannel, is_ipv6) and + wait_until(120, 10, 0, asichost.check_bgp_statistic, bgp_state_key, 1) or not wait_until(10, 10, 0, pc_active, asichost, portchannel)) # Step 3: Create tmp portchannel @@ -147,22 +170,28 @@ def test_po_update(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro add_tmp_portchannel_members = True # Step 5: Add portchannel ip to tmp portchannel - asichost.config_ip_intf(tmp_portchannel, portchannel_ip + "/31", "add") + asichost.config_ip_intf(tmp_portchannel, portchannel_ip + "/" + prefix_len, "add") int_facts = asichost.interface_facts()['ansible_facts'] - pytest_assert(int_facts['ansible_interface_facts'][tmp_portchannel]['ipv4']['address'] == portchannel_ip) + if is_ipv6: + tmp_pc_ipv6_addrs = [ipv6_info['address'] + for ipv6_info in int_facts['ansible_interface_facts'][tmp_portchannel].get('ipv6', [])] + pytest_assert(portchannel_ip in tmp_pc_ipv6_addrs, + "IPv6 address {} not found on {}".format(portchannel_ip, tmp_portchannel)) + else: + pytest_assert(int_facts['ansible_interface_facts'][tmp_portchannel]['ipv4']['address'] == portchannel_ip) add_tmp_portchannel_ip = True time.sleep(30) int_facts = asichost.interface_facts()['ansible_facts'] pytest_assert(int_facts['ansible_interface_facts'][tmp_portchannel]['link']) pytest_assert( - has_bgp_neighbors(duthost, tmp_portchannel) and - wait_until(120, 10, 0, asichost.check_bgp_statistic, 'ipv4_idle', 0) + has_bgp_neighbors(duthost, tmp_portchannel, is_ipv6) and + wait_until(120, 10, 0, asichost.check_bgp_statistic, bgp_state_key, 0) or wait_until(10, 10, 0, pc_active, asichost, tmp_portchannel)) finally: # Recover all states if add_tmp_portchannel_ip: - asichost.config_ip_intf(tmp_portchannel, portchannel_ip + "/31", "remove") + asichost.config_ip_intf(tmp_portchannel, portchannel_ip + "/" + prefix_len, "remove") time.sleep(5) if add_tmp_portchannel_members: @@ -173,15 +202,15 @@ def test_po_update(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro if create_tmp_portchannel: asichost.config_portchannel(tmp_portchannel, "del") if remove_portchannel_ip: - asichost.config_ip_intf(portchannel, portchannel_ip + "/31", "add") + asichost.config_ip_intf(portchannel, portchannel_ip + "/" + prefix_len, "add") if remove_portchannel_members: for member in portchannel_members: asichost.config_portchannel_member(portchannel, member, "add") time.sleep(5) pytest_assert( - has_bgp_neighbors(duthost, portchannel) and - wait_until(120, 10, 0, asichost.check_bgp_statistic, 'ipv4_idle', 0) + has_bgp_neighbors(duthost, portchannel, is_ipv6) and + wait_until(120, 10, 0, asichost.check_bgp_statistic, bgp_state_key, 0) or wait_until(10, 10, 0, pc_active, asichost, portchannel)) diff --git a/tests/platform_tests/mellanox/test_check_sfp_eeprom.py b/tests/platform_tests/mellanox/test_check_sfp_eeprom.py index 0b5f9e665c2..204876c5fc1 100644 --- a/tests/platform_tests/mellanox/test_check_sfp_eeprom.py +++ b/tests/platform_tests/mellanox/test_check_sfp_eeprom.py @@ -6,7 +6,7 @@ from tests.common.platform.transceiver_utils import parse_sfp_eeprom_infos from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert -from tests.platform_tests.conftest import check_pmon_uptime_minutes +from tests.common.platform.processes_utils import check_pmon_uptime_minutes pytestmark = [ pytest.mark.asic('mellanox', 'nvidia-bluefield'), diff --git a/tests/platform_tests/test_xcvr_info_in_db.py b/tests/platform_tests/test_xcvr_info_in_db.py index a4ee31408e5..43805a03328 100644 --- a/tests/platform_tests/test_xcvr_info_in_db.py +++ b/tests/platform_tests/test_xcvr_info_in_db.py @@ -11,7 +11,7 @@ from tests.common.fixtures.conn_graph_facts import conn_graph_facts # noqa F401 from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert -from tests.platform_tests.conftest import check_pmon_uptime_minutes +from tests.common.platform.processes_utils import check_pmon_uptime_minutes pytestmark = [ pytest.mark.topology('any') diff --git a/tests/qos/files/dynamic_buffer_param.json b/tests/qos/files/dynamic_buffer_param.json index c57667d79d1..e7eb17fd25d 100644 --- a/tests/qos/files/dynamic_buffer_param.json +++ b/tests/qos/files/dynamic_buffer_param.json @@ -27,7 +27,8 @@ }, "extra_overhead": { "8": "95232", - "default": "58368" + "default": "58368", + "x86_64-nvidia_sn6600_simx-r0": "96256" }, "shared-headroom-pool": { "size": "1024000", @@ -38,10 +39,16 @@ "0": "[BUFFER_PROFILE_TABLE:ingress_lossy_pg_zero_profile]" }, "BUFFER_QUEUE_TABLE": { - "0-2": "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]", - "3-4": "[BUFFER_PROFILE_TABLE:egress_lossless_zero_profile]", - "5-6": "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]", - "7-15": "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]" + "default": + {"0-2": "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]", + "3-4": "[BUFFER_PROFILE_TABLE:egress_lossless_zero_profile]", + "5-6": "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]", + "7-15": "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]"}, + "x86_64-nvidia_sn6600_simx-r0": + {"0-2": "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]", + "3-4": "[BUFFER_PROFILE_TABLE:egress_lossless_zero_profile]", + "5-6": "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]", + "7-11": "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]"} }, "BUFFER_PORT_INGRESS_PROFILE_LIST_TABLE": ["[BUFFER_PROFILE_TABLE:ingress_lossless_zero_profile]"], "BUFFER_PORT_EGRESS_PROFILE_LIST_TABLE": ["[BUFFER_PROFILE_TABLE:egress_lossless_zero_profile]", "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]"] @@ -56,7 +63,12 @@ "x86_64-nvidia_sn4800_simx-r0": "400000", "x86_64-nvidia_sn5600-r0": "800000", "x86_64-nvidia_sn5640-r0": "800000", - "x86_64-nvidia_sn5600_simx-r0": "800000" + "x86_64-nvidia_sn5600_simx-r0": "800000", + "x86_64-nvidia_sn6600_simx-r0": "800000" + }, + "supported_speeds_to_test": { + "default": ["10000", "50000"], + "x86_64-nvidia_sn6600_simx-r0": ["100000"] } } } diff --git a/tests/qos/test_buffer.py b/tests/qos/test_buffer.py index c195421090e..e3348b60553 100644 --- a/tests/qos/test_buffer.py +++ b/tests/qos/test_buffer.py @@ -39,6 +39,7 @@ PORTS_WITH_8LANES = None ASIC_TYPE = None +PLATFORM_SUPPORTED_SPEEDS_TO_TEST = None TESTPARAM_HEADROOM_OVERRIDE = None TESTPARAM_LOSSLESS_PG = None TESTPARAM_SHARED_HEADROOM_POOL = None @@ -227,6 +228,7 @@ def load_test_parameters(duthost): global TESTPARAM_ADMIN_DOWN global ASIC_TYPE global MAX_SPEED_8LANE_PORT + global PLATFORM_SUPPORTED_SPEEDS_TO_TEST param_file_name = "qos/files/dynamic_buffer_param.json" with open(param_file_name) as file: @@ -234,6 +236,7 @@ def load_test_parameters(duthost): logging.info("Loaded test parameters {} from {}".format( params, param_file_name)) ASIC_TYPE = duthost.facts['asic_type'] + platform = duthost.facts['platform'] vendor_specific_param = params[ASIC_TYPE] DEFAULT_CABLE_LENGTH_LIST = vendor_specific_param['default_cable_length'] TESTPARAM_HEADROOM_OVERRIDE = vendor_specific_param['headroom-override'] @@ -241,8 +244,17 @@ def load_test_parameters(duthost): TESTPARAM_SHARED_HEADROOM_POOL = vendor_specific_param['shared-headroom-pool'] TESTPARAM_EXTRA_OVERHEAD = vendor_specific_param['extra_overhead'] TESTPARAM_ADMIN_DOWN = vendor_specific_param['admin-down'] - MAX_SPEED_8LANE_PORT = vendor_specific_param['max_speed_8lane_platform'].get( - duthost.facts['platform']) + MAX_SPEED_8LANE_PORT = vendor_specific_param['max_speed_8lane_platform'].get(platform) + if buffer_queue_table := TESTPARAM_ADMIN_DOWN['BUFFER_QUEUE_TABLE'].get(platform): # noqa: F841 + TESTPARAM_ADMIN_DOWN['BUFFER_QUEUE_TABLE'] = buffer_queue_table + else: + TESTPARAM_ADMIN_DOWN['BUFFER_QUEUE_TABLE'] = TESTPARAM_ADMIN_DOWN['BUFFER_QUEUE_TABLE'].get('default') + if platform_extra_overhead := TESTPARAM_EXTRA_OVERHEAD.get(platform): + TESTPARAM_EXTRA_OVERHEAD['default'] = platform_extra_overhead + if platform_supported_speeds_to_test := vendor_specific_param['supported_speeds_to_test'].get(platform): + PLATFORM_SUPPORTED_SPEEDS_TO_TEST = platform_supported_speeds_to_test + else: + PLATFORM_SUPPORTED_SPEEDS_TO_TEST = vendor_specific_param['supported_speeds_to_test'].get('default') # For ingress profile list, we need to check whether the ingress lossy profile exists ingress_lossy_pool = duthost.shell( @@ -850,7 +862,7 @@ def make_expected_profile_name(speed, cable_length, **kwargs): return expected_profile -@pytest.fixture(params=['50000', '10000']) +@pytest.fixture(params=['50000', '10000', '100000']) def speed_to_test(request): """Used to parametrized test cases for speeds @@ -860,6 +872,11 @@ def speed_to_test(request): Return: speed_to_test """ + global PLATFORM_SUPPORTED_SPEEDS_TO_TEST + if not PLATFORM_SUPPORTED_SPEEDS_TO_TEST: + pytest.skip("buffer is not dynamic - PLATFORM_SUPPORTED_SPEEDS_TO_TEST wasn't set") + if request.param not in PLATFORM_SUPPORTED_SPEEDS_TO_TEST: + pytest.skip(f"Skipping case for speed {request.param} because it is not tested by the platform") return request.param @@ -998,7 +1015,7 @@ def test_change_speed_cable(duthosts, rand_one_dut_hostname, conn_graph_facts, """ duthost = duthosts[rand_one_dut_hostname] supported_speeds = duthost.shell( - 'redis-cli -n 6 hget "PORT_TABLE|{}" supported_speeds'.format(port_to_test))['stdout'] + 'redis-cli -n 6 hget "PORT_TABLE|{}" supported_speeds'.format(port_to_test))['stdout'].split(',') if supported_speeds and speed_to_test not in supported_speeds: pytest.skip('Speed is not supported by the port, skip') original_speed = duthost.shell( @@ -2598,10 +2615,10 @@ def _update_headroom_exceed_Larger_size(param_name): # This should make it exceed the limit, so the profile should not applied to the APPL_DB time.sleep(20) size_in_appldb = duthost.shell( - f'redis-cli hget "BUFFER_PROFILE_TABLE:test-headroom" {param_name}')['stdout'] + f'redis-cli hget "BUFFER_PROFILE_TABLE: test-headroom" {param_name}')['stdout'] pytest_assert(size_in_appldb == maximum_profile[param_name], f'The profile with a large size was applied to APPL_DB, which can make headroom exceeding. ' - f'size_in_appldb:{size_in_appldb}, ' + f'size_in_appldb: {size_in_appldb}, ' f'maximum_profile_{param_name}: {maximum_profile[param_name]}') param_name = "size" if disable_shp else "xoff" diff --git a/tests/restapi/helper.py b/tests/restapi/helper.py index ac8e46c5260..e72ae568b1d 100644 --- a/tests/restapi/helper.py +++ b/tests/restapi/helper.py @@ -34,7 +34,7 @@ def apply_cert_config(duthost): time.sleep(5) # Restart RESTAPI server with the updated config - dut_command = "sudo systemctl restart restapi" + dut_command = "docker restart restapi" duthost.shell(dut_command) time.sleep(RESTAPI_SERVER_START_WAIT_TIME) @@ -50,6 +50,6 @@ def set_trusted_client_cert_subject_name(duthost, new_subject_name): time.sleep(5) # Restart RESTAPI server with the updated config - dut_command = "sudo systemctl restart restapi" + dut_command = "docker restart restapi" duthost.shell(dut_command) time.sleep(RESTAPI_SERVER_START_WAIT_TIME) diff --git a/tests/route/test_static_route.py b/tests/route/test_static_route.py index ba417bc750b..4f129ba195c 100644 --- a/tests/route/test_static_route.py +++ b/tests/route/test_static_route.py @@ -15,7 +15,7 @@ from tests.common.dualtor.mux_simulator_control import mux_server_url # noqa F811 from tests.common.dualtor.dual_tor_utils import show_muxcable_status from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F811 -from tests.common.utilities import wait_until, get_intf_by_sub_intf +from tests.common.utilities import wait_until, get_intf_by_sub_intf, is_ipv6_only_topology from tests.common.utilities import get_neighbor_ptf_port_list from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.assertions import pytest_require @@ -471,6 +471,11 @@ def get_nexthops(duthost, tbinfo, ipv6=False, count=1): vlan_intf = get_vlan_interface_info(duthost, tbinfo, vlan_if_name, "ipv6") else: vlan_intf = get_vlan_interface_info(duthost, tbinfo, vlan_if_name, "ipv4") + + # Check if the requested IP version is available (e.g., IPv6-only topology has no IPv4) + if not vlan_intf or 'prefixlen' not in vlan_intf: + return None, None, None, None + prefix_len = vlan_intf['prefixlen'] is_backend_topology = mg_facts.get(constants.IS_BACKEND_TOPOLOGY_KEY, False) @@ -511,9 +516,12 @@ def test_static_route(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfhos toggle_all_simulator_ports_to_rand_selected_tor_m, is_route_flow_counter_supported): # noqa F811 duthost = rand_selected_dut unselected_duthost = rand_unselected_dut - prefix_len, nexthop_addrs, nexthop_devs, nexthop_interfaces = get_nexthops(duthost, tbinfo) - run_static_route_test(duthost, unselected_duthost, ptfadapter, ptfhost, tbinfo, "1.1.1.0/24", - nexthop_addrs, prefix_len, nexthop_devs, nexthop_interfaces, is_route_flow_counter_supported) + ipv6 = is_ipv6_only_topology(tbinfo) + prefix = "2000:1::/64" if ipv6 else "1.1.1.0/24" + prefix_len, nexthop_addrs, nexthop_devs, nexthop_interfaces = get_nexthops(duthost, tbinfo, ipv6=ipv6) + run_static_route_test(duthost, unselected_duthost, ptfadapter, ptfhost, tbinfo, prefix, + nexthop_addrs, prefix_len, nexthop_devs, nexthop_interfaces, + is_route_flow_counter_supported, ipv6=ipv6) @pytest.mark.disable_loganalyzer @@ -522,10 +530,12 @@ def test_static_route_ecmp(rand_selected_dut, rand_unselected_dut, ptfadapter, p toggle_all_simulator_ports_to_rand_selected_tor_m, is_route_flow_counter_supported): # noqa F811 duthost = rand_selected_dut unselected_duthost = rand_unselected_dut - prefix_len, nexthop_addrs, nexthop_devs, nexthop_interfaces = get_nexthops(duthost, tbinfo, count=3) - run_static_route_test(duthost, unselected_duthost, ptfadapter, ptfhost, tbinfo, "2.2.2.0/24", + ipv6 = is_ipv6_only_topology(tbinfo) + prefix = "2000:2::/64" if ipv6 else "2.2.2.0/24" + prefix_len, nexthop_addrs, nexthop_devs, nexthop_interfaces = get_nexthops(duthost, tbinfo, ipv6=ipv6, count=3) + run_static_route_test(duthost, unselected_duthost, ptfadapter, ptfhost, tbinfo, prefix, nexthop_addrs, prefix_len, nexthop_devs, nexthop_interfaces, - is_route_flow_counter_supported, config_reload_test=True) + is_route_flow_counter_supported, ipv6=ipv6, config_reload_test=True) def test_static_route_ipv6(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfhost, tbinfo, diff --git a/tests/test_interfaces.py b/tests/test_interfaces.py index 90026cee946..81c0d663f27 100644 --- a/tests/test_interfaces.py +++ b/tests/test_interfaces.py @@ -1,7 +1,9 @@ from netaddr import IPAddress import pytest +from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert +from tests.common.platform.processes_utils import check_pmon_uptime_minutes pytestmark = [ pytest.mark.topology('any', 't1-multi-asic'), @@ -13,6 +15,10 @@ def test_interfaces(duthosts, enum_frontend_dut_hostname, tbinfo, enum_asic_inde """compare the interfaces between observed states and target state""" duthost = duthosts[enum_frontend_dut_hostname] + + pytest_assert(wait_until(360, 10, 0, check_pmon_uptime_minutes, duthost), + "Pmon docker is not ready for test") + asic_host = duthost.asic_instance(enum_asic_index) host_facts = asic_host.interface_facts()['ansible_facts']['ansible_interface_facts'] mg_facts = asic_host.get_extended_minigraph_facts(tbinfo)