Merge branch 'main' into pr_testbed_configs

DentOS_Framework/DentOsTestbed/src/dent_os_testbed/test/test_suite/functional/L1/test_l1_config.py

@@ -49,7 +49,7 @@ async def test_l1_settings_(testbed, l1_settings):
     await asyncio.sleep(10)
 
     try:
-        await tb_get_qualified_ports(dent_devices[0], tgen_dev.links_dict[device_host_name][1], speed, duplex, required_ports=1)
+        await tb_get_qualified_ports(dent_devices[0], [port], speed, duplex, required_ports=1)
     except ValueError as e:
         pytest.skip(str(e))
 
@@ -90,7 +90,7 @@ async def test_l1_settings_(testbed, l1_settings):
         await tgen_utils_update_l1_config(tgen_dev, tg_port, speed=speed, autoneg=True, duplex=duplex)
     except AssertionError as e:
         pytest.skip(f'TGen does not support requested mode\n{e}')
-    await asyncio.sleep(20)  # wait needed in case port was down before
+    await asyncio.sleep(40)  # wait needed in case port was down before
 
     out = await Ethtool.show(input_data=[{device_host_name: [{'devname': port}]}],  parse_output=True)
     assert out[0][device_host_name]['rc'] == 0, 'Failed getting port duplex, speed.'

DentOS_Framework/DentOsTestbed/src/dent_os_testbed/test/test_suite/functional/ipv6/test_ipv6_route.py

@@ -439,7 +439,7 @@ async def test_ipv6_nh_state(testbed):
             'type': 'raw',
             'protocol': 'ipv6',
             'ip_source': dev_groups[tg_ports[0]][0]['name'],
-            'rate': 10,  # %
+            'rate': 5,  # %
             'frame_rate_type': 'line_rate',
             'srcMac': '02:00:00:00:00:01',
             'dstMac': dut_mac[tg_to_swp[tg_ports[0]]],

DentOS_Framework/DentOsTestbed/src/dent_os_testbed/test/test_suite/functional/lacp/test_lacp_distribution_over_ecmp.py

@@ -56,6 +56,7 @@ async def test_lacp_ecmp_distribution_over_lag(testbed):
     tgen_lag_2 = ('LAG_1', [tg_ports[1]])
     tgen_lag_3 = ('LAG_2', tg_ports[2:])
     rate = 10000
+    tolerance = 0.10
 
     # 1. Enable IPv4 forwarding
     # 2. Create 3 bonds
@@ -142,7 +143,7 @@ async def test_lacp_ecmp_distribution_over_lag(testbed):
         if row['Port Name'] not in tgen_lag_3[1]:
             continue
         err_msg = f'Expected packets  {row["Valid Frames Rx."]}, actual packets: {tx_packets / 4}'
-        assert isclose(int(row['Valid Frames Rx.']), tx_packets / 4, rel_tol=0.05), err_msg
+        assert isclose(int(row['Valid Frames Rx.']), tx_packets / 4, rel_tol=tolerance), err_msg
     total_received = sum([int(row['Valid Frames Rx.']) for row in stats.Rows])
-    assert isclose(total_received, tx_packets, rel_tol=0.05),\
+    assert isclose(total_received, tx_packets, rel_tol=tolerance),\
         f'Expected packets  {total_received}, actual packets: {tx_packets}'

DentOS_Framework/DentOsTestbed/src/dent_os_testbed/test/test_suite/functional/lacp/test_lacp_loopback_detection.py

@@ -4,6 +4,7 @@
 
 from dent_os_testbed.lib.ip.ip_link import IpLink
 from dent_os_testbed.lib.mstpctl.mstpctl import Mstpctl
+from dent_os_testbed.lib.ethtool.ethtool import Ethtool
 
 from dent_os_testbed.utils.test_utils.tgen_utils import (tgen_utils_get_dent_devices_with_tgen,
                                                          tgen_utils_setup_streams, tgen_utils_start_traffic,
@@ -45,6 +46,7 @@ async def test_lacp_loopback_detection(testbed, version):
         bonds[f'bond_{idx+1}'] = port
     bridge = 'bridge_1'
     wait_time = 40 if version == 'stp' else 20
+    expected_rate = 0.14  # multiplied by port speed
 
     # 1. Create bridge entities and 6 bonds and set link up on them
     out = await IpLink.add(input_data=[{dent: [{'device': bond,
@@ -123,6 +125,9 @@ async def test_lacp_loopback_detection(testbed, version):
     await tgen_utils_stop_traffic(tgen_dev)
 
     # 7. Verify device remain stable afterwards (during storming).
+    out = await Ethtool.show(input_data=[{dent: [{'devname': dut_ixia_ports[2]}]}],  parse_output=True)
+    speed = int(out[0][dent]['parsed_output']['speed'][:-4])
+    expected_rate = int(expected_rate * speed)
     rc, out = await device.run_cmd("echo 'Hello World'")
     assert rc == 0, 'FAIL: DUT crashed due to storming'
     assert out.strip() == 'Hello World', f'Expected <Hello World> got {out}'
@@ -131,8 +136,8 @@ async def test_lacp_loopback_detection(testbed, version):
     # Verify there is storming
     for row in stats.Rows:
         if row['Port Name'] == tg_ports[0]:
-            err_msg = f'Expected 1400 got : {float(row["Rx. Rate (Mbps)"])}'
-            assert float(row['Rx. Rate (Mbps)']) > 1400, err_msg
+            err_msg = f'Expected {expected_rate} got : {float(row["Rx. Rate (Mbps)"])}'
+            assert float(row['Rx. Rate (Mbps)']) > expected_rate, err_msg
 
     # 8. Set bridge stp_state to 1.
     out = await IpLink.set(input_data=[{dent: [{

DentOS_Framework/DentOsTestbed/src/dent_os_testbed/test/test_suite/functional/lacp/test_lacp_routing_over_bridge.py

@@ -147,4 +147,4 @@ async def test_lacp_routing_over_bridge(testbed):
     stats = await tgen_utils_get_traffic_stats(tgen_dev, 'Traffic Item Statistics')
     for row in stats.Rows:
         err_msg = f"Expected 0.00 loss, actual {float(row['Loss %'])}"
-        assert tgen_utils_get_loss(row) == 0.000, err_msg
+        assert tgen_utils_get_loss(row) < 0.1, err_msg

DentOS_Framework/DentOsTestbed/src/dent_os_testbed/test/test_suite/functional/lacp/test_lacp_routing_over_lacp.py

@@ -1,4 +1,6 @@
 import asyncio
+from math import isclose
+
 import pytest
 
 from dent_os_testbed.lib.ip.ip_link import IpLink
@@ -139,5 +141,5 @@ async def test_lacp_routing_over_lacp(testbed):
     stats = await tgen_utils_get_traffic_stats(tgen_dev, 'Traffic Item Statistics')
     for row in stats.Rows:
         err_msg = f"Expected 0.00 loss, actual {float(row['Loss %'])}"
-        assert tgen_utils_get_loss(row) == 0.000, err_msg
+        assert isclose(tgen_utils_get_loss(row), 0.00, abs_tol=0.1), err_msg
     await tgen_utils_stop_traffic(tgen_dev)

DentOS_Framework/DentOsTestbed/src/dent_os_testbed/test/test_suite/functional/lacp/test_lacp_routing_over_vlan_device.py

@@ -1,4 +1,6 @@
 import asyncio
+from math import isclose
+
 import pytest
 
 from dent_os_testbed.lib.ip.ip_link import IpLink
@@ -12,9 +14,9 @@
     tgen_utils_dev_groups_from_config,
     tgen_utils_get_traffic_stats,
     tgen_utils_setup_streams,
-    tgen_utils_get_loss,
     tgen_utils_start_traffic,
     tgen_utils_get_swp_info,
+    tgen_utils_get_loss,
 )
 pytestmark = [pytest.mark.suite_functional_lacp,
               pytest.mark.asyncio,
@@ -167,4 +169,4 @@ async def test_lacp_routing_over_vlan_device(testbed):
     stats = await tgen_utils_get_traffic_stats(tgen_dev, 'Traffic Item Statistics')
     for row in stats.Rows:
         err_msg = f"Expected 0.00 loss, actual {float(row['Loss %'])}"
-        assert tgen_utils_get_loss(row) == 0.000, err_msg
+        assert isclose(tgen_utils_get_loss(row), 0.00, abs_tol=0.1), err_msg

DentOS_Framework/DentOsTestbed/src/dent_os_testbed/test/test_suite/functional/lacp/test_lacp_traffic_during_topology_convergence.py

@@ -4,6 +4,7 @@
 
 from dent_os_testbed.lib.ip.ip_link import IpLink
 from dent_os_testbed.lib.mstpctl.mstpctl import Mstpctl
+from dent_os_testbed.lib.ethtool.ethtool import Ethtool
 
 from dent_os_testbed.utils.test_utils.tgen_utils import (tgen_utils_get_dent_devices_with_tgen,
                                                          tgen_utils_setup_streams, tgen_utils_start_traffic,
@@ -12,7 +13,7 @@
                                                          tgen_utils_traffic_generator_connect, )
 pytestmark = [pytest.mark.suite_functional_lacp,
               pytest.mark.asyncio,
-              pytest.mark.usefixtures('cleanup_tgen', 'cleanup_bonds', 'cleanup_bridges')]
+              pytest.mark.usefixtures('cleanup_tgen', 'cleanup_bonds', 'cleanup_bridges', 'enable_mstpd')]
 
 
 @pytest.mark.parametrize('version', ['stp', 'rstp'])
@@ -60,6 +61,7 @@ async def test_lacp_traffic_during_topology_convergence(testbed, version):
     bridge_names = list(bridges.keys())
     tolerance = 0.15
     wait_time = 40 if 'version' == 'stp' else 20
+    expected_rate = 0.14  # multiplied by port speed
     # 1. Create 3 bridge entities and 6 bonds and set link up on them
     out = await IpLink.add(input_data=[{dent: [{'device': bond,
                                                 'type': 'bond',
@@ -147,13 +149,17 @@ async def test_lacp_traffic_during_topology_convergence(testbed, version):
     rc, out = await device.run_cmd("echo 'Hello World'")
     assert rc == 0, 'FAIL: DUT crashed due to storming'
     assert out.strip() == 'Hello World', f'Expected <Hello World> got {out}'
+    await asyncio.sleep(wait_time)
 
     # 7. Verify there is a storming
+    out = await Ethtool.show(input_data=[{dent: [{'devname': dut_tgen_ports[2]}]}],  parse_output=True)
+    speed = int(out[0][dent]['parsed_output']['speed'][:-4])
+    expected_rate = round(int(expected_rate * speed), 2)
     stats = await tgen_utils_get_traffic_stats(tgen_dev, 'Port Statistics')
     for row in stats.Rows:
         if row['Port Name'] in tg_ports[:2]:
-            err_msg = 'Expected 1400 got : {float(row["Rx. Rate (Mbps)"])}'
-            assert float(row['Rx. Rate (Mbps)']) > 1400, err_msg
+            err_msg = f'Expected {int(expected_rate * speed)} got : {float(row["Rx. Rate (Mbps)"])}'
+            assert float(row['Rx. Rate (Mbps)']) > expected_rate, err_msg
 
     # 8. Set bridge stp_state to 1.
     out = await IpLink.set(input_data=[{dent: [{
@@ -228,8 +234,8 @@ async def test_lacp_traffic_during_topology_convergence(testbed, version):
     stats = await tgen_utils_get_traffic_stats(tgen_dev, 'Port Statistics')
     for row in stats.Rows:
         if row['Port Name'] == tg_ports[1]:
-            err_msg = 'Expected 1400 got : {float(row["Rx. Rate (Mbps)"])}'
-            assert isclose(float(row['Rx. Rate (Mbps)']), 1400, rel_tol=tolerance), err_msg
+            err_msg = f'Expected 1400 got : {float(row["Rx. Rate (Mbps)"])}'
+            assert isclose(float(row['Rx. Rate (Mbps)']), expected_rate, rel_tol=tolerance), err_msg
 
     # 16. Stop the traffic. Verify there is no storming
     await tgen_utils_stop_traffic(tgen_dev)

DentOS_Framework/DentOsTestbed/src/dent_os_testbed/test/test_suite/functional/stp/test_stp_loopback.py

@@ -4,6 +4,7 @@
 
 from dent_os_testbed.lib.ip.ip_link import IpLink
 from dent_os_testbed.lib.mstpctl.mstpctl import Mstpctl
+from dent_os_testbed.lib.ethtool.ethtool import Ethtool
 
 from dent_os_testbed.utils.test_utils.tgen_utils import (tgen_utils_get_dent_devices_with_tgen,
                                                          tgen_utils_setup_streams, tgen_utils_start_traffic,
@@ -45,6 +46,7 @@ async def test_stp_loopback_detection(testbed, version):
     loopback_ports = {}
     for idx, port in enumerate(device.links_dict[dent][0] + device.links_dict[dent][1]):
         loopback_ports[f'loopback_{idx+1}'] = port
+    expected_rate = 0.14  # multiplied by port speed
 
     # 1. Create bridge entity
     out = await IpLink.add(input_data=[{dent: [{
@@ -114,12 +116,16 @@ async def test_stp_loopback_detection(testbed, version):
     assert rc == 0, 'FAIL: DUT crashed due to storming'
     assert out.strip() == 'Hello World', f'Expected <Hello World> got {out}'
 
+    out = await Ethtool.show(input_data=[{dent: [{'devname': dut_ixia_ports[2]}]}],  parse_output=True)
+    speed = int(out[0][dent]['parsed_output']['speed'][:-4])
+    expected_rate = round(int(expected_rate * speed), 2)
     stats = await tgen_utils_get_traffic_stats(tgen_dev, 'Port Statistics')
+
     # Verify there is storming
     for row in stats.Rows:
         if row['Port Name'] == tg_ports[0]:
-            err_msg = f'Expected 1400 got : {float(row["Rx. Rate (Mbps)"])}'
-            assert float(row['Rx. Rate (Mbps)']) > 1400, err_msg
+            err_msg = f'Expected {int(expected_rate * speed)} got : {float(row["Rx. Rate (Mbps)"])}'
+            assert float(row['Rx. Rate (Mbps)']) > expected_rate, err_msg
 
     # 8. Set bridge stp_state to 1.
     out = await IpLink.set(input_data=[{dent: [{

DentOS_Framework/DentOsTestbed/src/dent_os_testbed/test/test_suite/functional/stp/test_stp_with_traffic.py

@@ -332,6 +332,7 @@ async def test_stp_switching_off_on(testbed, version):
 
         # 7. Verify traffic is forwarded  and port 2 doesn't receive any traffic
         stats = await tgen_utils_get_traffic_stats(tgen_dev, 'Flow Statistics')
+        await asyncio.sleep(10)
         for row in stats.Rows:
             if row['Traffic Item'] == traffic and row['Rx Port'] == tg_ports[1]:
                 err_msg = f'Expected 0.0 got : {float(row["Rx Rate (Mbps)"])}'

tools/USAGE.MD

@@ -0,0 +1,30 @@
+# Utility Usage
+
+This folder holds tools for use with dentOS testing.
+
+## Test Utils
+
+`test_utils.py` is a test utility that interacts with pytest XMLs to aggregate, sort, and export data across XMLs.
+
+Available usage modes are:
+
+- `list` : show basic results from all test suites
+  - can use `-s` flag to sort listby passes, fails, etc.
+- `tree` : show results as a tree grouped by subtype
+- `html` : export test results as a markdown-compatible html table
+- `csv` : export test results as a csv
+- `run_fails` : generate commands to rerun all failed tests
+
+### Usage
+
+`./test_utils.py {MODE} -d {XML Directory}`
+
+### Examples
+
+Listing test suite results sorted by number passed
+
+`./test_utils.py list -d ../DentOS_Framework/DentOsTestbed/logs -s p`
+
+Exporting test data to a csv file
+
+`./test_utils.py csv -d ../DentOS_Framework/DentOsTestbed/logs > test.csv`