From 6a166f8ba173a9a0d0107bd37302c5d9ec1d1289 Mon Sep 17 00:00:00 2001 From: yaqiangz Date: Thu, 26 Dec 2024 05:39:18 +0000 Subject: [PATCH 1/4] [sanity_check][bgp] Add default route check in sanity --- tests/common/plugins/sanity_check/checks.py | 41 ++++++++++++++------ tests/common/plugins/sanity_check/recover.py | 17 +++++++- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/tests/common/plugins/sanity_check/checks.py b/tests/common/plugins/sanity_check/checks.py index 8a33df090e4..b79e9efae06 100644 --- a/tests/common/plugins/sanity_check/checks.py +++ b/tests/common/plugins/sanity_check/checks.py @@ -166,6 +166,12 @@ def _check_bgp_on_dut(*args, **kwargs): dut = kwargs['node'] results = kwargs['results'] + def _check_default_route(version, asichost): + # Return True if successfully get default route + res = asichost.shell("ip {} route show default".format("" if version == 4 else "-6"), + module_ignore_errors=True) + return not res["rc"] and len(res["stdout"].strip()) != 0 + def _check_bgp_status_helper(): asic_check_results = [] bgp_facts = dut.bgp_facts(asic_index='all') @@ -187,23 +193,32 @@ def _check_bgp_status_helper(): if a_asic_neighbors is not None and len(a_asic_neighbors) > 0: down_neighbors = [k for k, v in list(a_asic_neighbors.items()) if v['state'] != 'established'] + asic_key = 'bgp' if dut.facts['num_asic'] == 1 else 'bgp' + str(asic_index) if down_neighbors: - if dut.facts['num_asic'] == 1: - check_result['bgp'] = {'down_neighbors': down_neighbors} - else: - check_result['bgp' + str(asic_index)] = {'down_neighbors': down_neighbors} + check_result[asic_key] = {'down_neighbors': down_neighbors} a_asic_result = True else: a_asic_result = False - if dut.facts['num_asic'] == 1: - if 'bgp' in check_result: - check_result['bgp'].pop('down_neighbors', None) - else: - if 'bgp' + str(asic_index) in check_result: - check_result['bgp' + str(asic_index)].pop('down_neighbors', None) + if asic_key in check_result: + check_result[asic_key].pop('down_neighbors', None) else: a_asic_result = True + # Add default route check for default route missing issue in below scenario: + # 1) Loopbackv4 ip address replace, it would cause all bgp routes missing + # 2) Announce or withdraw routes in some test cases and doesn't recover it + asic_host = dut.asic_instance(asic_index) + if not _check_default_route(4, asic_host): + if asic_key not in check_result: + check_result[asic_key] = {} + check_result[asic_key]["no_v4_default_route"] = True + a_asic_result = True + if not _check_default_route(6, asic_host): + if asic_key not in check_result: + check_result[asic_key] = {} + check_result[asic_key]["no_v6_default_route"] = True + a_asic_result = True + asic_check_results.append(a_asic_result) if any(asic_check_results): @@ -243,8 +258,12 @@ def _check_bgp_status_helper(): if 'down_neighbors' in check_result[a_result]: logger.info('BGP neighbors down: %s on bgp instance %s on dut %s' % ( check_result[a_result]['down_neighbors'], a_result, dut.hostname)) + if "no_v4_default_route" in check_result[a_result]: + logger.info('Deafult v4 route for {} {} is missing'.format(dut.hostname, a_result)) + if "no_v6_default_route" in check_result[a_result]: + logger.info('Deafult v6 route for {} {} is missing'.format(dut.hostname, a_result)) else: - logger.info('No BGP neighbors are down on %s' % dut.hostname) + logger.info('No BGP neighbors are down or default route missing on %s' % dut.hostname) mgFacts = dut.get_extended_minigraph_facts(tbinfo) if dut.num_asics() == 1 and tbinfo['topo']['type'] != 't2' and \ diff --git a/tests/common/plugins/sanity_check/recover.py b/tests/common/plugins/sanity_check/recover.py index 33b19d48b40..14fd2ca81fb 100644 --- a/tests/common/plugins/sanity_check/recover.py +++ b/tests/common/plugins/sanity_check/recover.py @@ -147,6 +147,12 @@ def _recover_with_command(dut, cmd, wait_time): wait(wait_time, msg="Wait {} seconds for system to be stable.".format(wait_time)) +def re_announce_routes(localhost, topo_name, ptf_ip): + localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="withdraw", path="../ansible/") + localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="announce", path="../ansible/") + return None + + def adaptive_recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_results, wait_time): outstanding_action = None for result in check_results: @@ -155,7 +161,16 @@ def adaptive_recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_result action = _recover_interfaces(dut, fanouthosts, result, wait_time) elif result['check_item'] == 'services': action = _recover_services(dut, result) - elif result['check_item'] == 'bgp' or result['check_item'] == "neighbor_macsec_empty": + elif result['check_item'] == 'bgp': + # If there is only default route missing issue, only need to re-announce routes to recover + if ("no_v4_default_route" in result['bgp'] and len(result['bgp']) == 1 or + "no_v6_default_route" in result['bgp'] and len(result['bgp']) == 1 or + ("no_v4_default_route" in result['bgp'] and "no_v6_default_route" in result['bgp'] and + len(result['bgp']) == 2)): + action = re_announce_routes(localhost, tbinfo["topo"]["name"], tbinfo["ptf_ip"]) + else: + action = neighbor_vm_restore(dut, nbrhosts, tbinfo, result) + elif result['check_item'] == "neighbor_macsec_empty": action = neighbor_vm_restore(dut, nbrhosts, tbinfo, result) elif result['check_item'] in ['processes', 'mux_simulator']: action = 'config_reload' From 33da16832b51ce61f6e7ea8e641216f540f01cc6 Mon Sep 17 00:00:00 2001 From: yaqiangz Date: Thu, 26 Dec 2024 09:16:30 +0000 Subject: [PATCH 2/4] Skip multi-asic --- tests/common/plugins/sanity_check/checks.py | 29 +++++++++++---------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/common/plugins/sanity_check/checks.py b/tests/common/plugins/sanity_check/checks.py index b79e9efae06..600e6b35f94 100644 --- a/tests/common/plugins/sanity_check/checks.py +++ b/tests/common/plugins/sanity_check/checks.py @@ -166,10 +166,10 @@ def _check_bgp_on_dut(*args, **kwargs): dut = kwargs['node'] results = kwargs['results'] - def _check_default_route(version, asichost): + def _check_default_route(version, dut): # Return True if successfully get default route - res = asichost.shell("ip {} route show default".format("" if version == 4 else "-6"), - module_ignore_errors=True) + res = dut.shell("ip {} route show default".format("" if version == 4 else "-6"), + module_ignore_errors=True) return not res["rc"] and len(res["stdout"].strip()) != 0 def _check_bgp_status_helper(): @@ -207,17 +207,18 @@ def _check_bgp_status_helper(): # Add default route check for default route missing issue in below scenario: # 1) Loopbackv4 ip address replace, it would cause all bgp routes missing # 2) Announce or withdraw routes in some test cases and doesn't recover it - asic_host = dut.asic_instance(asic_index) - if not _check_default_route(4, asic_host): - if asic_key not in check_result: - check_result[asic_key] = {} - check_result[asic_key]["no_v4_default_route"] = True - a_asic_result = True - if not _check_default_route(6, asic_host): - if asic_key not in check_result: - check_result[asic_key] = {} - check_result[asic_key]["no_v6_default_route"] = True - a_asic_result = True + # Chassis and multi_asic is not supported for now + if not dut.is_multi_asic and not dut.get_facts().get("modular_chassis"): + if not _check_default_route(4, dut): + if asic_key not in check_result: + check_result[asic_key] = {} + check_result[asic_key]["no_v4_default_route"] = True + a_asic_result = True + if not _check_default_route(6, dut): + if asic_key not in check_result: + check_result[asic_key] = {} + check_result[asic_key]["no_v6_default_route"] = True + a_asic_result = True asic_check_results.append(a_asic_result) From b4c35a51eb6e3de36f9b9f08d53dc6bf1be56208 Mon Sep 17 00:00:00 2001 From: yaqiangz Date: Thu, 26 Dec 2024 09:37:16 +0000 Subject: [PATCH 3/4] Fix modular_chassis --- tests/common/plugins/sanity_check/checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/plugins/sanity_check/checks.py b/tests/common/plugins/sanity_check/checks.py index 600e6b35f94..f476a89e28d 100644 --- a/tests/common/plugins/sanity_check/checks.py +++ b/tests/common/plugins/sanity_check/checks.py @@ -208,7 +208,7 @@ def _check_bgp_status_helper(): # 1) Loopbackv4 ip address replace, it would cause all bgp routes missing # 2) Announce or withdraw routes in some test cases and doesn't recover it # Chassis and multi_asic is not supported for now - if not dut.is_multi_asic and not dut.get_facts().get("modular_chassis"): + if not dut.is_multi_asic and not (dut.get_facts().get("modular_chassis").lower() == "true"): if not _check_default_route(4, dut): if asic_key not in check_result: check_result[asic_key] = {} From d1023ccfdb42f6981d8757928f51e3fca538e48c Mon Sep 17 00:00:00 2001 From: yaqiangz Date: Thu, 26 Dec 2024 13:06:52 +0000 Subject: [PATCH 4/4] Update chassis --- tests/common/plugins/sanity_check/checks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/common/plugins/sanity_check/checks.py b/tests/common/plugins/sanity_check/checks.py index f476a89e28d..d18c64bd020 100644 --- a/tests/common/plugins/sanity_check/checks.py +++ b/tests/common/plugins/sanity_check/checks.py @@ -208,7 +208,8 @@ def _check_bgp_status_helper(): # 1) Loopbackv4 ip address replace, it would cause all bgp routes missing # 2) Announce or withdraw routes in some test cases and doesn't recover it # Chassis and multi_asic is not supported for now - if not dut.is_multi_asic and not (dut.get_facts().get("modular_chassis").lower() == "true"): + if not dut.is_multi_asic and not (dut.get_facts().get("modular_chassis") is True or + dut.get_facts().get("modular_chassis") == "True"): if not _check_default_route(4, dut): if asic_key not in check_result: check_result[asic_key] = {}