diff --git a/tests/common/plugins/sanity_check/checks.py b/tests/common/plugins/sanity_check/checks.py index 8a33df090e4..d18c64bd020 100644 --- a/tests/common/plugins/sanity_check/checks.py +++ b/tests/common/plugins/sanity_check/checks.py @@ -166,6 +166,12 @@ def _check_bgp_on_dut(*args, **kwargs): dut = kwargs['node'] results = kwargs['results'] + def _check_default_route(version, dut): + # Return True if successfully get default route + res = dut.shell("ip {} route show default".format("" if version == 4 else "-6"), + module_ignore_errors=True) + return not res["rc"] and len(res["stdout"].strip()) != 0 + def _check_bgp_status_helper(): asic_check_results = [] bgp_facts = dut.bgp_facts(asic_index='all') @@ -187,23 +193,34 @@ def _check_bgp_status_helper(): if a_asic_neighbors is not None and len(a_asic_neighbors) > 0: down_neighbors = [k for k, v in list(a_asic_neighbors.items()) if v['state'] != 'established'] + asic_key = 'bgp' if dut.facts['num_asic'] == 1 else 'bgp' + str(asic_index) if down_neighbors: - if dut.facts['num_asic'] == 1: - check_result['bgp'] = {'down_neighbors': down_neighbors} - else: - check_result['bgp' + str(asic_index)] = {'down_neighbors': down_neighbors} + check_result[asic_key] = {'down_neighbors': down_neighbors} a_asic_result = True else: a_asic_result = False - if dut.facts['num_asic'] == 1: - if 'bgp' in check_result: - check_result['bgp'].pop('down_neighbors', None) - else: - if 'bgp' + str(asic_index) in check_result: - check_result['bgp' + str(asic_index)].pop('down_neighbors', None) + if asic_key in check_result: + check_result[asic_key].pop('down_neighbors', None) else: a_asic_result = True + # Add default route check for default route missing issue in below scenario: + # 1) Loopbackv4 ip address replace, it would cause all bgp routes missing + # 2) Announce or withdraw routes in some test cases and doesn't recover it + # Chassis and multi_asic is not supported for now + if not dut.is_multi_asic and not (dut.get_facts().get("modular_chassis") is True or + dut.get_facts().get("modular_chassis") == "True"): + if not _check_default_route(4, dut): + if asic_key not in check_result: + check_result[asic_key] = {} + check_result[asic_key]["no_v4_default_route"] = True + a_asic_result = True + if not _check_default_route(6, dut): + if asic_key not in check_result: + check_result[asic_key] = {} + check_result[asic_key]["no_v6_default_route"] = True + a_asic_result = True + asic_check_results.append(a_asic_result) if any(asic_check_results): @@ -243,8 +260,12 @@ def _check_bgp_status_helper(): if 'down_neighbors' in check_result[a_result]: logger.info('BGP neighbors down: %s on bgp instance %s on dut %s' % ( check_result[a_result]['down_neighbors'], a_result, dut.hostname)) + if "no_v4_default_route" in check_result[a_result]: + logger.info('Deafult v4 route for {} {} is missing'.format(dut.hostname, a_result)) + if "no_v6_default_route" in check_result[a_result]: + logger.info('Deafult v6 route for {} {} is missing'.format(dut.hostname, a_result)) else: - logger.info('No BGP neighbors are down on %s' % dut.hostname) + logger.info('No BGP neighbors are down or default route missing on %s' % dut.hostname) mgFacts = dut.get_extended_minigraph_facts(tbinfo) if dut.num_asics() == 1 and tbinfo['topo']['type'] != 't2' and \ diff --git a/tests/common/plugins/sanity_check/recover.py b/tests/common/plugins/sanity_check/recover.py index 33b19d48b40..14fd2ca81fb 100644 --- a/tests/common/plugins/sanity_check/recover.py +++ b/tests/common/plugins/sanity_check/recover.py @@ -147,6 +147,12 @@ def _recover_with_command(dut, cmd, wait_time): wait(wait_time, msg="Wait {} seconds for system to be stable.".format(wait_time)) +def re_announce_routes(localhost, topo_name, ptf_ip): + localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="withdraw", path="../ansible/") + localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="announce", path="../ansible/") + return None + + def adaptive_recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_results, wait_time): outstanding_action = None for result in check_results: @@ -155,7 +161,16 @@ def adaptive_recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_result action = _recover_interfaces(dut, fanouthosts, result, wait_time) elif result['check_item'] == 'services': action = _recover_services(dut, result) - elif result['check_item'] == 'bgp' or result['check_item'] == "neighbor_macsec_empty": + elif result['check_item'] == 'bgp': + # If there is only default route missing issue, only need to re-announce routes to recover + if ("no_v4_default_route" in result['bgp'] and len(result['bgp']) == 1 or + "no_v6_default_route" in result['bgp'] and len(result['bgp']) == 1 or + ("no_v4_default_route" in result['bgp'] and "no_v6_default_route" in result['bgp'] and + len(result['bgp']) == 2)): + action = re_announce_routes(localhost, tbinfo["topo"]["name"], tbinfo["ptf_ip"]) + else: + action = neighbor_vm_restore(dut, nbrhosts, tbinfo, result) + elif result['check_item'] == "neighbor_macsec_empty": action = neighbor_vm_restore(dut, nbrhosts, tbinfo, result) elif result['check_item'] in ['processes', 'mux_simulator']: action = 'config_reload'