From cb055fb87b8f0d05ac27e9fbc2cc7d869b860a63 Mon Sep 17 00:00:00 2001 From: Tzu-Mainn Chen Date: Thu, 14 Nov 2024 14:25:19 -0500 Subject: [PATCH] Improve perfomance of node network functions Performance improved by retrieving a neutron port directory and avoiding later API calls; and by parallelizing port forwarding queries. --- esi/lib/networks.py | 62 +++++++++++++++++------------ esi/lib/nodes.py | 6 +-- esi/tests/unit/lib/test_networks.py | 14 +++---- esi/tests/unit/lib/test_nodes.py | 16 ++++---- 4 files changed, 53 insertions(+), 45 deletions(-) diff --git a/esi/lib/networks.py b/esi/lib/networks.py index 1b29fdb..ca62f3c 100644 --- a/esi/lib/networks.py +++ b/esi/lib/networks.py @@ -44,24 +44,37 @@ def network_and_port_list(connection, filter_network=None): f1 = executor.submit(get_ports, connection, filter_network) f2 = executor.submit(connection.network.networks) f3 = executor.submit(connection.network.ips) - network_ports = list(f1.result()) + network_ports_dict = {port.id: port for port in list(f1.result())} networks_dict = {network.id: network for network in list(f2.result())} floating_ips = list(f3.result()) - for floating_ip in floating_ips: - # no need to do this for floating IPs associated with a port, - # as port forwarding is irrelevant in such a case - if not floating_ip.port_id: - pfwds = list(connection.network.port_forwardings(floating_ip=floating_ip)) + fip_futures = [] + for floating_ip in floating_ips: + fip_futures.append(executor.submit( + get_floating_ip_and_port_forwarding, + connection, + floating_ip)) + for fip_future in fip_futures: + floating_ip, pfwds = fip_future.result() if len(pfwds): - floating_ip.port_id = pfwds[0].internal_port_id port_forwardings_dict[floating_ip.port_id] = pfwds - floating_ips_dict[floating_ip.port_id] = floating_ip + floating_ips_dict[floating_ip.port_id] = floating_ip - return network_ports, networks_dict, floating_ips_dict, port_forwardings_dict + return network_ports_dict, networks_dict, floating_ips_dict, port_forwardings_dict -def get_networks_from_port(connection, port, networks_dict={}, floating_ips_dict={}): +def get_floating_ip_and_port_forwarding(connection, floating_ip): + # no need to do this for floating IPs associated with a port, + # as port forwarding is irrelevant in such a case + pfwds = [] + if not floating_ip.port_id: + pfwds = list(connection.network.port_forwardings(floating_ip=floating_ip)) + if len(pfwds): + floating_ip.port_id = pfwds[0].internal_port_id + return floating_ip, pfwds + + +def get_networks_from_port(connection, port, networks_dict={}, network_ports_dict={}, floating_ips_dict={}): """Gets associated network objects from a port object :param connection: An OpenStack connection @@ -85,22 +98,19 @@ def get_networks_from_port(connection, port, networks_dict={}, floating_ips_dict parent_network = connection.network.get_network(network=port.network_id) if port.trunk_details: - with concurrent.futures.ThreadPoolExecutor() as executor: - subport_futures = [] - subport_infos = port.trunk_details['sub_ports'] - for subport_info in subport_infos: - subport_futures.append(executor.submit( - connection.network.get_port, - port=subport_info['port_id'] - )) - for subport_future in subport_futures: - subport = subport_future.result() - if subport.network_id in networks_dict: - trunk_network = networks_dict[subport.network_id] - else: - trunk_network = connection.network.get_network(subport.network_id) - trunk_ports.append(subport) - trunk_networks.append(trunk_network) + subport_infos = port.trunk_details['sub_ports'] + for subport_info in subport_infos: + if subport_info['port_id'] in network_ports_dict: + subport = network_ports_dict[subport_info['port_id']] + else: + subport = connection.network.get_port(subport_info['port_id']) + + if subport.network_id in networks_dict: + trunk_network = networks_dict[subport.network_id] + else: + trunk_network = connection.network.get_network(subport.network_id) + trunk_ports.append(subport) + trunk_networks.append(trunk_network) floating_network_id = getattr(floating_ips_dict.get(port.id), 'floating_network_id', None) diff --git a/esi/lib/nodes.py b/esi/lib/nodes.py index 3fd0eae..ac64046 100644 --- a/esi/lib/nodes.py +++ b/esi/lib/nodes.py @@ -89,7 +89,7 @@ def network_list(connection, filter_node=None, filter_network=None): filter_network = f3.result() f2 = executor.submit(networks.network_and_port_list, connection, filter_network) baremetal_nodes, baremetal_ports = f1.result() - network_ports, networks_dict, floating_ips_dict, port_forwardings_dict = f2.result() + network_ports_dict, networks_dict, floating_ips_dict, port_forwardings_dict = f2.result() data = [] for baremetal_node in baremetal_nodes: @@ -102,14 +102,14 @@ def network_list(connection, filter_node=None, filter_network=None): network_port_id = baremetal_port.internal_info.get('tenant_vif_port_id', None) if network_port_id: - network_port = next((np for np in network_ports - if np.id == network_port_id), None) + network_port = network_ports_dict.get(network_port_id) if network_port is not None and (not filter_network or filter_network.id == network_port.network_id): parent_network, trunk_networks, trunk_ports, floating_network \ = networks.get_networks_from_port(connection, network_port, networks_dict, + network_ports_dict, floating_ips_dict) network_info.append({ diff --git a/esi/tests/unit/lib/test_networks.py b/esi/tests/unit/lib/test_networks.py index 378fc18..7a0b9f4 100644 --- a/esi/tests/unit/lib/test_networks.py +++ b/esi/tests/unit/lib/test_networks.py @@ -168,10 +168,10 @@ def test_network_and_port_list(self): actual = networks.network_and_port_list(self.connection) expected = ( - [ - self.neutron_port1, - self.neutron_port2 - ], + { + 'neutron_port_uuid_1': self.neutron_port1, + 'neutron_port_uuid_2': self.neutron_port2 + }, { 'network_uuid_1': self.network1, 'network_uuid_2': self.network2, @@ -192,9 +192,9 @@ def test_network_and_port_list_network_filter(self): actual = networks.network_and_port_list(self.connection, self.network1) expected = ( - [ - self.neutron_port1, - ], + { + 'neutron_port_uuid_1': self.neutron_port1, + }, { 'network_uuid_1': self.network1, 'network_uuid_2': self.network2, diff --git a/esi/tests/unit/lib/test_nodes.py b/esi/tests/unit/lib/test_nodes.py index 17e3bc6..87dcaf4 100644 --- a/esi/tests/unit/lib/test_nodes.py +++ b/esi/tests/unit/lib/test_nodes.py @@ -253,14 +253,14 @@ def mock_baremetal_ports(details=False, node_id=None): def mock_network_ports(network_id=None): if network_id == 'network_uuid_1': - return [self.neutron_port1] + return [self.neutron_port1, self.sub_port1] elif network_id == 'network_uuid_2': - return [self.neutron_port2] + return [self.neutron_port2, self.sub_port2] if network_id == 'network_uuid_3': return [self.neutron_port3] elif network_id: return [] - return [self.neutron_port1, self.neutron_port2, self.neutron_port3] + return [self.neutron_port1, self.neutron_port2, self.neutron_port3, self.sub_port1, self.sub_port2] self.connection.network.ports.side_effect = mock_network_ports def mock_find_network(name_or_id=None, ignore_missing=True): @@ -398,8 +398,7 @@ def test_network_list(self): self.connection.network.ports.assert_called_once_with() self.connection.network.port_forwardings.assert_called_once_with(floating_ip=self.floating_ip_pfw) self.connection.network.get_network.assert_not_called() - self.connection.network.get_port.assert_any_call(port='sub_port_uuid_1') - self.connection.network.get_port.assert_any_call(port='sub_port_uuid_2') + self.connection.network.get_port.assert_not_called() def test_network_list_filter_node(self): filter_node = 'node1' @@ -448,8 +447,7 @@ def test_network_list_filter_node(self): self.connection.network.ports.assert_called_once_with() self.connection.network.port_forwardings.assert_called_once_with(floating_ip=self.floating_ip_pfw) self.connection.network.get_network.assert_not_called() - self.connection.network.get_port.assert_any_call(port='sub_port_uuid_1') - self.connection.network.get_port.assert_any_call(port='sub_port_uuid_2') + self.connection.network.get_port.assert_not_called() def test_network_list_filter_network(self): filter_node = None @@ -487,8 +485,8 @@ def test_network_list_filter_network(self): self.connection.network.ports.assert_called_once_with(network_id='network_uuid_3') self.connection.network.port_forwardings.assert_called_once_with(floating_ip=self.floating_ip_pfw) self.connection.network.get_network.assert_not_called() - self.connection.network.get_port.assert_any_call(port='sub_port_uuid_1') - self.connection.network.get_port.assert_any_call(port='sub_port_uuid_2') + self.connection.network.get_port.assert_any_call('sub_port_uuid_1') + self.connection.network.get_port.assert_any_call('sub_port_uuid_2') def test_network_list_filter_node_network(self): filter_node = '11111111-2222-3333-4444-bbbbbbbbbbbb'