diff --git a/config/chi_to_fabric_stitching.fab b/config/chi_to_fabric_stitching.fab index 1e75ad99..faa9ebde 100644 --- a/config/chi_to_fabric_stitching.fab +++ b/config/chi_to_fabric_stitching.fab @@ -37,7 +37,7 @@ resource: site: '{{ var.fabric_site }}' count: 1 image: default_rocky_8 - nic_model: NIC_ConnectX_5 + # nic_model: NIC_ConnectX_5. Defaults to NIC_Basic - chi_node: provider: '{{ chi.chi_provider }}' site: '{{ var.chi_site }}' diff --git a/fabfed/controller/controller.py b/fabfed/controller/controller.py index 17ebc09a..5f0e6e99 100644 --- a/fabfed/controller/controller.py +++ b/fabfed/controller/controller.py @@ -2,7 +2,7 @@ from typing import List, Union, Dict from fabfed.exceptions import ControllerException -from fabfed.model.state import BaseState, ProviderState +from fabfed.model.state import ResourceState, ProviderState from fabfed.util.config import WorkflowConfig from .helper import ControllerResourceListener, partition_layer3_config from fabfed.policy.policy_helper import ProviderPolicy @@ -45,11 +45,13 @@ def init(self, *, session: str, provider_factory: ProviderFactory, provider_stat name = provider_config.attributes.get('name') name = f"{session}-{name}" if name else session - provider_factory.init_provider(type=provider_config.type, - label=provider_config.label, - name=name, - attributes=provider_config.attributes, - logger=self.logger) + provider = provider_factory.init_provider(type=provider_config.type, + label=provider_config.label, + name=name, + attributes=provider_config.attributes, + logger=self.logger) + saved_state = next(filter(lambda s: s.label == provider.label, provider_states), None) + provider.set_saved_state(saved_state) self.resources = self.config.get_resource_configs() networks = [resource for resource in self.resources if resource.is_network] @@ -141,9 +143,11 @@ def init(self, *, session: str, provider_factory: ProviderFactory, provider_stat network.attributes[Constants.RES_PEER_LAYER3].append(other.attributes[Constants.RES_LAYER3]) for network in [net for net in networks if net.attributes.get(Constants.NETWORK_STITCH_WITH)]: - stitch_with = network.attributes.get(Constants.NETWORK_STITCH_WITH) + temp_label = network.attributes[Constants.LABEL] + stitch_with = network.attributes[Constants.NETWORK_STITCH_WITH] stitch_info = network.attributes.get(Constants.RES_STITCH_INFO) + assert stitch_info is not None, f"network {temp_label} does not have {stitch_info}" self.logger.info(f"{network}:stitch_with={stitch_with}: stitch_info={stitch_info}") self.resources = [r for r in self.resources if r.attributes[Constants.RES_COUNT] > 0] @@ -169,12 +173,10 @@ def plan(self, provider_states: List[ProviderState]): creation_details['in_config_file'] = True creation_details['provider_supports_modifiable'] = provider.supports_modify() - for resource in resources: if resource.label in resource_state_map: - states = resource_state_map.get(resource.label) - provider_state = states[0].attributes.get('provider_state') resource_dict = resource.attributes - resource_dict[Constants.RES_CREATION_DETAILS].update(provider_state.creation_details[resource.label]) + resource_dict[Constants.RES_CREATION_DETAILS].update( + provider.saved_state.creation_details[resource.label]) resource_dict[Constants.RES_CREATION_DETAILS]['total_count'] = resource_dict[Constants.RES_COUNT] planned_resources.append(resource) @@ -183,23 +185,22 @@ def plan(self, provider_states: List[ProviderState]): if state_label not in resources_labels: state = states[0] resource_dict = state.attributes.copy() - provider_state = resource_dict.pop('provider_state') + provider_state = resource_dict.pop(Constants.PROVIDER_STATE) provider = pf.get_provider(label=provider_state.label) - creation_details = provider_state.creation_details[state_label] + import copy + + creation_details: Dict = copy.deepcopy(provider_state.creation_details[state_label]) creation_details['in_config_file'] = False creation_details['provider_supports_modifiable'] = provider.supports_modify() from ..util.config_models import ProviderConfig var_name, _ = tuple(provider_state.label.split('@')) - provider_config = ProviderConfig(type=provider.type, name=var_name, attrs=provider.config) - var_name, _ = tuple(state.label.split('@')) - # resource_config = ResourceConfig(type=state.type, name=var_name, provider=provider_config, - # attrs=resource_dict) resource_dict = {} + var_name, _ = tuple(state.label.split('@')) if var_name != creation_details['name_prefix']: resource_dict['name'] = creation_details['name_prefix'] @@ -332,14 +333,14 @@ def apply(self, provider_states: List[ProviderState]): raise ControllerException(exceptions) @staticmethod - def _build_state_map(provider_states: List[ProviderState]) -> Dict[str, List[BaseState]]: + def _build_state_map(provider_states: List[ProviderState]) -> Dict[str, List[ResourceState]]: resource_state_map = dict() for provider_state in provider_states: temp_list = provider_state.states() for state in temp_list: - state.attributes['provider_state'] = provider_state + state.attributes[Constants.PROVIDER_STATE] = provider_state if state.label in resource_state_map: resource_state_map[state.label].append(state) @@ -355,8 +356,6 @@ def destroy(self, *, provider_states: List[ProviderState]): failed_resources = [] for provider_state in provider_states: - key = provider_state.label - for k in provider_state.failed: failed_resources.append(k) @@ -415,6 +414,10 @@ def destroy(self, *, provider_states: List[ProviderState]): skip_resources.update([external_state.label for external_state in external_states]) exceptions.append(e) + if not remaining_resources: + provider_states.clear() + return + provider_states_copy = provider_states.copy() provider_states.clear() @@ -422,109 +425,17 @@ def destroy(self, *, provider_states: List[ProviderState]): provider_state.node_states.clear() provider_state.network_states.clear() provider_state.service_states.clear() - provider = self.provider_factory.get_provider(label=provider_state.label) - provider_state.failed = provider.failed - - for remaining_resource in remaining_resources: - resource_state = resource_state_map[remaining_resource.label] - - if remaining_resource.provider.label == provider_state.label: - if remaining_resource.is_network: - provider_state.network_states.extend(resource_state) - elif remaining_resource.is_node: - provider_state.node_states.extend(resource_state) - elif remaining_resource.is_service: - provider_state.service_states.extend(resource_state) - - if provider_state.node_states or provider_state.network_states or provider_state.service_states: - provider_states.append(provider_state) - - if exceptions: - raise ControllerException(exceptions) - - def delete(self, *, provider_states: List[ProviderState]): - exceptions = [] - resource_state_map = Controller._build_state_map(provider_states) - provider_resource_map = dict() - - for provider_state in provider_states: - key = provider_state.label - provider_resource_map[key] = list() - - temp = self.resources - temp.reverse() - - for resource in temp: - if resource.label in resource_state_map: - key = resource.provider.label - external_dependencies = resource.attributes.get(Constants.EXTERNAL_DEPENDENCIES, []) - external_states = [resource_state_map[ed.resource.label] for ed in external_dependencies] - resource.attributes[Constants.EXTERNAL_DEPENDENCY_STATES] = sum(external_states, []) - provider_resource_map[key].append(resource) - resource.attributes[Constants.SAVED_STATES] = resource_state_map[resource.label] - - remaining_resources = list() - skip_resources = set() - - for resource in temp: - if resource.label not in resource_state_map: - continue - - provider_label = resource.provider.label - provider = self.provider_factory.get_provider(label=provider_label) - external_states = resource.attributes[Constants.EXTERNAL_DEPENDENCY_STATES] - - if resource.label in skip_resources: - self.logger.warning(f"Skipping deleting resource: {resource} with {provider_label}") - remaining_resources.append(resource) - skip_resources.update([external_state.label for external_state in external_states]) - continue - - fabric_work_around = False - # TODO: THIS FABRIC SPECIFIC AS WE DON"T SUPPORT SLICE MODIFY API JUST YET - for remaining_resource in remaining_resources: - if provider_label == remaining_resource.provider.label \ - and "@fabric" in remaining_resource.provider.label: - fabric_work_around = True - break - - if fabric_work_around: - self.logger.warning(f"Skipping deleting fabric resource: {resource} with {provider_label}") - remaining_resources.append(resource) - skip_resources.update([external_state.label for external_state in external_states]) - continue - - try: - provider.delete_resource(resource=resource.attributes) - except Exception as e: - self.logger.warning(f"Exception occurred while deleting resource: {e} using {provider_label}") - remaining_resources.append(resource) - skip_resources.update([external_state.label for external_state in external_states]) - exceptions.append(e) - provider_states_copy = provider_states.copy() - provider_states.clear() + if self.provider_factory.has_provider(label=provider_state.label): + provider = self.provider_factory.get_provider(label=provider_state.label) + provider_state.failed = provider.failed - for provider_state in provider_states_copy: - provider_state.node_states.clear() - provider_state.network_states.clear() - provider_state.service_states.clear() - provider = self.provider_factory.get_provider(label=provider_state.label) - provider_state.failed = provider.failed + for remaining_resource in [r for r in remaining_resources if r.provider.label == provider_state.label]: + resource_states = resource_state_map[remaining_resource.label] + provider_state.add_all(resource_states) - for remaining_resource in remaining_resources: - resource_state = resource_state_map[remaining_resource.label] - - if remaining_resource.provider.label == provider_state.label: - if remaining_resource.is_network: - provider_state.network_states.extend(resource_state) - elif remaining_resource.is_node: - provider_state.node_states.extend(resource_state) - elif remaining_resource.is_service: - provider_state.service_states.extend(resource_state) - - if provider_state.node_states or provider_state.network_states or provider_state.service_states: - provider_states.append(provider_state) + if provider_state.states(): + provider_states.append(provider_state) if exceptions: raise ControllerException(exceptions) diff --git a/fabfed/controller/helper.py b/fabfed/controller/helper.py index c547a1e1..ab3abe42 100644 --- a/fabfed/controller/helper.py +++ b/fabfed/controller/helper.py @@ -101,7 +101,9 @@ def find_node_clusters(*, resources): visited_networks.append(peer.label) nodes.extend(find_nodes_related_to_network(network=peer, resources=resources)) visited_nodes.extend([n.label for n in nodes]) - clusters.append(nodes) + + if nodes: + clusters.append(nodes) for net in networks: if net.label not in visited_networks: @@ -111,7 +113,9 @@ def find_node_clusters(*, resources): visited_networks.append(net.label) nodes = find_nodes_related_to_network(network=net, resources=resources) visited_nodes.extend([n.label for n in nodes]) - clusters.append(nodes) + + if nodes: + clusters.append(nodes) nodes = [r for r in resources if r.is_node] diff --git a/fabfed/controller/provider_factory.py b/fabfed/controller/provider_factory.py index cf62a83e..84485626 100644 --- a/fabfed/controller/provider_factory.py +++ b/fabfed/controller/provider_factory.py @@ -9,7 +9,7 @@ def __init__(self): self._providers: Dict[str, Provider] = {} # noinspection PyBroadException - def init_provider(self, *, type: str, label: str, name: str, attributes, logger): + def init_provider(self, *, type: str, label: str, name: str, attributes, logger) -> Provider: if type not in Constants.PROVIDER_CLASSES: from fabfed.exceptions import ProviderTypeNotSupported raise ProviderTypeNotSupported(type) @@ -30,11 +30,15 @@ def init_provider(self, *, type: str, label: str, name: str, attributes, logger) raise ProviderException(f"Exception encountered while initializing {label}: {e}") self._providers[label] = provider + return provider @property def providers(self) -> List[Provider]: return list(self._providers.values()) + def has_provider(self, *, label: str) -> bool: + return label in self._providers + def get_provider(self, *, label: str) -> Provider: return self._providers[label] diff --git a/fabfed/model/state.py b/fabfed/model/state.py index 0e4970f9..2d4cbb24 100644 --- a/fabfed/model/state.py +++ b/fabfed/model/state.py @@ -4,6 +4,7 @@ from fabfed.util.config_models import Config, ResourceConfig, BaseConfig, ProviderConfig, Dependency, DependencyInfo from fabfed.model import ResolvedDependency +from fabfed.util.constants import Constants class BaseState: @@ -13,19 +14,43 @@ def __init__(self, type: str, label: str, attributes: Dict): self.attributes = attributes -class NetworkState(BaseState): +class ResourceState(BaseState): + def __init__(self, type: str, label: str, attributes: Dict): + super().__init__(type, label, attributes) + self.type = type + self.label = label + self.attributes = attributes + + @property + def name(self) -> str: + return self.attributes['name'] + + @property + def is_node_state(self): + return self.type == Constants.RES_TYPE_NODE + + @property + def is_network_state(self): + return self.type == Constants.RES_TYPE_NETWORK + + @property + def is_service_state(self): + return self.type == Constants.RES_TYPE_SERVICE + + +class NetworkState(ResourceState): def __init__(self, *, label, attributes): - super().__init__("network", label, attributes) + super().__init__(Constants.RES_TYPE_NETWORK, label, attributes) -class NodeState(BaseState): +class NodeState(ResourceState): def __init__(self, *, label, attributes): - super().__init__("node", label, attributes) + super().__init__(Constants.RES_TYPE_NODE, label, attributes) -class ServiceState(BaseState): +class ServiceState(ResourceState): def __init__(self, *, label, attributes): - super().__init__("service", label, attributes) + super().__init__(Constants.RES_TYPE_SERVICE, label, attributes) class ProviderState(BaseState): @@ -41,8 +66,57 @@ def __init__(self, label, attributes, network_states: List[NetworkState], self.failed = failed self.creation_details = creation_details - def states(self) -> List[BaseState]: - return self.network_states + self.node_states + self.service_states + def add_if_not_found(self, resource_state: ResourceState): + from typing import cast + assert resource_state.type in [Constants.RES_TYPE_NODE, + Constants.RES_TYPE_NETWORK, + Constants.RES_TYPE_SERVICE] + + def exists(state, states: list): + temp = next(filter(lambda s: s.label == state.label and s.name == state.name, states), None) + + return temp is not None + + if exists(resource_state, self.states()): + return + + if resource_state.is_node_state: + self.node_states.append(cast(NodeState, resource_state)) + elif resource_state.is_network_state: + self.network_states.append(cast(NetworkState, resource_state)) + elif resource_state.is_service_state: + self.service_states.append(cast(ServiceState, resource_state)) + + def add(self, resource_state: ResourceState): + from typing import cast + assert resource_state.type in [Constants.RES_TYPE_NODE, + Constants.RES_TYPE_NETWORK, + Constants.RES_TYPE_SERVICE] + + def exists(state, states: list): + temp = next(filter(lambda s: s.label == state.label and s.name == state.name, states), None) + + return temp is not None + + assert not exists(resource_state, self.states()) + + if resource_state.is_node_state: + self.node_states.append(cast(NodeState, resource_state)) + elif resource_state.is_network_state: + self.network_states.append(cast(NetworkState, resource_state)) + elif resource_state.is_service_state: + self.service_states.append(cast(ServiceState, resource_state)) + + def add_all(self, resource_states: List[ResourceState]): + for resource_state in resource_states: + self.add(resource_state) + + def states(self) -> List[ResourceState]: + temp = [] + temp.extend(self.network_states) + temp.extend(self.node_states) + temp.extend(self.service_states) + return temp def number_of_created_resources(self): count = 0 @@ -68,6 +142,7 @@ def number_of_total_resources(self): return count + def provider_constructor(loader: yaml.SafeLoader, node: yaml.nodes.MappingNode) -> ProviderState: return ProviderState(**loader.construct_mapping(node)) diff --git a/fabfed/provider/api/provider.py b/fabfed/provider/api/provider.py index 1502b6e4..9a05b93f 100644 --- a/fabfed/provider/api/provider.py +++ b/fabfed/provider/api/provider.py @@ -1,6 +1,6 @@ import logging from abc import ABC, abstractmethod -from typing import List, Dict +from typing import List, Dict, Union from fabfed.model import Resource, Node, Network, Service from fabfed.model.state import ProviderState @@ -28,6 +28,17 @@ def __init__(self, *, type, label, name, logger: logging.Logger, config: dict): self.pending_internal = [] self.add_duration = self.create_duration = self.delete_duration = self.init_duration = 0 + self._saved_state: ProviderState = Union[ProviderState, None] + self._existing_map: Dict[str, List[str]] = {} + self._added_map: Union[Dict[str, List[str]], None] = None + + @property + def existing_map(self) -> Dict[str, List[str]]: + return self._existing_map + + @property + def added_map(self) -> Dict[str, List[str]]: + return self._added_map @property def resources(self) -> List: @@ -40,6 +51,13 @@ def resources(self) -> List: def nodes(self) -> List[Node]: return self._nodes + @property + def saved_state(self) -> Union[ProviderState, None]: + return self._saved_state + + def set_saved_state(self, state: Union[ProviderState, None]): + self._saved_state = state + @property def networks(self) -> List[Network]: return self._networks @@ -134,6 +152,62 @@ def init(self): def supports_modify(self): return False + def resource_name(self, resource: dict, idx: int = 0): + return f"{self.name}-{resource[Constants.RES_NAME_PREFIX]}-{idx}" + + def add_to_existing_map(self, resource: dict): + label = resource.get(Constants.LABEL) + self.existing_map[label] = [] + + if self.saved_state and resource.get(Constants.LABEL) in self.saved_state.creation_details: + provider_saved_creation_details = self.saved_state.creation_details[label] + + for n in range(0, provider_saved_creation_details['total_count']): + resource_name = self.resource_name(resource, n) + self.existing_map[label].append(resource_name) + + def _compute_added_map(self): + if self.added_map is None: + self._added_map = {} + + for net in self.networks: + if net.label not in self.added_map: + self.added_map[net.label] = [] + + self.added_map[net.label].append(net.name) + + for node in self.nodes: + if node.label not in self.added_map: + self.added_map[node.label] = [] + + self.added_map[node.label].append(node.name) + + for service in self.services: + if service.label not in self.added_map: + self.added_map[service.label] = [] + + self.added_map[service.label].append(service.name) + + @property + def modified(self): + if self.saved_state: + self._compute_added_map() + return self._existing_map and self.added_map != self._existing_map + + return False + + def retrieve_attribute_from_saved_state(self, resource, resource_name, attribute): + if self.saved_state: + for state in resource[Constants.SAVED_STATES]: + if state.attributes['name'] == resource_name: + ret = state.attributes.get(attribute) + + if isinstance(ret, list) and len(ret) == 1: + return ret[0] + + return ret + return None + def validate_resource(self, *, resource: dict): label = resource.get(Constants.LABEL) @@ -144,6 +218,7 @@ def validate_resource(self, *, resource: dict): self.creation_details[label]['failed_count'] = 0 self.creation_details[label]['created_count'] = 0 self.creation_details[label]['name_prefix'] = resource[Constants.RES_NAME_PREFIX] + self.add_to_existing_map(resource) import time @@ -165,7 +240,7 @@ def add_resource(self, *, resource: dict): count = resource.get(Constants.RES_COUNT, 1) label = resource.get(Constants.LABEL) assert count > 0 - assert label not in self._added + assert label not in self._added, f"{label} already in {self._added}" if len(resource[Constants.EXTERNAL_DEPENDENCIES]) > len(resource[Constants.RESOLVED_EXTERNAL_DEPENDENCIES]): self.logger.info(f"Adding {label} to pending using {self.label}") @@ -197,6 +272,8 @@ def add_resource(self, *, resource: dict): label = resource.get(Constants.LABEL) self.failed[label] = 'ADD' + failed_count = resource[Constants.RES_COUNT] - len(self.creation_details[label]['resources']) + self.creation_details[label]['failed_count'] = failed_count raise e finally: end = time.time() @@ -210,20 +287,36 @@ def create_resource(self, *, resource: dict): if self.no_longer_pending: self.logger.info(f"Checking internal dependencies using {self.label}") - for no_longer_pending_resource in self.no_longer_pending: - self.add_resource(resource=no_longer_pending_resource) + temp_no_longer_pending = self._no_longer_pending + self._no_longer_pending = [] - temp = self.pending_internal - self.pending_internal = [] + for no_longer_pending_resource in temp_no_longer_pending: + external_dependency_label = no_longer_pending_resource[Constants.LABEL] - for internal_dependency in temp: - internal_dependency_label = internal_dependency[Constants.LABEL] - self.logger.info(f"Adding internal_dependency {internal_dependency_label}") - self.add_resource(resource=internal_dependency) + try: + self.logger.info(f"Adding no longer pending external_dependency {external_dependency_label}") + self.add_resource(resource=no_longer_pending_resource) + added = True + except Exception as e: + self.logger.warning(f"Adding no longer pending externally {external_dependency_label} failed: {e}") + added = False + self.no_longer_pending.append(no_longer_pending_resource) - self._no_longer_pending = [] + if added: + temp = self.pending_internal + self.pending_internal = [] + + for internal_dependency in temp: + internal_dependency_label = internal_dependency[Constants.LABEL] + self.logger.info(f"Adding internal_dependency {internal_dependency_label}") + + try: + self.add_resource(resource=internal_dependency) + except Exception as e2: + self.logger.warning( + f"Adding no longer pending internally {internal_dependency_label} failed using {e2}") - self.logger.info(f"Creating {label} using {self.label}") + self.logger.info(f"Creating {label} using {self.label}: {self._added}") if label in self._added: try: @@ -272,12 +365,6 @@ def cleanup_attrs(attrs): service_states = [ServiceState(label=s.label, attributes=cleanup_attrs(vars(s))) for s in services] pending = [res['label'] for res in self.pending] pending_internal = [res['label'] for res in self.pending_internal] - # # nodes = [n for n in self.nodes if n.labelin self.failed] - # for n in nodes: - # self.failed[n.label] = {"phase": "xxx" , 'resource': cleanup_attrs(vars(n))} - # for n in networks: - # self.failed[n.label] = {"phase": "yyy" , 'resource': cleanup_attrs(vars(n))} - return ProviderState(self.label, dict(name=self.name), net_states, node_states, service_states, pending, pending_internal, self.failed, self.creation_details) diff --git a/fabfed/provider/chi/chi_constants.py b/fabfed/provider/chi/chi_constants.py index 0276e6f0..1d48b837 100644 --- a/fabfed/provider/chi/chi_constants.py +++ b/fabfed/provider/chi/chi_constants.py @@ -13,6 +13,14 @@ CHI_SLICE_PRIVATE_KEY_LOCATION = "slice-private-key-location" CHI_SLICE_PUBLIC_KEY_LOCATION = "slice-public-key-location" +CHI_CONF_ATTRS = [CHI_USER, + CHI_PASSWORD, + CHI_KEY_PAIR, + CHI_PROJECT_NAME, + CHI_PROJECT_ID, + CHI_SLICE_PRIVATE_KEY_LOCATION, + CHI_SLICE_PUBLIC_KEY_LOCATION] + DEFAULT_NETWORK = "sharednet1" DEFAULT_NETWORKS = [DEFAULT_NETWORK, "sharedwan1", "containernet1"] DEFAULT_FLAVOR = "m1.medium" diff --git a/fabfed/provider/chi/chi_network.py b/fabfed/provider/chi/chi_network.py index 5e8be7e0..40fcbf81 100644 --- a/fabfed/provider/chi/chi_network.py +++ b/fabfed/provider/chi/chi_network.py @@ -13,9 +13,13 @@ from .chi_constants import INCLUDE_ROUTER +from fabfed.util.utils import get_logger + +logger: logging.Logger = get_logger() + + class ChiNetwork(Network): - def __init__(self, *, label, name: str, site: str, project_name: str, - logger: logging.Logger, layer3: Config, stitch_provider: str): + def __init__(self, *, label, name: str, site: str, project_name: str, layer3: Config, stitch_provider: str): super().__init__(label=label, name=name, site=site) self.project_name = project_name self.subnet = layer3.attributes.get(Constants.RES_SUBNET) diff --git a/fabfed/provider/chi/chi_node.py b/fabfed/provider/chi/chi_node.py index 505eaac2..14818616 100644 --- a/fabfed/provider/chi/chi_node.py +++ b/fabfed/provider/chi/chi_node.py @@ -36,10 +36,14 @@ from fabfed.util.constants import Constants from .chi_constants import INCLUDE_ROUTER +from fabfed.util.utils import get_logger + +logger: logging.Logger = get_logger() + class ChiNode(Node): def __init__(self, *, label, name: str, image: str, site: str, flavor: str, project_name: str, - logger: logging.Logger, key_pair: str, network: str): + key_pair: str, network: str): super().__init__(label=label, name=name, image=image, site=site, flavor=flavor) self.project_name = project_name self.key_pair = key_pair @@ -50,7 +54,6 @@ def __init__(self, *, label, name: str, image: str, site: str, flavor: str, proj self.username = "cc" self.user = self.username self.state = None - # self.name = f'{prefix}-{self.name}' self.lease_name = f'{self.name}-lease' self.addresses = [] self.reservations = [] diff --git a/fabfed/provider/chi/chi_provider.py b/fabfed/provider/chi/chi_provider.py index 9edfd113..681a5d0e 100644 --- a/fabfed/provider/chi/chi_provider.py +++ b/fabfed/provider/chi/chi_provider.py @@ -1,5 +1,7 @@ import logging import os +from fabfed.exceptions import ResourceTypeNotSupported, ProviderException +from typing import List from fabfed.provider.api.provider import Provider from fabfed.util.constants import Constants @@ -12,45 +14,54 @@ class ChiProvider(Provider): - def __init__(self, *, type, label, name, config: dict): + def __init__(self, *, type, label, name, config: dict[str, str]): super().__init__(type=type, label=label, name=name, logger=logger, config=config) self.helper = None def setup_environment(self): - pass + site = "CHI@UC" + config = self.config - def supports_modify(self): - return True + for attr in CHI_CONF_ATTRS: + if config.get(attr) is None: + raise ProviderException(f"{self.name}: Expecting a value for {attr}") - def _setup_environment(self, *, site: str): - """ - Setup the environment variables for Chameleon - Should be invoked before any of the chi packages are imported otherwise, none of the CHI APIs work and - fail with BAD REQUEST error - @param site: site name - """ - config = self.config - credential_file = config.get(Constants.CREDENTIAL_FILE, None) + if not isinstance(config[CHI_PROJECT_ID], dict): + raise ProviderException(f"{self.name}: Expecting a dictionary for {CHI_PROJECT_ID}") + + temp = {} - if credential_file: - from fabfed.util import utils + for k, v in config[CHI_PROJECT_ID].items(): + temp[k.lower()] = config[CHI_PROJECT_ID][k] - profile = config.get(Constants.PROFILE) - config = utils.load_yaml_from_file(credential_file) + if "uc" not in temp or "tacc" not in temp: + raise ProviderException(f"{self.name}: Expecting a dictionary for {CHI_PROJECT_ID} for uc and tacc") - if profile not in config: - from fabfed.exceptions import ProviderException + config[CHI_PROJECT_ID] = temp - raise ProviderException( - f"credential file {credential_file} does not have a section for keyword {profile}") + from fabfed.util.utils import can_read, is_private_key, absolute_path - self.config = config = config[profile] + pkey = config[CHI_SLICE_PRIVATE_KEY_LOCATION] + pkey = absolute_path(pkey) + + if not can_read(pkey) or not is_private_key(pkey): + raise ProviderException(f"{self.name}: unable to read/parse ssh key in {pkey}") + + self.config[CHI_SLICE_PRIVATE_KEY_LOCATION] = pkey + + pub_key = self.config[CHI_SLICE_PUBLIC_KEY_LOCATION] + pubkey = absolute_path(pub_key) + + if not can_read(pubkey): + raise ProviderException(f"{self.name}: unable to read/parse ssh key in {pubkey}") + + self.config[CHI_SLICE_PUBLIC_KEY_LOCATION] = pub_key site_id = self.__get_site_identifier(site=site) os.environ['OS_AUTH_URL'] = config.get(CHI_AUTH_URL, DEFAULT_AUTH_URLS)[site_id] os.environ['OS_IDENTITY_API_VERSION'] = "3" os.environ['OS_INTERFACE'] = "public" - os.environ['OS_PROJECT_ID'] = config.get(CHI_PROJECT_ID, DEFAULT_PROJECT_IDS)[site_id] + os.environ['OS_PROJECT_ID'] = config.get(CHI_PROJECT_ID)[site_id] os.environ['OS_USERNAME'] = config.get(CHI_USER) os.environ['OS_PROTOCOL'] = "openid" os.environ['OS_AUTH_TYPE'] = "v3oidcpassword" @@ -64,6 +75,32 @@ def _setup_environment(self, *, site: str): os.environ['OS_SLICE_PRIVATE_KEY_FILE'] = config.get(CHI_SLICE_PRIVATE_KEY_LOCATION) os.environ['OS_SLICE_PUBLIC_KEY_FILE'] = config.get(CHI_SLICE_PUBLIC_KEY_LOCATION) + def do_validate_resource(self, *, resource: dict): + label = resource[Constants.LABEL] + rtype = resource.get(Constants.RES_TYPE) + + if rtype not in [Constants.RES_TYPE_NETWORK, Constants.RES_TYPE_NODE]: + raise ResourceTypeNotSupported(f"resource {rtype} not supported") + + site = resource.get(Constants.RES_SITE) + + if site is None and Constants.CONFIG in resource: + site = resource[Constants.CONFIG].get(Constants.RES_SITE) + + if site is None: + raise ProviderException(f"{self.label} expecting a site in {rtype} resource {label}") + + resource[Constants.RES_SITE] = site + + def supports_modify(self): + return True + + def _setup_environment(self, *, site: str): + site_id = self.__get_site_identifier(site=site) + os.environ['OS_AUTH_URL'] = self.config.get(CHI_AUTH_URL, DEFAULT_AUTH_URLS)[site_id] + os.environ['OS_PROJECT_ID'] = self.config.get(CHI_PROJECT_ID)[site_id] + os.environ['OS_CLIENT_ID'] = self.config.get(CHI_CLIENT_ID, DEFAULT_CLIENT_IDS)[site_id] + @staticmethod def __get_site_identifier(*, site: str): if site == "CHI@UC": @@ -80,22 +117,36 @@ def __get_site_identifier(*, site: str): return "uc" def do_add_resource(self, *, resource: dict): - site = resource.get(Constants.RES_SITE) + label = resource[Constants.LABEL] + rtype = resource[Constants.RES_TYPE] + site = resource[Constants.RES_SITE] + creation_details = resource[Constants.RES_CREATION_DETAILS] + + if not creation_details['in_config_file']: + return + self._setup_environment(site=site) key_pair = self.config[CHI_KEY_PAIR] project_name = self.config[CHI_PROJECT_NAME] - label = resource.get(Constants.LABEL) - rtype = resource.get(Constants.RES_TYPE) - if rtype == Constants.RES_TYPE_NETWORK.lower(): + if rtype == Constants.RES_TYPE_NETWORK: layer3 = resource.get(Constants.RES_LAYER3) - stitch_info = resource.get(Constants.RES_STITCH_INFO) - assert stitch_info and stitch_info.consumer, f"resource {label} missing stitch provider" - net_name = f'{self.name}-{resource.get(Constants.RES_NAME_PREFIX)}' + from typing import Union + from fabfed.policy.policy_helper import StitchInfo + stitch_info: Union[str, StitchInfo] = resource.get(Constants.RES_STITCH_INFO) + assert stitch_info, f"resource {label} missing stitch info" + + if isinstance(stitch_info, str): + stitch_provider = stitch_info + else: + stitch_provider = stitch_info.consumer + + assert stitch_provider, f"resource {label} missing stitch provider" + net_name = self.resource_name(resource) from fabfed.provider.chi.chi_network import ChiNetwork - net = ChiNetwork(label=label, name=net_name, site=site, logger=self.logger, - layer3=layer3, stitch_provider=stitch_info.consumer, + net = ChiNetwork(label=label, name=net_name, site=site, + layer3=layer3, stitch_provider=stitch_provider, project_name=project_name) self._networks.append(net) @@ -109,20 +160,19 @@ def do_add_resource(self, *, resource: dict): net = util.get_single_value_for_dependency(resource=resource, attribute='network') network = net.name - node_count = resource.get(Constants.RES_COUNT, 1) + node_count = resource[Constants.RES_COUNT] image = resource.get(Constants.RES_IMAGE) - node_name_prefix = resource.get(Constants.RES_NAME_PREFIX) flavor = resource.get(Constants.RES_FLAVOR, DEFAULT_FLAVOR) - label = resource.get(Constants.LABEL) for n in range(0, node_count): - node_name = f"{self.name}-{node_name_prefix}{n}" + node_name = self.resource_name(resource, n) + self.existing_map[label].append(node_name) from fabfed.provider.chi.chi_node import ChiNode - node = ChiNode(label=label, name=node_name, image=image, site=site, flavor=flavor, logger=self.logger, + node = ChiNode(label=label, name=node_name, image=image, site=site, flavor=flavor, key_pair=key_pair, network=network, project_name=project_name) - self._nodes.append(node) + self.nodes.append(node) if self.resource_listener: self.resource_listener.on_added(source=self, provider=self, resource=node) @@ -133,17 +183,55 @@ def do_create_resource(self, *, resource: dict): label = resource.get(Constants.LABEL) rtype = resource.get(Constants.RES_TYPE) - if rtype == Constants.RES_TYPE_NETWORK.lower(): - temp = [net for net in self._networks if net.label == label] + if rtype == Constants.RES_TYPE_NETWORK: + if self.modified: + net_name = self.resource_name(resource) - for net in temp: - net.create() + if net_name in self.existing_map[label] and net_name not in self.added_map.get(label, list()): + from fabfed.provider.chi.chi_network import ChiNetwork + from ...util.config_models import Config - if self.resource_listener: - self.resource_listener.on_created(source=self, provider=self, resource=net) + layer3 = Config("", "", {}) + project_name = self.config[CHI_PROJECT_NAME] + + net = ChiNetwork(label=label, name=net_name, site=site, + layer3=layer3, stitch_provider='', project_name=project_name) + net.delete() + + self.logger.info(f"Deleted network: {net_name} at site {site}") + + if self.resource_listener: + self.resource_listener.on_deleted(source=self, provider=self, resource=net) + + return + + net = next(filter(lambda n: n.label == label, self.networks)) + net.create() + + if self.resource_listener: + self.resource_listener.on_created(source=self, provider=self, resource=net) else: - temp = [node for node in self._nodes if node.label == label] + from fabfed.provider.chi.chi_node import ChiNode + + temp: List[ChiNode] = [node for node in self._nodes if node.label == label] + + if self.modified: + for node_name in self.existing_map[label]: + key_pair = self.config[CHI_KEY_PAIR] + project_name = self.config[CHI_PROJECT_NAME] + + if node_name not in self.added_map.get(label, list()): + from fabfed.provider.chi.chi_node import ChiNode + + node = ChiNode(label=label, name=node_name, image='', site=site, flavor='', + key_pair=key_pair, network='', project_name=project_name) + node.delete() + + self.logger.info(f"Deleted node: {node_name} at site {site}") + + if self.resource_listener: + self.resource_listener.on_deleted(source=self, provider=self, resource=node) for node in temp: node.create() @@ -167,14 +255,14 @@ def do_delete_resource(self, *, resource: dict): rtype = resource.get(Constants.RES_TYPE) if rtype == Constants.RES_TYPE_NETWORK.lower(): - net_name = f'{self.name}-{resource.get(Constants.RES_NAME_PREFIX)}' + net_name = self.resource_name(resource) self.logger.debug(f"Deleting network: {net_name} at site {site}") from fabfed.provider.chi.chi_network import ChiNetwork from ...util.config_models import Config layer3 = Config("", "", {}) - net = ChiNetwork(label=label, name=net_name, site=site, logger=self.logger, + net = ChiNetwork(label=label, name=net_name, site=site, layer3=layer3, stitch_provider=None, project_name=project_name) net.delete() self.logger.info(f"Deleted network: {net_name} at site {site}") @@ -185,13 +273,12 @@ def do_delete_resource(self, *, resource: dict): node_count = resource.get(Constants.RES_COUNT, 1) for n in range(0, node_count): - node_name_prefix = resource.get(Constants.RES_NAME_PREFIX) - node_name = f"{self.name}-{node_name_prefix}{n}" + node_name = self.resource_name(resource, n) self.logger.debug(f"Deleting node: {node_name} at site {site}") from fabfed.provider.chi.chi_node import ChiNode - node = ChiNode(label=label, name=node_name, image=None, site=site, flavor=None, logger=self.logger, + node = ChiNode(label=label, name=node_name, image=None, site=site, flavor=None, key_pair=key_pair, network=None, project_name=project_name) node.delete() self.logger.info(f"Deleted node: {node_name} at site {site}") diff --git a/fabfed/provider/cloudlab/cloudlab_provider.py b/fabfed/provider/cloudlab/cloudlab_provider.py index a8fe5aa2..a392f268 100644 --- a/fabfed/provider/cloudlab/cloudlab_provider.py +++ b/fabfed/provider/cloudlab/cloudlab_provider.py @@ -13,8 +13,8 @@ class CloudlabProvider(Provider): def __init__(self, *, type, label, name, config: dict): super().__init__(type=type, label=label, name=name, logger=logger, config=config) - self.supported_resources = [Constants.RES_TYPE_NETWORK.lower(), Constants.RES_TYPE_NODE.lower()] - self.modified = False + self.supported_resources = [Constants.RES_TYPE_NETWORK, Constants.RES_TYPE_NODE] + self._handled_modify = False def setup_environment(self): for attr in CLOUDLAB_CONF_ATTRS: @@ -81,62 +81,71 @@ def do_validate_resource(self, *, resource: dict): if rtype not in self.supported_resources: raise ResourceTypeNotSupported(f"{rtype} for {label}") - if rtype == Constants.RES_TYPE_NODE.lower(): - return + if rtype == Constants.RES_TYPE_NODE: + creation_details = resource[Constants.RES_CREATION_DETAILS] - stitch_info = resource.get(Constants.RES_STITCH_INFO) + if not creation_details['in_config_file']: + return - if not stitch_info: - raise ProviderException(f"{self.label} expecting stitch info in {rtype} resource {label}") + if not resource[Constants.INTERNAL_DEPENDENCIES]: + raise ProviderException(f"{self.label} expecting node {label}'s to depend on clouldlab network") + + return interfaces = resource.get(Constants.RES_INTERFACES, list()) if interfaces and 'vlan' not in interfaces[0]: raise ProviderException(f"{self.label} expecting {label}'s interface to have a vlan") + creation_details = resource[Constants.RES_CREATION_DETAILS] + + if not creation_details['in_config_file']: + return + + stitch_info = resource.get(Constants.RES_STITCH_INFO) + + if not stitch_info: + raise ProviderException(f"{self.label} expecting stitch info in {rtype} resource {label}") + def _get_interfaces(self, resource): interfaces = resource.get(Constants.RES_INTERFACES, list()) - cloudlab_stitch_port = get_stitch_port_for_provider(resource=resource, provider=self.type) if not interfaces: + cloudlab_stitch_port = get_stitch_port_for_provider(resource=resource, provider=self.type) + if 'option' in cloudlab_stitch_port and Constants.RES_INTERFACES in cloudlab_stitch_port['option']: interfaces = cloudlab_stitch_port['option'][Constants.RES_INTERFACES] return interfaces def do_add_resource(self, *, resource: dict): - rtype = resource.get(Constants.RES_TYPE) - label = resource.get(Constants.LABEL) - creation_details = resource[Constants.RES_CREATION_DETAILS] - self.modified = self.modified \ - or (creation_details['created_count'] > 0 and - creation_details['created_count'] != creation_details['total_count']) or \ - not creation_details['in_config_file'] if not creation_details['in_config_file']: return - if rtype == Constants.RES_TYPE_NETWORK.lower(): - states = resource[Constants.SAVED_STATES] + rtype = resource.get(Constants.RES_TYPE) + label = resource.get(Constants.LABEL) + + if rtype == Constants.RES_TYPE_NETWORK: + net_name = self.resource_name(resource) + interface = self.retrieve_attribute_from_saved_state(resource, net_name, attribute='interface') - if states: - state = states[0] - vlan = state.attributes['interface'][0]['vlan'] - interfaces = self._get_interfaces(resource) + assert interface, "expecting a vlan interface in saved state for {label}" + vlan = interface['vlan'] - if isinstance(vlan, str): - vlan = int(vlan) + if isinstance(vlan, str): + vlan = int(vlan) - if interfaces and interfaces[0]['vlan'] != vlan: - self.logger.warning( - f"{self.label} ignoring: {label}'s interface does not match provisioned vlan {vlan}") + interfaces = self._get_interfaces(resource) - resource[Constants.RES_INTERFACES] = [{'vlan': vlan}] + if interfaces and interfaces[0]['vlan'] != vlan: + self.logger.warning( + f"{self.label} ignoring: {label}'s interface does not match provisioned vlan {vlan}") - name_prefix = resource[Constants.RES_NAME_PREFIX] + resource[Constants.RES_INTERFACES] = [{'vlan': vlan}] - if rtype == Constants.RES_TYPE_NODE.lower(): + if rtype == Constants.RES_TYPE_NODE: from .cloudlab_node import CloudlabNode import fabfed.provider.api.dependency_util as util @@ -145,7 +154,7 @@ def do_add_resource(self, *, resource: dict): node_count = resource[Constants.RES_COUNT] for idx in range(0, node_count): - node_name = f'{self.name}-{name_prefix}' + node_name = self.resource_name(resource, idx) node = CloudlabNode(label=label, name=f'{node_name}-{idx}', provider=self, network=net) self._nodes.append(node) self.resource_listener.on_added(source=self, provider=self, resource=node) @@ -153,7 +162,7 @@ def do_add_resource(self, *, resource: dict): from .cloudlab_network import CloudNetwork - net_name = f'{self.name}-{name_prefix}' + net_name = self.resource_name(resource) profile = resource.get(Constants.RES_PROFILE) cloudlab_stitch_port = get_stitch_port_for_provider(resource=resource, provider=self.type) @@ -188,21 +197,37 @@ def do_create_resource(self, *, resource: dict): rtype = resource.get(Constants.RES_TYPE) label = resource.get(Constants.LABEL) - if self.modified: + if not self._handled_modify and self.modified: + assert rtype == Constants.RES_TYPE_NETWORK, "cloudlab expects network to be created first" + self._handled_modify = True + try: self.logger.info(f"Deleting cloudlab resources ....") - self._networks[0].delete() + net_name = self.resource_name(resource) + logger.debug(f"Deleting network: {net_name}") + + from .cloudlab_network import CloudNetwork + + profile = resource.get(Constants.RES_PROFILE) + interfaces = resource.get(Constants.RES_INTERFACES, list()) + layer3 = resource.get(Constants.RES_LAYER3) + net = CloudNetwork(label=label, name=net_name, provider=self, profile=profile, interfaces=interfaces, + layer3=layer3, cluster=None) + + net.delete() self.logger.info(f"Done deleting cloudlab resources ....") - self.modified = False; + self.resource_listener.on_deleted(source=self, provider=self, resource=net) except Exception as e: self.logger.error(f"Exception deleting cloudlab resources ....", e) - if rtype == Constants.RES_TYPE_NETWORK.lower(): - self._networks[0].create() - self.resource_listener.on_created(source=self, provider=self, resource=self._networks[0]) + if rtype == Constants.RES_TYPE_NETWORK: + if self.networks: + self._networks[0].create() + self.resource_listener.on_created(source=self, provider=self, resource=self._networks[0]) + return - if rtype == Constants.RES_TYPE_NODE.lower(): + if rtype == Constants.RES_TYPE_NODE: for node in [node for node in self._nodes if node.label == label]: self.logger.debug(f"Creating node: {vars(node)}") node.create() @@ -215,11 +240,11 @@ def do_delete_resource(self, *, resource: dict): assert rtype in self.supported_resources label = resource.get(Constants.LABEL) - if rtype == Constants.RES_TYPE_NODE.lower(): + if rtype == Constants.RES_TYPE_NODE: # DO NOTHING return - net_name = f'{self.name}-{resource.get(Constants.RES_NAME_PREFIX)}' + net_name = self.resource_name(resource) logger.debug(f"Deleting network: {net_name}") from .cloudlab_network import CloudNetwork diff --git a/fabfed/provider/fabric/fabric_node.py b/fabfed/provider/fabric/fabric_node.py index 89aa1828..43e2cba5 100644 --- a/fabfed/provider/fabric/fabric_node.py +++ b/fabfed/provider/fabric/fabric_node.py @@ -17,8 +17,8 @@ def __init__(self, *, label, delegate: Delegate, nic_model=None, network_label): logger.info(f" Node {self.name} construtor called ... ") self._delegate = delegate self.nic_model = nic_model - slice_object = delegate.get_slice() - self.slice_name = slice_object.get_name() + self._slice_object = delegate.get_slice() + self.slice_name = self._slice_object.get_name() self.mgmt_ip = delegate.get_management_ip() self.mgmt_ip = str(self.mgmt_ip) if self.mgmt_ip else None self.network_label = network_label @@ -38,6 +38,9 @@ def __init__(self, *, label, delegate: Delegate, nic_model=None, network_label): self.components = [dict(name=c.get_name(), model=c.get_model()) for c in delegate.get_components()] self.addr_list = {} + def handle_networking(self): + slice_object = self._slice_object + if not self.mgmt_ip: logger.warning(f" Node {self.name} has no management ip ") return diff --git a/fabfed/provider/fabric/fabric_provider.py b/fabfed/provider/fabric/fabric_provider.py index 79d304cf..95a13244 100644 --- a/fabfed/provider/fabric/fabric_provider.py +++ b/fabfed/provider/fabric/fabric_provider.py @@ -8,6 +8,7 @@ logger = get_logger() + class FabricProvider(Provider): def __init__(self, *, type, label, name, config: dict): super().__init__(type=type, label=label, name=name, logger=logger, config=config) @@ -15,13 +16,14 @@ def __init__(self, *, type, label, name, config: dict): self.retry = 5 self.slice_init = False + # TODO Should not be needed for fablib 1.6.4 from fabrictestbed_extensions.fablib.constants import Constants as FC from fabfed.util.utils import get_base_dir FC.DEFAULT_FABRIC_CONFIG_DIR = get_base_dir(self.name) FC.DEFAULT_FABRIC_RC = f"{FC.DEFAULT_FABRIC_CONFIG_DIR}/fabric_rc" - def _to_abs_for(self, env_var: str, config:dict): + def _to_abs_for(self, env_var: str, config: dict): path = config.get(env_var) from pathlib import Path @@ -33,7 +35,7 @@ def _to_abs_for(self, env_var: str, config:dict): f.read() logger.debug(f"was able to read {path}") except Exception as e: - raise ProviderException(f"{self.name}:Unable to read {path} for {env_var}") + raise ProviderException(f"{self.name}:Unable to read {path} for {env_var}:{e}") try: if path.endswith(".json"): @@ -42,7 +44,7 @@ def _to_abs_for(self, env_var: str, config:dict): json.load(fp) except Exception as e: - raise ProviderException(f"{self.name}:Unable to parse {path} to json for {env_var}") + raise ProviderException(f"{self.name}:Unable to parse {path} to json for {env_var}:{e}") return path @@ -131,4 +133,3 @@ def do_create_resource(self, *, resource: dict): def do_delete_resource(self, *, resource: dict): self._init_slice(True) self.slice.delete_resource(resource=resource) - diff --git a/fabfed/provider/fabric/fabric_slice.py b/fabfed/provider/fabric/fabric_slice.py index 418fd97c..1a5ce160 100644 --- a/fabfed/provider/fabric/fabric_slice.py +++ b/fabfed/provider/fabric/fabric_slice.py @@ -1,5 +1,5 @@ import logging -from typing import List +from typing import List, Union import fabfed.provider.api.dependency_util as util from fabfed.model import Node, Network @@ -21,7 +21,7 @@ def __init__(self, *, provider: FabricProvider, logger: logging.Logger): from fabrictestbed_extensions.fablib.slice import Slice - self.slice_object: Slice = None + self.slice_object: Union[Slice, None] = None self.retry = 10 self.existing_nodes = [] self.existing_networks = [] @@ -63,11 +63,11 @@ def pending(self): def _add_network(self, resource: dict): label = resource[Constants.LABEL] - name_prefix = resource[Constants.RES_NAME_PREFIX] + net_name = self.provider.resource_name(resource) - if name_prefix in self.existing_networks: - delegate = self.slice_object.get_network(name_prefix) - assert delegate is not None, "expected to find network {name_prefix} in slice {self.name}" + if net_name in self.existing_networks: + delegate = self.slice_object.get_network(net_name) + assert delegate is not None, "expected to find network {net_name} in slice {self.name}" layer3 = resource.get(Constants.RES_LAYER3) peer_layer3 = resource.get(Constants.RES_PEER_LAYER3) peering = resource.get(Constants.RES_PEERING) @@ -87,7 +87,7 @@ def _add_network(self, resource: dict): return - network_builder = NetworkBuilder(label, self.provider, self.slice_object, name_prefix, resource) + network_builder = NetworkBuilder(label, self.provider, self.slice_object, net_name, resource) network_builder.handle_facility_port() temp = [] if util.has_resolved_internal_dependencies(resource=resource, attribute='interface'): @@ -108,7 +108,6 @@ def _add_network(self, resource: dict): def _add_node(self, resource: dict): node_count = resource[Constants.RES_COUNT] - name_prefix = resource[Constants.RES_NAME_PREFIX] label = resource[Constants.LABEL] states = resource[Constants.SAVED_STATES] state_map = {} @@ -117,7 +116,7 @@ def _add_node(self, resource: dict): state_map[state.attributes['name']] = state.attributes for i in range(node_count): - name = f"{name_prefix}{i}" + name = self.provider.resource_name(resource, i) if name in self.existing_nodes: delegate = self.slice_object.get_node(name) @@ -182,8 +181,7 @@ def _submit_and_wait(self) -> str or None: self.logger.info(f"Slice provisioning successful {self.slice_object.get_state()}") - days = DEFAULT_RENEWAL_IN_DAYS - + # days = DEFAULT_RENEWAL_IN_DAYS # try: # import datetime # end_date = (datetime.datetime.now() + datetime.timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S %z") @@ -205,6 +203,7 @@ def _reload_nodes(self): n.set_network_label(node.network_label) n.set_used_dataplane_ipv4(node.used_dataplane_ipv4()) temp.append(n) + n.handle_networking() self.provider._nodes = temp @@ -297,7 +296,7 @@ def _do_handle_node_networking(self): temp = [n for n in self.nodes if n.network_label == network.label] for node in temp: - if node.used_dataplane_ipv4(): + if node.used_dataplane_ipv4() and node.used_dataplane_ipv4() in available_ips: available_ips.remove(node.used_dataplane_ipv4()) for node in temp: @@ -370,7 +369,17 @@ def create_resource(self, *, resource: dict): self.slice_object.get_fim_topology().remove_node(name=n) self.slice_modified = True - self.logger.info(f"Done emoving node {n} from slice {self.name}") + self.logger.info(f"Done removing node {n} from slice {self.name}") + + aset = set(self.existing_networks) + bset = {n.name for n in self.networks} + diff = aset - bset + + if diff: + for n in diff: + self.slice_object.get_fim_topology().remove_network_service(name=n) + self.slice_modified = True + self.logger.info(f"Done removing network {n} from slice {self.name}") for network in [net for net in self.networks if net.name in self.existing_networks]: temp = (bset - aset) @@ -440,4 +449,3 @@ def delete_resource(self, *, resource: dict): self.slice_object = None self.slice_created = False self.logger.info(f"Destroyed slice {self.name}") # TODO EMIT DELETE EVENT - diff --git a/fabfed/provider/fabric/fabric_slice_helper.py b/fabfed/provider/fabric/fabric_slice_helper.py index 774ca910..97c1707c 100644 --- a/fabfed/provider/fabric/fabric_slice_helper.py +++ b/fabfed/provider/fabric/fabric_slice_helper.py @@ -129,7 +129,9 @@ def init_slice(name: str, destroy_phase): logger.info(f"Found slice {name}:state={slice_object.get_state()}") except Exception: if not destroy_phase: - return fablib.new_slice(name=name) + slice_object = fablib.new_slice(name=name) + logger.info(f"Created fresh slice {name}:state={slice_object.get_state()}") + return slice_object if destroy_phase: return slice_object diff --git a/fabfed/provider/sense/sense_provider.py b/fabfed/provider/sense/sense_provider.py index b52ecc56..f7406ede 100644 --- a/fabfed/provider/sense/sense_provider.py +++ b/fabfed/provider/sense/sense_provider.py @@ -3,7 +3,6 @@ from fabfed.provider.api.provider import Provider from fabfed.util.constants import Constants from fabfed.util.utils import get_logger -from . import sense_utils from .sense_exceptions import SenseException logger = get_logger() @@ -12,7 +11,9 @@ class SenseProvider(Provider): def __init__(self, *, type, label, name, config: dict): super().__init__(type=type, label=label, name=name, logger=logger, config=config) - self.supported_resources = [Constants.RES_TYPE_NETWORK.lower(), Constants.RES_TYPE_NODE.lower()] + self.supported_resources = [Constants.RES_TYPE_NETWORK, Constants.RES_TYPE_NODE] + self.initialized = False + self._handled_modify = False def setup_environment(self): from fabfed.util import utils @@ -29,9 +30,6 @@ def setup_environment(self): ) self.config = config[profile] - from .sense_client import init_client - - init_client(self.config) @property def private_key_file_location(self): @@ -62,27 +60,47 @@ def _handle_peering_config(self, resource): return peering + def _init_client(self): + if not self.initialized: + from .sense_client import init_client + + init_client(self.config) + self.initialized = True + def do_add_resource(self, *, resource: dict): + self._init_client() + + creation_details = resource[Constants.RES_CREATION_DETAILS] + + if not creation_details['in_config_file']: + return + label = resource.get(Constants.LABEL) rtype = resource.get(Constants.RES_TYPE) if rtype not in self.supported_resources: raise ResourceTypeNotSupported(f"{rtype} for {label}") - name_prefix = resource.get(Constants.RES_NAME_PREFIX) - - if rtype == Constants.RES_TYPE_NODE.lower(): + if rtype == Constants.RES_TYPE_NODE: from .sense_node import SenseNode import fabfed.provider.api.dependency_util as util + from . import sense_utils assert util.has_resolved_internal_dependencies(resource=resource, attribute='network') net = util.get_single_value_for_dependency(resource=resource, attribute='network') profile_uuid = sense_utils.get_profile_uuid(profile=net.profile) vms = sense_utils.get_vms_specs_from_profile(profile_uuid=profile_uuid) - node_name = f'{self.name}-{name_prefix}' + + count = resource[Constants.RES_COUNT] + + if count != len(vms): + from fabfed.exceptions import ProviderException + + raise ProviderException(f'count {count} != sense_vm_specs: {len(vms)}') for idx, vm in enumerate(vms): - node = SenseNode(label=label, name=f'{node_name}-{idx}', network=net.name, spec=vm, provider=self) + node_name = self.resource_name(resource, idx) + node = SenseNode(label=label, name=f'{node_name}', network=net.name, spec=vm, provider=self) self._nodes.append(node) if self.resource_listener: @@ -101,7 +119,7 @@ def do_add_resource(self, *, resource: dict): if not interface.get(Constants.RES_BANDWIDTH): interface[Constants.RES_BANDWIDTH] = resource.get(Constants.RES_BANDWIDTH) - net_name = f'{self.name}-{name_prefix}' + net_name = self.resource_name(resource) profile = resource.get(Constants.RES_PROFILE) if not profile and sense_stitch_port: @@ -125,12 +143,32 @@ def do_add_resource(self, *, resource: dict): self.resource_listener.on_added(source=self, provider=self, resource=net) def do_create_resource(self, *, resource: dict): + assert self.initialized rtype = resource.get(Constants.RES_TYPE) assert rtype in self.supported_resources + label = resource.get(Constants.LABEL) + + if not self._handled_modify and self.modified: + assert rtype == Constants.RES_TYPE_NETWORK, "sense expects network to be created first" + self._handled_modify = True + + self.logger.info(f"Deleting sense resources ....") + + from .sense_network import SenseNetwork + + net_name = self.resource_name(resource) + net = SenseNetwork(label=label, name=net_name, bandwidth=None, profile=None, layer3=None, interfaces=None, + peering=None) + + try: + net.delete() + self.logger.warning(f"Deleted sense resources ....") + except Exception as e: + self.logger.error(f"Exception deleting cloudlab resources ....", e) label = resource.get(Constants.LABEL) - if rtype == Constants.RES_TYPE_NODE.lower(): + if rtype == Constants.RES_TYPE_NODE: for node in [node for node in self._nodes if node.label == label]: self.logger.debug(f"Creating node: {vars(node)}") node.create() @@ -156,11 +194,11 @@ def do_delete_resource(self, *, resource: dict): assert rtype in self.supported_resources label = resource.get(Constants.LABEL) - if rtype == Constants.RES_TYPE_NODE.lower(): + if rtype == Constants.RES_TYPE_NODE: # DO NOTHING return - net_name = f'{self.name}-{resource.get(Constants.RES_NAME_PREFIX)}' + net_name = self.resource_name(resource) logger.debug(f"Deleting network: {net_name}") diff --git a/fabfed/util/constants.py b/fabfed/util/constants.py index 12152e9e..df3b28f7 100644 --- a/fabfed/util/constants.py +++ b/fabfed/util/constants.py @@ -105,5 +105,7 @@ class Constants: "dummy": "fabfed.provider.dummy.dummy_provider.DummyProvider" } + RECONCILE_STATES = True RUN_SSH_TESTER = True COPY_TOKENS = False + PROVIDER_STATE = 'provider_state' diff --git a/fabfed/util/node_tester.py b/fabfed/util/node_tester.py index 1b4fb4e4..01f880d8 100644 --- a/fabfed/util/node_tester.py +++ b/fabfed/util/node_tester.py @@ -115,7 +115,6 @@ def run_ssh_test(self, *, command='ls -l', retry=5, retry_interval=10): def run_dataplane_test(self, *, command='ping -c 3', retry=3, retry_interval=10): for helper in self.helpers: - print(f"Hello ", helper.label) logger.info(f"SSH executing {command} on Node: {helper.label}") for attempt in range(retry): diff --git a/fabfed/util/parser.py b/fabfed/util/parser.py index a689952d..62f74298 100644 --- a/fabfed/util/parser.py +++ b/fabfed/util/parser.py @@ -142,8 +142,8 @@ def _validate_configs(configs: List[Config]): @staticmethod def _validate_resources(resources: List[ResourceConfig]): - if len(resources) == 0: - raise ParseConfigException("no resources found ...") + # if len(resources) == 0: + # raise ParseConfigException("no resources found ...") if len(resources) != len(set(resources)): raise ParseConfigException(f'detected duplicate resources') diff --git a/fabfed/util/state.py b/fabfed/util/state.py index a89698f0..2928f9f5 100644 --- a/fabfed/util/state.py +++ b/fabfed/util/state.py @@ -89,7 +89,7 @@ def dump_plan(*, resources, to_json: bool, summary: bool = False): if not provider_supports_modifiable: in_config_file = details['in_config_file'] changed = not in_config_file or details['total_count'] != details['created_count'] - provider_resource_map[resource.provider.label] = provider_resource_map[resource.provider.label] or changed + provider_resource_map[resource.provider.label] = provider_resource_map[resource.provider.label] or changed for resource in resources: resource_dict = {} @@ -105,7 +105,7 @@ def dump_plan(*, resources, to_json: bool, summary: bool = False): resource_dict['to_be_deleted'] = 0 else: resource_dict['to_be_created'] = 0 - resource_dict['to_be_deleted'] = details['created_count'] - details['total_count'] + resource_dict['to_be_deleted'] = details['created_count'] - details['total_count'] else: resource_dict['to_be_created'] = 0 resource_dict['to_be_deleted'] = details['created_count'] @@ -354,7 +354,7 @@ def save_meta_data(meta_data: dict, friendly_name: str): raise StateException(f'Exception while saving state at temp file {file_path}:{e}') -def save_states(states: List[ProviderState], friendly_name): +def save_states(states: List[ProviderState], friendly_name: str): import yaml import os from fabfed.model.state import get_dumper @@ -375,6 +375,43 @@ def save_states(states: List[ProviderState], friendly_name): shutil.move(temp_file_path, file_path) +def reconcile_state(provider_state: ProviderState, saved_provider_state: ProviderState): + assert provider_state.number_of_created_resources() != provider_state.number_of_total_resources() + + creation_details = provider_state.creation_details + + for resource_state in saved_provider_state.states(): + resource_label = resource_state.label + + if resource_label in creation_details: + resource_details = creation_details[resource_label] + + if resource_details['total_count'] == resource_details['created_count']: + continue + + provider_state.add_if_not_found(resource_state) + + +def reconcile_states(provider_states: List[ProviderState], friendly_name: str) -> List[ProviderState]: + saved_states_map = load_states_as_dict(friendly_name) + if next(filter(lambda s: s.number_of_created_resources() > 0, saved_states_map.values()), None) is None: + return provider_states + + reconciled_states = [] + + for provider_state in provider_states: + if provider_state.label not in saved_states_map: + reconciled_states.append(provider_state) + elif provider_state.number_of_created_resources() == provider_state.number_of_total_resources(): + reconciled_states.append(provider_state) + else: + saved_provider_state = saved_states_map.get(provider_state.label) + reconcile_state(provider_state=provider_state, saved_provider_state=saved_provider_state) + reconciled_states.append(provider_state) + + return reconciled_states + + def save_stats(stats, friendly_name): import yaml import os diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 2475c45c..bdd68e34 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -61,7 +61,7 @@ def run_destroy_workflow(*, session, config_str) -> List[ProviderState]: controller = Controller(config=config, logger=logger) states = sutil.load_states(session) controller.init(session=session, provider_factory=default_provider_factory, provider_states=states) - controller.delete(provider_states=states) + controller.destroy(provider_states=states) sutil.save_states(states, session) if not states: diff --git a/tools/fabfed.py b/tools/fabfed.py index d5905783..a876fb6f 100644 --- a/tools/fabfed.py +++ b/tools/fabfed.py @@ -7,6 +7,7 @@ from fabfed.util import state as sutil from fabfed.util.config import WorkflowConfig from fabfed.util.stats import FabfedStats, Duration +from fabfed.util.constants import Constants def manage_workflow(args): @@ -57,7 +58,7 @@ def manage_workflow(args): sys.exit(1) states = sutil.load_states(args.session) - have_created_resources = next(filter(lambda s: s.number_of_created_resources() > 0, states), None) + # have_created_resources = next(filter(lambda s: s.number_of_created_resources() > 0, states), None) try: controller.init(session=args.session, provider_factory=default_provider_factory, provider_states=states) @@ -84,13 +85,11 @@ def manage_workflow(args): sys.exit(1) workflow_failed = False - interrupted = False try: controller.apply(provider_states=states) except KeyboardInterrupt as kie: logger.error(f"Keyboard Interrupt while creating resources ... {kie}") - interrupted = True workflow_failed = True except ControllerException as ce: logger.error(f"Exceptions while creating resources ... {ce}") @@ -105,25 +104,14 @@ def manage_workflow(args): providers_duration += stats.provider_duration.duration states = controller.get_states() - - if interrupted and have_created_resources: - saved_states_map = sutil.load_states_as_dict(args.session) - reconciled_states = [] - - for state in states: - if (state.number_of_failed_resources() + state.number_of_created_resources()) \ - == state.number_of_total_resources(): - reconciled_states.append(state) - else: - reconciled_states.append(saved_states_map.get(state.label, state)) - - states = reconciled_states - nodes, networks, services, pending, failed = utils.get_counters(states=states) if pending or failed: workflow_failed = True + if Constants.RECONCILE_STATES: + states = sutil.reconcile_states(states, args.session) + sutil.save_states(states, args.session) provider_stats = controller.get_stats() workflow_duration = time.time() - start