From b2ab0944fce0da3a2014e6a08c7517fa82daba6d Mon Sep 17 00:00:00 2001 From: Ricardo Branco Date: Thu, 29 Aug 2024 11:24:21 +0200 Subject: [PATCH] Remove support for Openstack --- README.md | 3 +- ocw/enums.py | 3 - ocw/lib/cleanup.py | 4 - ocw/lib/openstack.py | 116 ------------------------- requirements.txt | 1 - templates/pcw.ini | 6 -- tests/test_openstack.py | 181 ---------------------------------------- tests/test_pcwconfig.py | 2 +- webui/PCWConfig.py | 5 +- 9 files changed, 3 insertions(+), 318 deletions(-) delete mode 100644 ocw/lib/openstack.py delete mode 100644 tests/test_openstack.py diff --git a/README.md b/README.md index 4c04bf7e..d1c4c5eb 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ PCW has three main flows : c. Volumes in all regions defined d. VPC's ( deletion of VPC means deletion of all assigned to VPC entities first ( security groups , networks etc. )) - For GCE deleting disks, images & network resources (check details in [ocw/lib/gce.py](ocw/lib/gce.py)) - - For Openstack deleting instances, images & keypairs (check details in [ocw/lib/openstack.py](ocw/lib/openstack.py) 3. **Dump entities quantity ( implemented in [ocw/lib/dumpstate.py](ocw/lib/dumpstate.py) )**. To be able to react fast on possible bugs in PCW and/or unexpected creation of many resources there is ability to dump real time data from each CSP into defined InfluxDB instance. This allow building real-time dashboards and/or setup notification flow. @@ -55,7 +54,7 @@ Configuration of PCW happens via a global config file in `/etc/pcw.ini`. See [te ### CSP credentials To be able to connect to CSP PCW needs Service Principal details. Depending on namespaces defined in `pcw.ini` PCW will expect some JSON files to be created -under `/var/pcw/[namespace name]/[Azure/EC2/GCE/Openstack].json`. See [templates/var/example_namespace/](templates/var/example_namespace/) for examples. +under `/var/pcw/[namespace name]/[Azure/EC2/GCE].json`. See [templates/var/example_namespace/](templates/var/example_namespace/) for examples. PCW supports email notifications about left-over instances. See the `notify` section therein and their corresponding comments. diff --git a/ocw/enums.py b/ocw/enums.py index 6523ba57..370e8e97 100644 --- a/ocw/enums.py +++ b/ocw/enums.py @@ -20,7 +20,6 @@ class ProviderChoice(ChoiceEnum): GCE = 'Google' EC2 = 'EC2' AZURE = 'Azure' - OSTACK = 'Openstack' @staticmethod def from_str(provider): @@ -30,8 +29,6 @@ def from_str(provider): return ProviderChoice.EC2 if provider.upper() == ProviderChoice.AZURE: return ProviderChoice.AZURE - if provider.upper() == ProviderChoice.OSTACK: - return ProviderChoice.OSTACK raise ValueError(f"{provider} is not convertable to ProviderChoice") diff --git a/ocw/lib/cleanup.py b/ocw/lib/cleanup.py index 75cdd22e..22f204cc 100644 --- a/ocw/lib/cleanup.py +++ b/ocw/lib/cleanup.py @@ -4,7 +4,6 @@ from ocw.lib.azure import Azure from ocw.lib.ec2 import EC2 from ocw.lib.gce import GCE -from ocw.lib.openstack import Openstack from ocw.lib.eks import EKS from ocw.lib.emailnotify import send_mail, send_cluster_notification from ocw.enums import ProviderChoice @@ -26,9 +25,6 @@ def cleanup_run(): if ProviderChoice.GCE in providers: GCE(namespace).cleanup_all() - if ProviderChoice.OSTACK in providers: - Openstack(namespace).cleanup_all() - except Exception as ex: logger.exception("[%s] Cleanup failed!", namespace) send_mail(f'{type(ex).__name__} on Cleanup in [{namespace}]', traceback.format_exc()) diff --git a/ocw/lib/openstack.py b/ocw/lib/openstack.py deleted file mode 100644 index 1c2cb9e3..00000000 --- a/ocw/lib/openstack.py +++ /dev/null @@ -1,116 +0,0 @@ -from datetime import datetime, timezone -from typing import Dict -from dateutil.parser import parse -import openstack -from openstack.exceptions import OpenStackCloudException -from webui.PCWConfig import PCWConfig -from webui.settings import DEBUG -from .provider import Provider - - -class Openstack(Provider): - __instances: Dict[str, "Openstack"] = {} - - def __init__(self, namespace: str): - super().__init__(namespace) - self.client() - - def __new__(cls, namespace: str): - if namespace not in Openstack.__instances: - Openstack.__instances[namespace] = self = object.__new__(cls) - self.__client = None - return Openstack.__instances[namespace] - - def client(self) -> None: - if self.__client is None: - self.__client = openstack.connect( - debug=bool(DEBUG), - auth_url=self.get_data('auth_url'), - project_name=self.get_data('project_name'), - username=self.get_data('username'), - password=self.get_data('password'), - region_name=self.get_data('region_name'), - user_domain_name=self.get_data('user_domain_name'), - project_id=self.get_data('project_id'), - load_envvars=False, # Avoid reading OS_* environment variables - load_yaml_config=False, # Avoid reading clouds.yaml - ) - return self.__client - - def is_outdated(self, timestamp: str, param: str) -> bool: - now = datetime.now(timezone.utc) - max_days = PCWConfig.get_feature_property('cleanup', param, self._namespace) - return (now - parse(timestamp).astimezone(timezone.utc)).days > max_days - - def cleanup_all(self) -> None: - self._cleanup_instances() - self._cleanup_images() - self._cleanup_keypairs() - - def _cleanup_instances(self) -> None: - # Delete VM's & associated floating IP address(es) - try: - servers = [vm for vm in self.client().compute.servers() if vm.name.startswith("openqa-vm-")] - except OpenStackCloudException as exc: - self.log_warn("Got exception while listing instances: {}", exc) - return - self.log_dbg("Found {} servers", len(servers)) - for server in servers: - if self.is_outdated(server.created_at, "openstack-vm-max-age-days"): - if self.dry_run: - self.log_info("Instance termination {} skipped due to dry run mode", server.name) - else: - self.log_info("Deleting instance {}", server.name) - try: - if not self.client().delete_server( - server.name, - wait=False, - timeout=180, - delete_ips=True, # Delete floating IP address - delete_ip_retry=1): - self.log_err("Failed to delete instance {}", server.name) - except OpenStackCloudException as exc: - self.log_warn("Got exception while deleting instance {}: {}", server.name, exc) - - def _cleanup_images(self) -> None: - try: - images = [image for image in self.client().image.images() if "openqa" in image.tags] - except OpenStackCloudException as exc: - self.log_warn("Got exception while listing images: {}", exc) - return - self.log_dbg("Found {} images", len(images)) - for image in images: - if self.is_outdated(image.created_at, "openstack-image-max-age-days"): - if self.dry_run: - self.log_info("Image deletion {} skipped due to dry run mode", image.name) - else: - self.log_info("Deleting image {}", image.name) - try: - if not self.client().delete_image( - image.name, - wait=False, - timeout=3600): - self.log_err("Failed to delete image {}", image.name) - except OpenStackCloudException as exc: - self.log_warn("Got exception while deleting image {}: {}", image.name, exc) - - def _cleanup_keypairs(self) -> None: - try: - keypairs = [keypair for keypair in self.client().list_keypairs() if keypair.name.startswith("openqa")] - except OpenStackCloudException as exc: - self.log_warn("Got exception while listing keypairs: {}", exc) - return - self.log_dbg("Found {} keypairs", len(keypairs)) - for keypair in keypairs: - if keypair.created_at is None: - keypair.created_at = self.client().compute.get_keypair(keypair.name).created_at - if self.is_outdated(keypair.created_at, "openstack-key-max-days"): - if self.dry_run: - self.log_info("Keypair deletion {} skipped due to dry run mode", keypair.name) - else: - self.log_info("Deleting keypair {}", keypair.name) - try: - if not self.client().delete_keypair(keypair.name): - self.log_err("Failed to delete keypair {}", keypair.name) - except OpenStackCloudException as exc: - self.log_warn("Got exception while deleting keypair {}: {}", keypair.name, exc) diff --git a/requirements.txt b/requirements.txt index a0e1fa59..6a22543a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,6 @@ oauth2client google-api-python-client==2.131.0 google-cloud-storage==2.16.0 openqa_client -openstacksdk~=3.1.0 python-dateutil apscheduler kubernetes diff --git a/templates/pcw.ini b/templates/pcw.ini index b1e2c0bb..fc5dec3f 100644 --- a/templates/pcw.ini +++ b/templates/pcw.ini @@ -36,12 +36,6 @@ ec2-max-age-days = 1 gce-skip-networks = default,tf-network # Max age of data storage resources ( used in Azure and GCE ) max-age-hours = 1 -# Max age for images in Openstack -openstack-image-max-age-days = 3 -# Max age for VM's in Openstack -openstack-vm-max-age-days = 1 -# Max age for keys in Openstack -openstack-key-max-days = 1 # Specify with which namespace, we will do the cleanup. # if not specifed default/namespaces list will be taken instead namespaces = qac, sapha diff --git a/tests/test_openstack.py b/tests/test_openstack.py deleted file mode 100644 index 64821709..00000000 --- a/tests/test_openstack.py +++ /dev/null @@ -1,181 +0,0 @@ -from collections import namedtuple -from unittest.mock import MagicMock, patch -from datetime import datetime, timezone, timedelta -from pytest import fixture -from ocw.lib.openstack import Openstack -from webui.PCWConfig import PCWConfig - - -def assert_not_called_with(self, *args, **kwargs): - try: - self.assert_called_with(*args, **kwargs) - except AssertionError: - return - raise AssertionError('Expected %s to not have been called.' % self._format_mock_call_signature(args, kwargs)) - - -MagicMock.assert_not_called_with = assert_not_called_with - - -@fixture -def openstack_instance(): - with patch.object(Openstack, 'read_auth_json', return_value={}): - with patch.object(Openstack, 'get_data', return_value="CustomRegion"): - with patch('openstack.connect') as mock_connect: - mock_client = MagicMock() - mock_connect.return_value = mock_client - yield Openstack('test_namespace') - - -def test_is_outdated(openstack_instance): - now = datetime.now(timezone.utc) - - max_days = 10 - patch.object(PCWConfig, 'get_feature_property', return_value=max_days) - - # Test cases with different timestamps and max_days values - test_cases = [ - # Timestamp is within the valid range - { - "timestamp": (now - timedelta(days=1)).isoformat(), - "expected": False, - }, - # Timestamp is exactly at the max_days limit - { - "timestamp": (now - timedelta(days=max_days)).isoformat(), - "expected": True, - }, - # Timestamp exceeds the max_days limit - { - "timestamp": (now - timedelta(days=max_days+1)).isoformat(), - "expected": True, - }, - # Timestamp is in the future - { - "timestamp": (now + timedelta(days=max_days+1)).isoformat(), - "expected": False, - }, - ] - - for test in test_cases: - assert openstack_instance.is_outdated(test["timestamp"], "openstack-vm-max-age-days") == test["expected"] - - -def test_cleanup_all(openstack_instance): - openstack_instance.cleanup_all() - openstack_instance.client().compute.servers.assert_called_once() - openstack_instance.client().image.images.assert_called_once() - openstack_instance.client().list_keypairs.assert_called_once() - - -def test_cleanup_instances(openstack_instance): - # Prepare test data - outdated_server = MagicMock() - outdated_server.name = 'openqa-vm-outdated' - outdated_server.created_at = (datetime.now(timezone.utc) - timedelta(days=8)).isoformat() - - recent_server = MagicMock() - recent_server.name = 'openqa-vm-recent' - recent_server.created_at = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() - - openstack_instance.client().compute.servers.return_value = [outdated_server, recent_server] - - # Test with dry_run=False - openstack_instance.dry_run = False - openstack_instance._cleanup_instances() - - kwargs = {'wait': False, 'timeout': 180, 'delete_ips': True, 'delete_ip_retry': 1} - openstack_instance.client().delete_server.assert_called_once_with(outdated_server.name, **kwargs) - openstack_instance.client().delete_server.assert_not_called_with(recent_server.name) - - # Reset mocks - openstack_instance.client().delete_server.reset_mock() - - # Test with dry_run=True - openstack_instance.dry_run = True - openstack_instance._cleanup_instances() - - openstack_instance.client().delete_server.assert_not_called() - - -def test_cleanup_images(openstack_instance): - Image = namedtuple('Image', ['name', 'created_at', 'tags']) - - # Prepare test data - max_days = 7 - patch.object(PCWConfig, 'get_feature_property', return_value=max_days) - images = [ - Image( - name='openqa-image-outdated', - created_at=(datetime.now(timezone.utc) - timedelta(days=max_days+1)).isoformat(), - tags=['openqa'], - ), - Image( - name='openqa-image-recent', - created_at=(datetime.now(timezone.utc) - timedelta(days=1)).isoformat(), - tags=['openqa'], - ), - Image( - name='not-openqa-image', - created_at=(datetime.now(timezone.utc) - timedelta(days=max_days+1)).isoformat(), - tags=[], - ), - ] - openstack_instance.client().image.images.return_value = images - - # Test with dry_run=False - openstack_instance.dry_run = False - openstack_instance._cleanup_images() - - kwargs = {'wait': False, 'timeout': 3600} - openstack_instance.client().delete_image.assert_called_once_with(images[0].name, **kwargs) - openstack_instance.client().delete_image.assert_not_called_with(images[1].name) - openstack_instance.client().delete_image.assert_not_called_with(images[2].name) - - # Reset mocks - openstack_instance.client().delete_image.reset_mock() - - # Test with dry_run=True - openstack_instance.dry_run = True - openstack_instance._cleanup_images() - - openstack_instance.client().delete_image.assert_not_called() - - -def test_cleanup_keypairs(openstack_instance): - Keypair = namedtuple('Keypair', ['name', 'created_at']) - - # Prepare test data - max_days = 3 - keypairs = [ - Keypair( - name='openqa-keypair-outdated', - created_at=(datetime.now(timezone.utc) - timedelta(days=max_days+1)).isoformat(), - ), - Keypair( - name='openqa-keypair-recent', - created_at=(datetime.now(timezone.utc) - timedelta(days=1)).isoformat(), - ), - Keypair( - name='not-openqa-keypair', - created_at=(datetime.now(timezone.utc) - timedelta(days=max_days+1)).isoformat(), - ), - ] - openstack_instance.client().list_keypairs.return_value = keypairs - - # Test with dry_run=False - openstack_instance.dry_run = False - openstack_instance._cleanup_keypairs() - - openstack_instance.client().delete_keypair.assert_called_once_with(keypairs[0].name) - openstack_instance.client().delete_keypair.assert_not_called_with(keypairs[1].name) - openstack_instance.client().delete_keypair.assert_not_called_with(keypairs[2].name) - - # Reset mocks - openstack_instance.client().delete_keypair.reset_mock() - - # Test with dry_run=True - openstack_instance.dry_run = True - openstack_instance._cleanup_keypairs() - - openstack_instance.client().delete_keypair.assert_not_called() diff --git a/tests/test_pcwconfig.py b/tests/test_pcwconfig.py index b7710a55..75a90bcf 100644 --- a/tests/test_pcwconfig.py +++ b/tests/test_pcwconfig.py @@ -94,7 +94,7 @@ def test_get_namespaces_for_feature_default_feature_exists_namespace_in_feature( def test_get_providers_for_not_existed_feature(pcw_file): providers = PCWConfig.get_providers_for('get_providers_for', 'not_existent') assert type(providers) is list - assert not {'EC2', 'AZURE', 'GCE', 'OSTACK'} ^ set(providers) + assert not {'EC2', 'AZURE', 'GCE'} ^ set(providers) def test_get_providers_for_existed_feature(pcw_file): diff --git a/webui/PCWConfig.py b/webui/PCWConfig.py index 90f8489a..6284dc8e 100644 --- a/webui/PCWConfig.py +++ b/webui/PCWConfig.py @@ -61,9 +61,6 @@ def get_feature_property(feature: str, feature_property: str, namespace: str | N 'cleanup/ec2-max-age-days': {'default': -1, 'return_type': int}, 'cleanup/gce-bucket': {'default': None, 'return_type': str}, 'cleanup/max-age-hours': {'default': 24 * 7, 'return_type': int}, - 'cleanup/openstack-image-max-age-days': {'default': 3, 'return_type': int}, - 'cleanup/openstack-vm-max-age-days': {'default': 1, 'return_type': int}, - 'cleanup/openstack-key-max-days': {'default': 1, 'return_type': int}, 'updaterun/default_ttl': {'default': 44400, 'return_type': int}, 'notify/to': {'default': None, 'return_type': str}, 'notify/age-hours': {'default': 12, 'return_type': int}, @@ -94,7 +91,7 @@ def get_namespaces_for(feature: str) -> list: @staticmethod def get_providers_for(feature: str, namespace: str): return ConfigFile().getList(f'{feature}.namespace.{namespace}/providers', - ConfigFile().getList(f'{feature}/providers', ['EC2', 'AZURE', 'GCE', 'OSTACK'])) + ConfigFile().getList(f'{feature}/providers', ['EC2', 'AZURE', 'GCE'])) @staticmethod def get_k8s_clusters_for_provider(namespace: str, provider: str) -> list: