From ba89a23d1fca57b7640a337bf5c69392665b5e13 Mon Sep 17 00:00:00 2001 From: Ricardo Branco Date: Thu, 31 Aug 2023 12:23:41 +0200 Subject: [PATCH] Rename vault_namespace to namespace --- ocw/lib/aks.py | 8 +++---- ocw/lib/azure.py | 8 +++---- ocw/lib/db.py | 24 +++++++++---------- ocw/lib/ec2.py | 8 +++---- ocw/lib/eks.py | 8 +++---- ocw/lib/emailnotify.py | 4 ++-- ocw/lib/gce.py | 8 +++---- ocw/lib/gke.py | 8 +++---- ocw/lib/openstack.py | 8 +++---- ...t_namespace_instance_namespace_and_more.py | 22 +++++++++++++++++ ocw/models.py | 4 ++-- ocw/views.py | 2 +- tests/generators.py | 2 +- tests/test_db.py | 10 ++++---- tests/test_models.py | 2 +- 15 files changed, 74 insertions(+), 52 deletions(-) create mode 100644 ocw/migrations/0012_rename_vault_namespace_instance_namespace_and_more.py diff --git a/ocw/lib/aks.py b/ocw/lib/aks.py index 38a9b4a7..88766a28 100644 --- a/ocw/lib/aks.py +++ b/ocw/lib/aks.py @@ -9,11 +9,11 @@ class AKS(Provider): __instances = {} default_region: str = 'eu-central-1' - def __new__(cls, vault_namespace): - if vault_namespace not in AKS.__instances: - AKS.__instances[vault_namespace] = self = object.__new__(cls) + def __new__(cls, namespace): + if namespace not in AKS.__instances: + AKS.__instances[namespace] = self = object.__new__(cls) self.__kubectl_client = {} - return AKS.__instances[vault_namespace] + return AKS.__instances[namespace] def subscription(self) -> str: return self.get_data('subscription_id') diff --git a/ocw/lib/azure.py b/ocw/lib/azure.py index 604abbd3..984a9751 100644 --- a/ocw/lib/azure.py +++ b/ocw/lib/azure.py @@ -22,14 +22,14 @@ def __init__(self, namespace: str): self.check_credentials() self.__gallery = PCWConfig.get_feature_property('cleanup', 'azure-gallery-name', namespace) - def __new__(cls, vault_namespace: str) -> 'Azure': - if vault_namespace not in Azure.__instances: - Azure.__instances[vault_namespace] = self = object.__new__(cls) + def __new__(cls, namespace: str) -> 'Azure': + if namespace not in Azure.__instances: + Azure.__instances[namespace] = self = object.__new__(cls) self.__compute_mgmt_client = None self.__sp_credentials = None self.__resource_mgmt_client = None self.__blob_service_client = None - return Azure.__instances[vault_namespace] + return Azure.__instances[namespace] def subscription(self) -> str: return self.get_data('subscription_id') diff --git a/ocw/lib/db.py b/ocw/lib/db.py index 7ba61f61..3358eac4 100644 --- a/ocw/lib/db.py +++ b/ocw/lib/db.py @@ -22,9 +22,9 @@ def save_or_update_instance(csp_data: dict) -> None: provider = csp_data['provider'] namespace = csp_data['namespace'] - if Instance.objects.filter(provider=provider, instance_id=csp_data['id'], vault_namespace=namespace).exists(): + if Instance.objects.filter(provider=provider, instance_id=csp_data['id'], namespace=namespace).exists(): logger.debug("[%s] Update instance %s:%s", namespace, provider, csp_data['id']) - local_instance = Instance.objects.get(provider=provider, instance_id=csp_data['id'], vault_namespace=namespace) + local_instance = Instance.objects.get(provider=provider, instance_id=csp_data['id'], namespace=namespace) if local_instance.region != csp_data['region']: logger.info("[%s] Instance %s:%s changed region from %s to %s", namespace, provider, csp_data['id'], local_instance.region, csp_data['region']) @@ -38,7 +38,7 @@ def save_or_update_instance(csp_data: dict) -> None: logger.debug("[%s] Create instance %s:%s", namespace, provider, csp_data['id']) local_instance = Instance( provider=provider, - vault_namespace=namespace, + namespace=namespace, first_seen=csp_data['first_seen'], instance_id=csp_data['id'], ttl=timedelta(seconds=int(csp_data['tags'].get('openqa_ttl', csp_data['default_ttl']))), @@ -107,7 +107,7 @@ def gce_extract_data(csp_instance, namespace: str, default_ttl: int) -> dict: def _update_provider(provider: str, namespace: str, default_ttl: int) -> None: - instance_cnt = Instance.objects.filter(provider=provider, vault_namespace=namespace).update(active=False) + instance_cnt = Instance.objects.filter(provider=provider, namespace=namespace).update(active=False) logger.debug("%d got active state false", instance_cnt) if ProviderChoice.from_str(provider) == ProviderChoice.AZURE: instances = Azure(namespace).list_resource_groups() @@ -129,7 +129,7 @@ def _update_provider(provider: str, namespace: str, default_ttl: int) -> None: for i in instances: save_or_update_instance(gce_extract_data(i, namespace, default_ttl)) logger.info("%d instances from GCE successfully processed", len(instances)) - Instance.objects.filter(provider=provider, vault_namespace=namespace, + Instance.objects.filter(provider=provider, namespace=namespace, active=False).update(state=StateChoice.DELETED) @@ -166,11 +166,11 @@ def update_run() -> None: def delete_instance(instance: type[Instance]) -> None: if instance.provider == ProviderChoice.AZURE: - Azure(instance.vault_namespace).delete_resource(instance.instance_id) + Azure(instance.namespace).delete_resource(instance.instance_id) elif instance.provider == ProviderChoice.EC2: - EC2(instance.vault_namespace).delete_instance(instance.region, instance.instance_id) + EC2(instance.namespace).delete_instance(instance.region, instance.instance_id) elif instance.provider == ProviderChoice.GCE: - GCE(instance.vault_namespace).delete_instance(instance.instance_id, instance.region) + GCE(instance.namespace).delete_instance(instance.instance_id, instance.region) else: raise NotImplementedError( f"Provider({instance.provider}).delete() isn't implemented") @@ -183,22 +183,22 @@ def auto_delete_instances() -> None: for namespace in PCWConfig.get_namespaces_for('default'): logger.debug("Running auto_delete_instances for %s", namespace) instances = [ - i for i in Instance.objects.filter(state=StateChoice.ACTIVE, vault_namespace=namespace).exclude(ignore=True) + i for i in Instance.objects.filter(state=StateChoice.ACTIVE, namespace=namespace).exclude(ignore=True) if i.ttl_expired() or i.is_cancelled() ] logger.debug("Found %d instances for deletion", len(instances)) email_text = set() for instance in instances: if instance.ttl_expired(): - logger.debug("[%s] TTL expired for instance %s:%s %s", instance.vault_namespace, + logger.debug("[%s] TTL expired for instance %s:%s %s", instance.namespace, instance.provider, instance.instance_id, instance.all_time_fields()) else: - logger.debug("[%s] Job cancelled for instance %s:%s %s", instance.vault_namespace, + logger.debug("[%s] Job cancelled for instance %s:%s %s", instance.namespace, instance.provider, instance.instance_id, instance.all_time_fields()) try: delete_instance(instance) except Exception: - msg = f"[{instance.vault_namespace}] Deleting instance ({instance.provider}:{instance.instance_id}) failed" + msg = f"[{instance.namespace}] Deleting instance ({instance.provider}:{instance.instance_id}) failed" logger.exception(msg) email_text.add(f"{msg}\n\n{traceback.format_exc()}") diff --git a/ocw/lib/ec2.py b/ocw/lib/ec2.py index 6cca140c..d52a2b3c 100644 --- a/ocw/lib/ec2.py +++ b/ocw/lib/ec2.py @@ -23,15 +23,15 @@ def __init__(self, namespace: str): else: self.all_regions = self.get_all_regions() - def __new__(cls, vault_namespace: str): - if vault_namespace not in EC2.__instances: - EC2.__instances[vault_namespace] = self = object.__new__(cls) + def __new__(cls, namespace: str): + if namespace not in EC2.__instances: + EC2.__instances[namespace] = self = object.__new__(cls) self.__ec2_client = {} self.__ec2_resource = {} self.__secret = None self.__key = None - return EC2.__instances[vault_namespace] + return EC2.__instances[namespace] def check_credentials(self) -> None: self.__secret = self.get_data('secret_access_key') diff --git a/ocw/lib/eks.py b/ocw/lib/eks.py index 6f7b6d46..f388ca92 100644 --- a/ocw/lib/eks.py +++ b/ocw/lib/eks.py @@ -16,14 +16,14 @@ class EKS(Provider): default_region: str = 'eu-central-1' __cluster_regions = [] - def __new__(cls, vault_namespace): - if vault_namespace not in EKS.__instances: - EKS.__instances[vault_namespace] = self = object.__new__(cls) + def __new__(cls, namespace): + if namespace not in EKS.__instances: + EKS.__instances[namespace] = self = object.__new__(cls) self.__eks_client = {} self.__kubectl_client = {} self.__aws_dir = None - return EKS.__instances[vault_namespace] + return EKS.__instances[namespace] def __init__(self, namespace: str): super().__init__(namespace) diff --git a/ocw/lib/emailnotify.py b/ocw/lib/emailnotify.py index ecaba7e4..3e13b413 100644 --- a/ocw/lib/emailnotify.py +++ b/ocw/lib/emailnotify.py @@ -23,7 +23,7 @@ def draw_instance_table(objects): obj.provider, obj.instance_id, obj.cspinfo.get_tag('openqa_created_by', 'N/A'), - obj.vault_namespace, + obj.namespace, obj.age_formatted(), build_absolute_uri(reverse(views.delete, args=[obj.id])), "" if link is None else link['url'] @@ -40,7 +40,7 @@ def send_leftover_notification(): # Handle namespaces for namespace in PCWConfig.get_namespaces_for('notify'): receiver_email = PCWConfig.get_feature_property('notify', 'to', namespace) - namespace_objects = all_instances.filter(vault_namespace=namespace) + namespace_objects = all_instances.filter(namespace=namespace) if namespace_objects.filter(notified=False).count() > 0 and receiver_email: send_mail(f'CSP left overs - {namespace}', body_prefix + draw_instance_table(namespace_objects), receiver_email=receiver_email) diff --git a/ocw/lib/gce.py b/ocw/lib/gce.py index ab4348e4..5b3da8a1 100644 --- a/ocw/lib/gce.py +++ b/ocw/lib/gce.py @@ -12,10 +12,10 @@ class GCE(Provider): __instances = {} __skip_networks = frozenset({"default"}) - def __new__(cls, vault_namespace): - if vault_namespace not in GCE.__instances: - GCE.__instances[vault_namespace] = object.__new__(cls) - return GCE.__instances[vault_namespace] + def __new__(cls, namespace): + if namespace not in GCE.__instances: + GCE.__instances[namespace] = object.__new__(cls) + return GCE.__instances[namespace] def __init__(self, namespace): super().__init__(namespace) diff --git a/ocw/lib/gke.py b/ocw/lib/gke.py index f3d2ccdc..cefcdd87 100644 --- a/ocw/lib/gke.py +++ b/ocw/lib/gke.py @@ -9,13 +9,13 @@ class GKE(GCE): __instances = {} - def __new__(cls, vault_namespace): - if vault_namespace not in GKE.__instances: - GKE.__instances[vault_namespace] = self = object.__new__(cls) + def __new__(cls, namespace): + if namespace not in GKE.__instances: + GKE.__instances[namespace] = self = object.__new__(cls) self.__gke_client = None self.__kubectl_client = {} - return GKE.__instances[vault_namespace] + return GKE.__instances[namespace] def gke_client(self): if self.__gke_client is None: diff --git a/ocw/lib/openstack.py b/ocw/lib/openstack.py index 0612b52a..1c2cb9e3 100644 --- a/ocw/lib/openstack.py +++ b/ocw/lib/openstack.py @@ -15,11 +15,11 @@ def __init__(self, namespace: str): super().__init__(namespace) self.client() - def __new__(cls, vault_namespace: str): - if vault_namespace not in Openstack.__instances: - Openstack.__instances[vault_namespace] = self = object.__new__(cls) + def __new__(cls, namespace: str): + if namespace not in Openstack.__instances: + Openstack.__instances[namespace] = self = object.__new__(cls) self.__client = None - return Openstack.__instances[vault_namespace] + return Openstack.__instances[namespace] def client(self) -> None: if self.__client is None: diff --git a/ocw/migrations/0012_rename_vault_namespace_instance_namespace_and_more.py b/ocw/migrations/0012_rename_vault_namespace_instance_namespace_and_more.py new file mode 100644 index 00000000..d3af0e2d --- /dev/null +++ b/ocw/migrations/0012_rename_vault_namespace_instance_namespace_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.4 on 2023-08-31 10:55 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('ocw', '0011_alter_instance_provider'), + ] + + operations = [ + migrations.RenameField( + model_name='instance', + old_name='vault_namespace', + new_name='namespace', + ), + migrations.AlterUniqueTogether( + name='instance', + unique_together={('provider', 'instance_id', 'namespace')}, + ), + ] diff --git a/ocw/models.py b/ocw/models.py index bb6de19c..06d1c10f 100644 --- a/ocw/models.py +++ b/ocw/models.py @@ -31,7 +31,7 @@ class Instance(models.Model): help_text='Local computed state of that Instance') instance_id = models.CharField(max_length=200) region = models.CharField(max_length=64, default='') - vault_namespace = models.CharField(max_length=64, default='') + namespace = models.CharField(max_length=64, default='') notified = models.BooleanField(default=False) ignore = models.BooleanField(default=False) TAG_IGNORE = 'pcw_ignore' @@ -81,7 +81,7 @@ def is_cancelled(self) -> bool: return False class Meta: # pylint: disable=too-few-public-methods - unique_together = (('provider', 'instance_id', 'vault_namespace'),) + unique_together = (('provider', 'instance_id', 'namespace'),) class CspInfo(models.Model): diff --git a/ocw/views.py b/ocw/views.py index 87b4cc24..a56d8ebd 100644 --- a/ocw/views.py +++ b/ocw/views.py @@ -51,7 +51,7 @@ def instance_json(request): "ttl", "instance_id", "region", - "vault_namespace", + "namespace", "ignore" ), ) diff --git a/tests/generators.py b/tests/generators.py index 3c3b5151..560fcd3b 100644 --- a/tests/generators.py +++ b/tests/generators.py @@ -27,7 +27,7 @@ def generate_model_instance(jobid_tag, created_by_tag): cspinfo=CspInfo(tags=json_dump_tags, type=fake.uuid4()), provider=fake.uuid4(), instance_id=fake.uuid4(), - vault_namespace=fake.uuid4() + namespace=fake.uuid4() ) diff --git a/tests/test_db.py b/tests/test_db.py index ae18dbf4..47e06573 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -39,7 +39,7 @@ def __init__(self, provider): self.region = None self.save_called = False self.provider = provider - self.vault_namespace = fake.uuid4() + self.namespace = fake.uuid4() def save(self): self.save_called = True @@ -69,7 +69,7 @@ def extract_data(): @pytest.fixture def azure_fixture(monkeypatch, extract_data): - monkeypatch.setattr(Azure, '__new__', lambda cls, vault_namespace: AzureMock()) + monkeypatch.setattr(Azure, '__new__', lambda cls, namespace: AzureMock()) return extract_data @@ -219,7 +219,7 @@ def mocked_send_mail(arg1, arg2): def test_delete_instances_azure(monkeypatch): - monkeypatch.setattr(Azure, '__new__', lambda cls, vault_namespace: AzureMock()) + monkeypatch.setattr(Azure, '__new__', lambda cls, namespace: AzureMock()) instance = InstanceMock(ProviderChoice.AZURE) @@ -230,7 +230,7 @@ def test_delete_instances_azure(monkeypatch): def test_delete_instances_ec2(monkeypatch): - monkeypatch.setattr(EC2, '__new__', lambda cls, vault_namespace: EC2Mock()) + monkeypatch.setattr(EC2, '__new__', lambda cls, namespace: EC2Mock()) instance = InstanceMock(ProviderChoice.EC2) @@ -241,7 +241,7 @@ def test_delete_instances_ec2(monkeypatch): def test_delete_instances_gce(monkeypatch): - monkeypatch.setattr(GCE, '__new__', lambda cls, vault_namespace: GCEMock()) + monkeypatch.setattr(GCE, '__new__', lambda cls, namespace: GCEMock()) instance = InstanceMock(ProviderChoice.GCE) diff --git a/tests/test_models.py b/tests/test_models.py index f2d87ed6..a7954bde 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -34,7 +34,7 @@ def example_instance_data(): 'last_seen': datetime(2023, 1, 2, tzinfo=timezone.utc), 'instance_id': 'INSTANCE_ID', 'region': 'REGION', - 'vault_namespace': 'VAULT_NAMESPACE', + 'namespace': 'NAMESPACE', }