Intergrated ESI, allowing users to allocate ESI Network quotas

README.md

@@ -111,6 +111,9 @@ usage: coldfront add_openshift_resource [-h] --name NAME --auth-url AUTH_URL [--
 coldfront add_openshift_resource: error: the following arguments are required: --name, --auth-url
 ```
 
+### Configuring for ESI
+(Quan Pham) TODO Add instructions for configuring ESI
+
 ### Quotas
 
 The amount of quota to start out a resource allocation after approval, can be

ci/run_functional_tests_openstack.sh

@@ -13,6 +13,9 @@ export OPENSTACK_DEVSTACK_APPLICATION_CREDENTIAL_SECRET=$(
 export OPENSTACK_DEVSTACK_APPLICATION_CREDENTIAL_ID=$(
     openstack application credential show "$credential_name" -f value -c id)
 
+export OPENSTACK_ESI_APPLICATION_CREDENTIAL_SECRET=$OPENSTACK_DEVSTACK_APPLICATION_CREDENTIAL_SECRET
+export OPENSTACK_ESI_APPLICATION_CREDENTIAL_ID=$OPENSTACK_DEVSTACK_APPLICATION_CREDENTIAL_ID
+
 export OPENSTACK_PUBLIC_NETWORK_ID=$(openstack network show public -f value -c id)
 
 if [[ ! "${CI}" == "true" ]]; then
@@ -28,6 +31,7 @@ export KEYCLOAK_PASS="nomoresecret"
 export KEYCLOAK_REALM="master"
 
 coverage run --source="." -m django test coldfront_plugin_cloud.tests.functional.openstack
+coverage run --source="." -m django test coldfront_plugin_cloud.tests.functional.esi
 coverage report
 
 openstack application credential delete $OPENSTACK_DEVSTACK_APPLICATION_CREDENTIAL_ID

src/coldfront_plugin_cloud/attributes.py

@@ -75,6 +75,7 @@ class CloudAllocationAttribute:
 QUOTA_VOLUMES_GB = 'OpenStack Volume Quota (GiB)'
 
 QUOTA_FLOATING_IPS = 'OpenStack Floating IP Quota'
+QUOTA_NETWORKS = 'Openstack Network Quota'
 
 QUOTA_OBJECT_GB = 'OpenStack Swift Quota (GiB)'
 
@@ -96,6 +97,7 @@ class CloudAllocationAttribute:
     CloudAllocationAttribute(name=QUOTA_VCPU),
     CloudAllocationAttribute(name=QUOTA_VOLUMES),
     CloudAllocationAttribute(name=QUOTA_VOLUMES_GB),
+    CloudAllocationAttribute(name=QUOTA_NETWORKS),
     CloudAllocationAttribute(name=QUOTA_FLOATING_IPS),
     CloudAllocationAttribute(name=QUOTA_OBJECT_GB),
     CloudAllocationAttribute(name=QUOTA_GPU),

src/coldfront_plugin_cloud/esi.py

@@ -0,0 +1,29 @@
+import logging
+import functools
+import os
+
+from keystoneauth1.identity import v3
+from keystoneauth1 import session
+
+from coldfront_plugin_cloud import attributes, utils
+from coldfront_plugin_cloud.openstack import OpenStackResourceAllocator
+
+class ESIResourceAllocator(OpenStackResourceAllocator):
+
+    QUOTA_KEY_MAPPING = {
+        'network': {
+            'keys': {
+                attributes.QUOTA_FLOATING_IPS: 'floatingip',
+                attributes.QUOTA_NETWORKS: 'network'
+            }
+        }
+    }
+
+    QUOTA_KEY_MAPPING_ALL_KEYS = {quota_key: quota_name for k in QUOTA_KEY_MAPPING.values() for quota_key, quota_name in k['keys'].items()}
+
+    resource_type = 'esi'
+
+    def get_quota(self, project_id):
+        quotas = dict()
+        quotas = self._get_network_quota(quotas, project_id)
+        return quotas

src/coldfront_plugin_cloud/management/commands/add_openstack_resource.py

@@ -1,3 +1,4 @@
+from sre_constants import OP_IGNORE
 from django.core.management.base import BaseCommand
 from django.core.management import call_command
 
@@ -34,13 +35,21 @@ def add_arguments(self, parser):
         parser.add_argument('--network-cidr', type=str, default='192.168.0.0/24',
                             help='CIDR for default networks. '
                                  'Ignored if no --public-network.')
+        parser.add_argument('--esi', action='store_true',
+                            help='Indicates this is an ESI resource (default: False)')
 
     def handle(self, *args, **options):
+
+        if options['esi']:
+            resource_description = 'ESI Bare Metal environment'
+        else:
+            resource_description = 'OpenStack cloud environment'
+
         openstack, _ = Resource.objects.get_or_create(
             resource_type=ResourceType.objects.get(name='OpenStack'),
             parent_resource=None,
             name=options['name'],
-            description='OpenStack cloud environment',
+            description=resource_description,
             is_available=True,
             is_public=True,
             is_allocatable=True
@@ -82,18 +91,21 @@ def handle(self, *args, **options):
             resource=openstack,
             value=options['role']
         )
-        ResourceAttribute.objects.get_or_create(
-            resource_attribute_type=ResourceAttributeType.objects.get(
-                name='quantity_label'),
-            resource=openstack,
-            value='Units of computing to allocate to the project. 1 Unit = 1 Instance, 2 vCPU, 0 GPU, 4G RAM, 2 Volumes, 100 GB Volume Storage, and 1 GB Object Storage'
-        )
-        ResourceAttribute.objects.get_or_create(
-            resource_attribute_type=ResourceAttributeType.objects.get(
-                name='quantity_default_value'),
-            resource=openstack,
-            value=1
-        )
+
+        # Quantity values do not make sense for an ESI allocation
+        if not options['esi']:
+            ResourceAttribute.objects.get_or_create(
+                resource_attribute_type=ResourceAttributeType.objects.get(
+                    name='quantity_label'),
+                resource=openstack,
+                value='Units of computing to allocate to the project. 1 Unit = 1 Instance, 2 vCPU, 0 GPU, 4G RAM, 2 Volumes, 100 GB Volume Storage, and 1 GB Object Storage'
+            )
+            ResourceAttribute.objects.get_or_create(
+                resource_attribute_type=ResourceAttributeType.objects.get(
+                    name='quantity_default_value'),
+                resource=openstack,
+                value=1
+            )
 
         if options['public_network']:
             ResourceAttribute.objects.get_or_create(

src/coldfront_plugin_cloud/management/commands/register_cloud_attributes.py

@@ -126,6 +126,9 @@ def register_resource_type(self):
         resource_models.ResourceType.objects.get_or_create(
             name='OpenShift', description='OpenShift Cloud'
         )
+        resource_models.ResourceType.objects.get_or_create(
+            name='ESI', description='ESI Bare Metal Cloud'
+        )
 
     def handle(self, *args, **options):
         self.register_resource_type()

src/coldfront_plugin_cloud/management/commands/validate_allocations.py

@@ -4,6 +4,7 @@
 from coldfront_plugin_cloud import attributes
 from coldfront_plugin_cloud import openstack
 from coldfront_plugin_cloud import openshift
+from coldfront_plugin_cloud import esi
 from coldfront_plugin_cloud import utils
 from coldfront_plugin_cloud import tasks
 
@@ -66,90 +67,88 @@ def check_institution_specific_code(self, allocation, apply):
 
     def handle(self, *args, **options):
 
-        # Openstack Resources first
-        openstack_resources = Resource.objects.filter(
-            resource_type=ResourceType.objects.get(
-                name='OpenStack'
+        # Deal with Openstack and ESI resources
+        for resource_name in ['OpenStack', 'ESI']:
+            openstack_resources = Resource.objects.filter(
+                resource_type=ResourceType.objects.get(
+                    name=resource_name
+                )
             )
-        )
-        openstack_allocations = Allocation.objects.filter(
-            resources__in=openstack_resources,
-            status=AllocationStatusChoice.objects.get(name='Active')
-        )
-        for allocation in openstack_allocations:
-            self.check_institution_specific_code(allocation, options["apply"])
-            allocation_str = f'{allocation.pk} of project "{allocation.project.title}"'
-            msg = f'Starting resource validation for allocation {allocation_str}.'
-            logger.debug(msg)
-
-            failed_validation = False
-
-            allocator = openstack.OpenStackResourceAllocator(
-                allocation.resources.first(),
-                allocation
+            openstack_allocations = Allocation.objects.filter(
+                resources__in=openstack_resources,
+                status=AllocationStatusChoice.objects.get(name='Active')
             )
+            allocator = tasks.find_allocator(openstack_allocations.first())
 
-            project_id = allocation.get_attribute(attributes.ALLOCATION_PROJECT_ID)
-            if not project_id:
-                logger.error(f'{allocation_str} is active but has no Project ID set.')
-                continue
-
-            try:
-                allocator.identity.projects.get(project_id)
-            except http.NotFound:
-                logger.error(f'{allocation_str} has Project ID {project_id}. But'
-                             f' no project found in OpenStack.')
-                continue
-
-            quota = allocator.get_quota(project_id)
+            for allocation in openstack_allocations:
+                self.check_institution_specific_code(allocation, options["apply"])
+                allocation_str = f'{allocation.pk} of project "{allocation.project.title}"'
+                msg = f'Starting resource validation for allocation {allocation_str}.'
+                logger.debug(msg)
 
-            failed_validation = Command.sync_users(project_id, allocation, allocator, options["apply"])
+                failed_validation = False
 
-            obj_key = openstack.QUOTA_KEY_MAPPING['object']['keys'][attributes.QUOTA_OBJECT_GB]
+                project_id = allocation.get_attribute(attributes.ALLOCATION_PROJECT_ID)
+                if not project_id:
+                    logger.error(f'{allocation_str} is active but has no Project ID set.')
+                    continue
 
-            for attr in attributes.ALLOCATION_QUOTA_ATTRIBUTES:
-                if 'OpenStack' in attr.name:
-                    key = openstack.QUOTA_KEY_MAPPING_ALL_KEYS.get(attr.name, None)
-                    if not key:
-                        # Note(knikolla): Some attributes are only maintained
-                        # for bookkeeping purposes and do not have a
-                        # corresponding quota set on the service.
-                        continue
+                try:
+                    allocator.identity.projects.get(project_id)
+                except http.NotFound:
+                    logger.error(f'{allocation_str} has Project ID {project_id}. But'
+                                 f' no project found in OpenStack.')
+                    continue
 
-                    expected_value = allocation.get_attribute(attr.name)
-                    current_value = quota.get(key, None)
-                    if key == obj_key and expected_value <= 0:
-                        expected_obj_value = 1
-                        current_value = int(allocator.object(project_id).head_account().get(obj_key))
-                        if current_value != expected_obj_value:
+                quota = allocator.get_quota(project_id)
+
+                failed_validation = Command.sync_users(project_id, allocation, allocator, options["apply"])
+
+                obj_key = openstack.OpenStackResourceAllocator.QUOTA_KEY_MAPPING['object']['keys'][attributes.QUOTA_OBJECT_GB]
+
+                for attr in attributes.ALLOCATION_QUOTA_ATTRIBUTES:
+                    if 'OpenStack' in attr.name:
+                        key = allocator.QUOTA_KEY_MAPPING_ALL_KEYS.get(attr.name, None)
+                        if not key:
+                            # Note(knikolla): Some attributes are only maintained
+                            # for bookkeeping purposes and do not have a
+                            # corresponding quota set on the service.
+                            continue
+
+                        expected_value = allocation.get_attribute(attr.name)
+                        current_value = quota.get(key, None)
+                        if key == obj_key and expected_value <= 0:
+                            expected_obj_value = 1
+                            current_value = int(allocator.object(project_id).head_account().get(obj_key))
+                            if current_value != expected_obj_value:
+                                failed_validation = True
+                                msg = (f'Value for quota for {attr.name} = {current_value} does not match expected'
+                                       f' value of {expected_obj_value} on allocation {allocation_str}')
+                                logger.warning(msg)
+                        elif expected_value is None and current_value:
+                            msg = (f'Attribute "{attr.name}" expected on allocation {allocation_str} but not set.'
+                                   f' Current quota is {current_value}.')
+                            if options['apply']:
+                                utils.set_attribute_on_allocation(
+                                    allocation, attr.name, current_value
+                                )
+                                msg = f'{msg} Attribute set to match current quota.'
+                            logger.warning(msg)
+                        elif not current_value == expected_value:
                             failed_validation = True
                             msg = (f'Value for quota for {attr.name} = {current_value} does not match expected'
-                                   f' value of {expected_obj_value} on allocation {allocation_str}')
+                                   f' value of {expected_value} on allocation {allocation_str}')
                             logger.warning(msg)
-                    elif expected_value is None and current_value:
-                        msg = (f'Attribute "{attr.name}" expected on allocation {allocation_str} but not set.'
-                               f' Current quota is {current_value}.')
-                        if options['apply']:
-                            utils.set_attribute_on_allocation(
-                                allocation, attr.name, current_value
-                            )
-                            msg = f'{msg} Attribute set to match current quota.'
-                        logger.warning(msg)
-                    elif not current_value == expected_value:
-                        failed_validation = True
-                        msg = (f'Value for quota for {attr.name} = {current_value} does not match expected'
-                               f' value of {expected_value} on allocation {allocation_str}')
-                        logger.warning(msg)
 
-            if failed_validation and options['apply']:
-                try:
-                    allocator.set_quota(
-                        allocation.get_attribute(attributes.ALLOCATION_PROJECT_ID)
-                    )
-                except Exception as e:
-                    logger.error(f'setting openstack quota failed: {e}')
-                    continue
-                logger.warning(f'Quota for allocation {allocation_str} was out of date. Reapplied!')
+                if failed_validation and options['apply']:
+                    try:
+                        allocator.set_quota(
+                            allocation.get_attribute(attributes.ALLOCATION_PROJECT_ID)
+                        )
+                    except Exception as e:
+                        logger.error(f'setting {resource_name} quota failed: {e}')
+                        continue
+                    logger.warning(f'Quota for allocation {allocation_str} was out of date. Reapplied!')
 
         # Deal with OpenShift