Integrate ESI into Coldfront

README.md

@@ -78,12 +78,15 @@ dashboard or through the helper command:
 
 ```bash
 $ coldfront add_openstack_resource
-usage: coldfront add_openstack_resource [-h] --name NAME --auth-url AUTH_URL [--users-domain USERS_DOMAIN] [--projects-domain PROJECTS_DOMAIN] --idp IDP
-                                        [--protocol PROTOCOL] [--role ROLE] [--version] [-v {0,1,2,3}] [--settings SETTINGS] [--pythonpath PYTHONPATH] [--traceback]
-                                        [--no-color] [--force-color]
+usage: coldfront add_openstack_resource [-h] --name NAME --auth-url AUTH_URL [--users-domain USERS_DOMAIN] [--projects-domain PROJECTS_DOMAIN] --idp IDP [--protocol PROTOCOL] [--role ROLE]
+                                        [--public-network PUBLIC_NETWORK] [--network-cidr NETWORK_CIDR] [--esi] [--version] [-v {0,1,2,3}] [--settings SETTINGS] [--pythonpath PYTHONPATH] [--traceback]
+                                        [--no-color] [--force-color] [--skip-checks]
 coldfront add_openstack_resource: error: the following arguments are required: --name, --auth-url, --idp
 ```
 
+An Openstack resource can be specified as an ESI resource by setting the `--esi` command flag.
+ESI resource allocations will only have quotas for network resources by default.
+
 ### Configuring for OpenShift
 
 Note: OpenShift support requires deploying the [openshift-acct-mgt][]

ci/run_functional_tests_openstack.sh

@@ -13,6 +13,9 @@ export OPENSTACK_DEVSTACK_APPLICATION_CREDENTIAL_SECRET=$(
 export OPENSTACK_DEVSTACK_APPLICATION_CREDENTIAL_ID=$(
     openstack application credential show "$credential_name" -f value -c id)
 
+export OPENSTACK_ESI_APPLICATION_CREDENTIAL_SECRET=$OPENSTACK_DEVSTACK_APPLICATION_CREDENTIAL_SECRET
+export OPENSTACK_ESI_APPLICATION_CREDENTIAL_ID=$OPENSTACK_DEVSTACK_APPLICATION_CREDENTIAL_ID
+
 export OPENSTACK_PUBLIC_NETWORK_ID=$(openstack network show public -f value -c id)
 
 if [[ ! "${CI}" == "true" ]]; then
@@ -28,6 +31,7 @@ export KEYCLOAK_PASS="nomoresecret"
 export KEYCLOAK_REALM="master"
 
 coverage run --source="." -m django test coldfront_plugin_cloud.tests.functional.openstack
+coverage run --source="." -m django test coldfront_plugin_cloud.tests.functional.esi
 coverage report
 
 openstack application credential delete $OPENSTACK_DEVSTACK_APPLICATION_CREDENTIAL_ID

src/coldfront_plugin_cloud/attributes.py

@@ -75,6 +75,7 @@ class CloudAllocationAttribute:
 QUOTA_VOLUMES_GB = 'OpenStack Volume Quota (GiB)'
 
 QUOTA_FLOATING_IPS = 'OpenStack Floating IP Quota'
+QUOTA_NETWORKS = 'Openstack Network Quota'
 
 QUOTA_OBJECT_GB = 'OpenStack Swift Quota (GiB)'
 
@@ -96,6 +97,7 @@ class CloudAllocationAttribute:
     CloudAllocationAttribute(name=QUOTA_VCPU),
     CloudAllocationAttribute(name=QUOTA_VOLUMES),
     CloudAllocationAttribute(name=QUOTA_VOLUMES_GB),
+    CloudAllocationAttribute(name=QUOTA_NETWORKS),
     CloudAllocationAttribute(name=QUOTA_FLOATING_IPS),
     CloudAllocationAttribute(name=QUOTA_OBJECT_GB),
     CloudAllocationAttribute(name=QUOTA_GPU),

src/coldfront_plugin_cloud/esi.py

@@ -0,0 +1,29 @@
+import logging
+import functools
+import os
+
+from keystoneauth1.identity import v3
+from keystoneauth1 import session
+
+from coldfront_plugin_cloud import attributes, utils
+from coldfront_plugin_cloud.openstack import OpenStackResourceAllocator
+
+class ESIResourceAllocator(OpenStackResourceAllocator):
+
+    QUOTA_KEY_MAPPING = {
+        'network': {
+            'keys': {
+                attributes.QUOTA_FLOATING_IPS: 'floatingip',
+                attributes.QUOTA_NETWORKS: 'network'
+            }
+        }
+    }
+
+    QUOTA_KEY_MAPPING_ALL_KEYS = {quota_key: quota_name for k in QUOTA_KEY_MAPPING.values() for quota_key, quota_name in k['keys'].items()}
+
+    resource_type = 'esi'
+
+    def get_quota(self, project_id):
+        quotas = dict()
+        quotas = self._get_network_quota(quotas, project_id)
+        return quotas

src/coldfront_plugin_cloud/management/commands/add_openstack_resource.py

@@ -34,13 +34,23 @@ def add_arguments(self, parser):
         parser.add_argument('--network-cidr', type=str, default='192.168.0.0/24',
                             help='CIDR for default networks. '
                                  'Ignored if no --public-network.')
+        parser.add_argument('--esi', action='store_true',
+                            help='Indicates this is an ESI resource (default: False)')
 
     def handle(self, *args, **options):
+
+        if options['esi']:
+            resource_description = 'ESI Bare Metal environment'
+            resource_type = 'ESI'
+        else:
+            resource_description = 'OpenStack cloud environment'
+            resource_type = 'OpenStack'
+
         openstack, _ = Resource.objects.get_or_create(
-            resource_type=ResourceType.objects.get(name='OpenStack'),
+            resource_type=ResourceType.objects.get(name=resource_type),
             parent_resource=None,
             name=options['name'],
-            description='OpenStack cloud environment',
+            description=resource_description,
             is_available=True,
             is_public=True,
             is_allocatable=True
@@ -82,18 +92,21 @@ def handle(self, *args, **options):
             resource=openstack,
             value=options['role']
         )
-        ResourceAttribute.objects.get_or_create(
-            resource_attribute_type=ResourceAttributeType.objects.get(
-                name='quantity_label'),
-            resource=openstack,
-            value='Units of computing to allocate to the project. 1 Unit = 1 Instance, 2 vCPU, 0 GPU, 4G RAM, 2 Volumes, 100 GB Volume Storage, and 1 GB Object Storage'
-        )
-        ResourceAttribute.objects.get_or_create(
-            resource_attribute_type=ResourceAttributeType.objects.get(
-                name='quantity_default_value'),
-            resource=openstack,
-            value=1
-        )
+
+        # Quantity values do not make sense for an ESI allocation
+        if not options['esi']:
+            ResourceAttribute.objects.get_or_create(
+                resource_attribute_type=ResourceAttributeType.objects.get(
+                    name='quantity_label'),
+                resource=openstack,
+                value='Units of computing to allocate to the project. 1 Unit = 1 Instance, 2 vCPU, 0 GPU, 4G RAM, 2 Volumes, 100 GB Volume Storage, and 1 GB Object Storage'
+            )
+            ResourceAttribute.objects.get_or_create(
+                resource_attribute_type=ResourceAttributeType.objects.get(
+                    name='quantity_default_value'),
+                resource=openstack,
+                value=1
+            )
 
         if options['public_network']:
             ResourceAttribute.objects.get_or_create(

src/coldfront_plugin_cloud/management/commands/register_cloud_attributes.py

@@ -126,6 +126,9 @@ def register_resource_type(self):
         resource_models.ResourceType.objects.get_or_create(
             name='OpenShift', description='OpenShift Cloud'
         )
+        resource_models.ResourceType.objects.get_or_create(
+            name='ESI', description='ESI Bare Metal Cloud'
+        )
 
     def handle(self, *args, **options):
         self.register_resource_type()

src/coldfront_plugin_cloud/management/commands/validate_allocations.py

@@ -4,6 +4,7 @@
 from coldfront_plugin_cloud import attributes
 from coldfront_plugin_cloud import openstack
 from coldfront_plugin_cloud import openshift
+from coldfront_plugin_cloud import esi
 from coldfront_plugin_cloud import utils
 from coldfront_plugin_cloud import tasks
 
@@ -66,11 +67,9 @@ def check_institution_specific_code(self, allocation, apply):
 
     def handle(self, *args, **options):
 
-        # Openstack Resources first
+        # Deal with Openstack and ESI resources
         openstack_resources = Resource.objects.filter(
-            resource_type=ResourceType.objects.get(
-                name='OpenStack'
-            )
+            resource_type__name__in=['OpenStack', 'ESI']
         )
         openstack_allocations = Allocation.objects.filter(
             resources__in=openstack_resources,
@@ -84,10 +83,7 @@ def handle(self, *args, **options):
 
             failed_validation = False
 
-            allocator = openstack.OpenStackResourceAllocator(
-                allocation.resources.first(),
-                allocation
-            )
+            allocator = tasks.find_allocator(allocation)
 
             project_id = allocation.get_attribute(attributes.ALLOCATION_PROJECT_ID)
             if not project_id:
@@ -105,11 +101,11 @@ def handle(self, *args, **options):
 
             failed_validation = Command.sync_users(project_id, allocation, allocator, options["apply"])
 
-            obj_key = openstack.QUOTA_KEY_MAPPING['object']['keys'][attributes.QUOTA_OBJECT_GB]
+            obj_key = openstack.OpenStackResourceAllocator.QUOTA_KEY_MAPPING['object']['keys'][attributes.QUOTA_OBJECT_GB]
 
             for attr in attributes.ALLOCATION_QUOTA_ATTRIBUTES:
                 if 'OpenStack' in attr.name:
-                    key = openstack.QUOTA_KEY_MAPPING_ALL_KEYS.get(attr.name, None)
+                    key = allocator.QUOTA_KEY_MAPPING_ALL_KEYS.get(attr.name, None)
                     if not key:
                         # Note(knikolla): Some attributes are only maintained
                         # for bookkeeping purposes and do not have a
@@ -147,7 +143,7 @@ def handle(self, *args, **options):
                         allocation.get_attribute(attributes.ALLOCATION_PROJECT_ID)
                     )
                 except Exception as e:
-                    logger.error(f'setting openstack quota failed: {e}')
+                    logger.error(f'setting {allocation.resources.first()} quota failed: {e}')
                     continue
                 logger.warning(f'Quota for allocation {allocation_str} was out of date. Reapplied!')
 

src/coldfront_plugin_cloud/openstack.py

@@ -21,40 +21,8 @@
 # 1 GB = 1 000 000 000 B = 10^9 B
 GB_IN_BYTES = 2 ** 30
 
-# Map the attribute name in ColdFront, to the client of the respective
-# service, the version of the API, and the key in the payload.
-QUOTA_KEY_MAPPING = {
-    'compute': {
-        'keys': {
-            attributes.QUOTA_INSTANCES: 'instances',
-            attributes.QUOTA_VCPU: 'cores',
-            attributes.QUOTA_RAM: 'ram',
-        },
-    },
-    'network': {
-        'keys': {
-            attributes.QUOTA_FLOATING_IPS: 'floatingip',
-        }
-    },
-    'object': {
-        'keys': {
-            attributes.QUOTA_OBJECT_GB: 'x-account-meta-quota-bytes',
-        }
-    },
-    'volume': {
-        'keys': {
-            attributes.QUOTA_VOLUMES: 'volumes',
-            attributes.QUOTA_VOLUMES_GB: 'gigabytes',
-        }
-    },
-}
-
 COLDFRONT_RGW_SWIFT_INIT_USER = 'coldfront-swift-init'
 
-QUOTA_KEY_MAPPING_ALL_KEYS = dict()
-for service in QUOTA_KEY_MAPPING.keys():
-    QUOTA_KEY_MAPPING_ALL_KEYS.update(QUOTA_KEY_MAPPING[service]['keys'])
-
 def get_session_for_resource_via_password(resource, username, password, project_id):
     auth_url = resource.get_attribute(attributes.RESOURCE_AUTH_URL)
     user_domain = resource.get_attribute(attributes.RESOURCE_USER_DOMAIN)
@@ -94,6 +62,39 @@ def get_session_for_resource(resource):
 
 
 class OpenStackResourceAllocator(base.ResourceAllocator):
+    # Map the attribute name in ColdFront, to the client of the respective
+    # service, the version of the API, and the key in the payload.
+    QUOTA_KEY_MAPPING = {
+        'compute': {
+            'keys': {
+                attributes.QUOTA_INSTANCES: 'instances',
+                attributes.QUOTA_VCPU: 'cores',
+                attributes.QUOTA_RAM: 'ram',
+            },
+        },
+        'network': {
+            'keys': {
+                attributes.QUOTA_FLOATING_IPS: 'floatingip',
+            }
+        },
+        'object': {
+            'keys': {
+                attributes.QUOTA_OBJECT_GB: 'x-account-meta-quota-bytes',
+            }
+        },
+        'volume': {
+            'keys': {
+                attributes.QUOTA_VOLUMES: 'volumes',
+                attributes.QUOTA_VOLUMES_GB: 'gigabytes',
+            }
+        },
+    }
+
+    QUOTA_KEY_MAPPING_ALL_KEYS = {
+        quota_key: quota_name 
+        for k in QUOTA_KEY_MAPPING.values() 
+        for quota_key, quota_name in k['keys'].items()
+    }
 
     resource_type = 'openstack'
 
@@ -161,7 +162,7 @@ def set_quota(self, project_id):
         # If an attribute with the appropriate name is associated with an
         # allocation, set that as the quota. Otherwise, multiply
         # the quantity attribute via the mapping table above.
-        for service_name, service in QUOTA_KEY_MAPPING.items():
+        for service_name, service in self.QUOTA_KEY_MAPPING.items():
             # No need to do any calculations here, just go through each service
             # and set the value in the attribute.
             payload = dict()
@@ -188,7 +189,7 @@ def _set_object_quota(self, project_id, payload):
             # Note(knikolla): For consistency with other OpenStack
             # quotas we're storing this as GB on the attribute and
             # converting to bytes for Swift.
-            obj_q_mapping = QUOTA_KEY_MAPPING['object']['keys'][
+            obj_q_mapping = self.QUOTA_KEY_MAPPING['object']['keys'][
                 attributes.QUOTA_OBJECT_GB
             ]
             payload[obj_q_mapping] *= GB_IN_BYTES
@@ -235,22 +236,27 @@ def _init_rgw_for_project(self, project_id):
         logger.debug(f'rgw swift stat for {project_id}:\n{stat}')
         self.remove_role_from_user(COLDFRONT_RGW_SWIFT_INIT_USER, project_id)
 
+    def _get_network_quota(self, quotas, project_id):
+        network_quota = self.network.show_quota(project_id)['quota']
+        for k in self.QUOTA_KEY_MAPPING['network']['keys'].values():
+            quotas[k] = network_quota.get(k)
+
+        return quotas
+
     def get_quota(self, project_id):
         quotas = dict()
 
         compute_quota = self.compute.quotas.get(project_id)
-        for k in QUOTA_KEY_MAPPING['compute']['keys'].values():
+        for k in self.QUOTA_KEY_MAPPING['compute']['keys'].values():
             quotas[k] = compute_quota.__getattr__(k)
 
         volume_quota = self.volume.quotas.get(project_id)
-        for k in QUOTA_KEY_MAPPING['volume']['keys'].values():
+        for k in self.QUOTA_KEY_MAPPING['volume']['keys'].values():
             quotas[k] = volume_quota.__getattr__(k)
 
-        network_quota = self.network.show_quota(project_id)['quota']
-        for k in QUOTA_KEY_MAPPING['network']['keys'].values():
-            quotas[k] = network_quota.get(k)
+        quotas = self._get_network_quota(quotas, project_id)
 
-        key = QUOTA_KEY_MAPPING['object']['keys'][attributes.QUOTA_OBJECT_GB]
+        key = self.QUOTA_KEY_MAPPING['object']['keys'][attributes.QUOTA_OBJECT_GB]
         try:
             swift = self.object(project_id).head_account()
             quotas[key] = int(int(swift.get(key)) / GB_IN_BYTES)

src/coldfront_plugin_cloud/tasks.py

@@ -10,6 +10,7 @@
                                     base,
                                     openstack,
                                     openshift,
+                                    esi,
                                     utils)
 
 logger = logging.getLogger(__name__)
@@ -35,6 +36,10 @@
         attributes.QUOTA_REQUESTS_STORAGE: 20,
         attributes.QUOTA_REQUESTS_GPU: 0,
         attributes.QUOTA_PVC: 2
+    },
+    'esi': {
+        attributes.QUOTA_FLOATING_IPS: 0,
+        attributes.QUOTA_NETWORKS: 0
     }
 }
 
@@ -48,6 +53,10 @@
     },
     'openshift': {
         attributes.QUOTA_REQUESTS_GPU: 0,
+    },
+    'esi': {
+        attributes.QUOTA_FLOATING_IPS: 1,
+        attributes.QUOTA_NETWORKS: 1
     }
 }
 
@@ -56,6 +65,7 @@ def find_allocator(allocation) -> base.ResourceAllocator:
     allocators = {
         'openstack': openstack.OpenStackResourceAllocator,
         'openshift': openshift.OpenShiftResourceAllocator,
+        'esi': esi.ESIResourceAllocator,
     }
     # TODO(knikolla): It doesn't seem to be possible to select multiple resources
     # when requesting a new allocation, so why is this multivalued?