From b3b05274e7ce360d0185ce780cc45b78bad9f58e Mon Sep 17 00:00:00 2001 From: Ryan Belgrave Date: Sat, 23 Jun 2018 13:09:46 -0500 Subject: [PATCH] omg I did so much stuff and didn't commit * Remove all auth and switch to using OpenID Connect * Split out all APIs to their components; compute, iam, location * Remove model IDs and switch to names, this makes it easier to dev with k8s * Add swagger autogen support /swagger/json and /swagger/ui * Split out metadata server into it's own library * Make sure model controllers don't constantly save when nothing changed * Add updated_at field to models * Add caching via redis so we don't kill the k8s api and speed up responses * Use vm uuids when communicating with vcenter * Rename "global" to "system" and split out policies * Add IAM groups that link to openid groups * Service Accounts and IAM Groups have sandwich.local "emails" * Don't generate own user tokens, use OpenID Connect id tokens * Service Account keys expire after 10 years * Add IAM system and project policy documents (similar to GCP policy documents) * There still is no API yet to modify these documents * User/Service Account that creates a project is added as the owner --- deli/cache/__init__.py | 3 + deli/cache/client.py | 41 ++ deli/counter/auth/driver.py | 25 - deli/counter/auth/drivers/github/driver.py | 70 --- deli/counter/auth/drivers/github/router.py | 97 ---- deli/counter/auth/manager.py | 37 -- deli/counter/auth/policy.py | 457 +++++------------- deli/counter/auth/token.py | 269 +++++++---- deli/counter/auth/validation_models/github.py | 12 - deli/counter/cli/commands/gen_admin.py | 14 +- deli/counter/http/app.py | 16 +- .../mounts/root/errors}/__init__.py | 0 .../root/{routes/v1 => }/errors/quota.py | 0 deli/counter/http/mounts/root/mount.py | 61 ++- .../mounts/root/routes/auth}/__init__.py | 0 .../mounts/root/routes/auth/v1}/__init__.py | 0 .../http/mounts/root/routes/auth/v1/oauth.py | 71 +++ .../v1/validation_models}/__init__.py | 0 .../routes/auth/v1/validation_models/oauth.py | 14 + .../routes/{v1/auth => compute}/__init__.py | 0 .../v1}/__init__.py | 0 .../root/routes/{ => compute}/v1/flavor.py | 62 ++- .../mounts/root/routes/compute/v1/images.py | 146 ++++++ .../root/routes/{ => compute}/v1/instance.py | 234 ++++++--- .../root/routes/{ => compute}/v1/keypairs.py | 67 ++- .../routes/{ => compute}/v1/network_ports.py | 44 +- .../root/routes/{ => compute}/v1/networks.py | 78 ++- .../v1/validation_models}/__init__.py | 0 .../v1/validation_models/flavor.py | 6 +- .../v1/validation_models/images.py | 39 +- .../v1/validation_models/instances.py | 60 ++- .../v1/validation_models/keypairs.py | 6 +- .../v1/validation_models/network_ports.py | 10 +- .../v1/validation_models/networks.py | 14 +- .../v1/validation_models/volume.py | 16 +- .../root/routes/{ => compute}/v1/volume.py | 170 +++++-- .../counter/http/mounts/root/routes/health.py | 9 +- .../{v1/validation_models => iam}/__init__.py | 0 .../mounts/root/routes/iam/v1}/__init__.py | 0 .../http/mounts/root/routes/iam/v1/policy.py | 118 +++++ .../root/routes/{ => iam}/v1/project_quota.py | 38 +- .../mounts/root/routes/iam/v1/projects.py | 128 +++++ .../http/mounts/root/routes/iam/v1/role.py | 290 +++++++++++ .../routes/{ => iam}/v1/service_accounts.py | 290 +++++++---- .../iam/v1/validation_models}/__init__.py | 0 .../v1}/validation_models/policy.py | 0 .../iam/v1/validation_models/projects.py | 76 +++ .../auth => iam/v1}/validation_models/role.py | 4 +- .../v1/validation_models/service_accounts.py | 18 +- .../mounts/root/routes/location}/__init__.py | 0 .../root/routes/location/v1}/__init__.py | 0 .../root/routes/{ => location}/v1/regions.py | 84 +++- .../v1/validation_models}/__init__.py | 0 .../v1/validation_models/regions.py | 8 +- .../v1/validation_models/zones.py | 14 +- .../root/routes/{ => location}/v1/zones.py | 90 +++- .../http/mounts/root/routes/swagger.py | 83 ++++ .../http/mounts/root/routes/v1/auth/auth.py | 34 -- .../http/mounts/root/routes/v1/auth/policy.py | 44 -- .../http/mounts/root/routes/v1/auth/role.py | 178 ------- .../http/mounts/root/routes/v1/auth/tokens.py | 97 ---- .../v1/auth/validation_models/tokens.py | 22 - .../http/mounts/root/routes/v1/images.py | 219 --------- .../mounts/root/routes/v1/project_members.py | 108 ----- .../http/mounts/root/routes/v1/projects.py | 95 ---- .../routes/v1/validation_models/projects.py | 103 ---- deli/counter/http/router.py | 37 ++ deli/counter/http/spec/__init__.py | 0 deli/counter/http/spec/plugins/__init__.py | 0 deli/counter/http/spec/plugins/docstring.py | 112 +++++ deli/counter/settings.py | 27 +- deli/kubernetes/resources/const.py | 9 +- deli/kubernetes/resources/model.py | 208 +++++--- deli/kubernetes/resources/project.py | 105 ++-- .../resources/v1alpha1/flavor/model.py | 4 +- .../resources/v1alpha1/iam_group/__init__.py | 0 .../v1alpha1/iam_group/controller.py | 0 .../resources/v1alpha1/iam_group/model.py | 21 + .../resources/v1alpha1/iam_policy/__init__.py | 0 .../v1alpha1/iam_policy/controller.py | 67 +++ .../resources/v1alpha1/iam_policy/model.py | 52 ++ .../resources/v1alpha1/iam_role/__init__.py | 0 .../v1alpha1/{role => iam_role}/controller.py | 44 +- .../resources/v1alpha1/iam_role/model.py | 67 +++ .../v1alpha1/iam_service_account/__init__.py | 0 .../controller.py | 56 +-- .../v1alpha1/iam_service_account/model.py | 78 +++ .../resources/v1alpha1/image/controller.py | 29 +- .../resources/v1alpha1/image/model.py | 117 ++--- .../resources/v1alpha1/instance/controller.py | 106 ++-- .../resources/v1alpha1/instance/model.py | 135 +++--- .../resources/v1alpha1/network/controller.py | 6 +- .../resources/v1alpha1/network/model.py | 27 +- .../v1alpha1/project_member/controller.py | 31 -- .../v1alpha1/project_member/model.py | 45 -- .../v1alpha1/project_quota/controller.py | 30 +- .../resources/v1alpha1/project_quota/model.py | 4 +- .../resources/v1alpha1/region/controller.py | 6 +- .../resources/v1alpha1/region/model.py | 4 +- .../resources/v1alpha1/role/model.py | 84 ---- .../v1alpha1/service_account/model.py | 107 ---- .../resources/v1alpha1/volume/controller.py | 107 ++-- .../resources/v1alpha1/volume/model.py | 61 ++- .../resources/v1alpha1/zone/controller.py | 4 +- .../resources/v1alpha1/zone/model.py | 22 +- deli/manager/cli/commands/run.py | 59 ++- deli/manager/vmware.py | 8 +- deli/menu/cli/commands/run.py | 71 ++- deli/menu/cli/main.py | 4 +- deli/menu/metadata/__init__.py | 0 deli/menu/metadata/driver.py | 64 +++ deli/menu/metadata/vm_client.py | 60 +++ deli/menu/models/out_of_band.py | 30 -- deli/menu/vspc/async_telnet.py | 185 ------- deli/menu/vspc/server.py | 231 --------- deli/menu/vspc/vm_client.py | 298 ------------ docker-compose.yaml | 5 + docker/counter/.env-sample | 39 +- docker/counter/Dockerfile | 2 - docker/counter/wsgi.ini | 1 + docker/manager/Dockerfile | 2 - docker/menu/.env-sample | 11 +- docker/menu/Dockerfile | 2 - requirements.txt | 12 +- setup.py | 2 +- 125 files changed, 3536 insertions(+), 3651 deletions(-) create mode 100644 deli/cache/__init__.py create mode 100644 deli/cache/client.py delete mode 100644 deli/counter/auth/driver.py delete mode 100644 deli/counter/auth/drivers/github/driver.py delete mode 100644 deli/counter/auth/drivers/github/router.py delete mode 100644 deli/counter/auth/manager.py delete mode 100644 deli/counter/auth/validation_models/github.py rename deli/counter/{auth/drivers => http/mounts/root/errors}/__init__.py (100%) rename deli/counter/http/mounts/root/{routes/v1 => }/errors/quota.py (100%) rename deli/counter/{auth/drivers/github => http/mounts/root/routes/auth}/__init__.py (100%) rename deli/counter/{auth/validation_models => http/mounts/root/routes/auth/v1}/__init__.py (100%) create mode 100644 deli/counter/http/mounts/root/routes/auth/v1/oauth.py rename deli/counter/http/mounts/root/routes/{v1 => auth/v1/validation_models}/__init__.py (100%) create mode 100644 deli/counter/http/mounts/root/routes/auth/v1/validation_models/oauth.py rename deli/counter/http/mounts/root/routes/{v1/auth => compute}/__init__.py (100%) rename deli/counter/http/mounts/root/routes/{v1/auth/validation_models => compute/v1}/__init__.py (100%) rename deli/counter/http/mounts/root/routes/{ => compute}/v1/flavor.py (57%) create mode 100644 deli/counter/http/mounts/root/routes/compute/v1/images.py rename deli/counter/http/mounts/root/routes/{ => compute}/v1/instance.py (66%) rename deli/counter/http/mounts/root/routes/{ => compute}/v1/keypairs.py (57%) rename deli/counter/http/mounts/root/routes/{ => compute}/v1/network_ports.py (67%) rename deli/counter/http/mounts/root/routes/{ => compute}/v1/networks.py (63%) rename deli/counter/http/mounts/root/routes/{v1/errors => compute/v1/validation_models}/__init__.py (100%) rename deli/counter/http/mounts/root/routes/{ => compute}/v1/validation_models/flavor.py (89%) rename deli/counter/http/mounts/root/routes/{ => compute}/v1/validation_models/images.py (58%) rename deli/counter/http/mounts/root/routes/{ => compute}/v1/validation_models/instances.py (65%) rename deli/counter/http/mounts/root/routes/{ => compute}/v1/validation_models/keypairs.py (92%) rename deli/counter/http/mounts/root/routes/{ => compute}/v1/validation_models/network_ports.py (78%) rename deli/counter/http/mounts/root/routes/{ => compute}/v1/validation_models/networks.py (90%) rename deli/counter/http/mounts/root/routes/{ => compute}/v1/validation_models/volume.py (80%) rename deli/counter/http/mounts/root/routes/{ => compute}/v1/volume.py (66%) rename deli/counter/http/mounts/root/routes/{v1/validation_models => iam}/__init__.py (100%) rename deli/{kubernetes/resources/v1alpha1/project_member => counter/http/mounts/root/routes/iam/v1}/__init__.py (100%) create mode 100644 deli/counter/http/mounts/root/routes/iam/v1/policy.py rename deli/counter/http/mounts/root/routes/{ => iam}/v1/project_quota.py (52%) create mode 100644 deli/counter/http/mounts/root/routes/iam/v1/projects.py create mode 100644 deli/counter/http/mounts/root/routes/iam/v1/role.py rename deli/counter/http/mounts/root/routes/{ => iam}/v1/service_accounts.py (52%) rename deli/{kubernetes/resources/v1alpha1/role => counter/http/mounts/root/routes/iam/v1/validation_models}/__init__.py (100%) rename deli/counter/http/mounts/root/routes/{v1/auth => iam/v1}/validation_models/policy.py (100%) create mode 100644 deli/counter/http/mounts/root/routes/iam/v1/validation_models/projects.py rename deli/counter/http/mounts/root/routes/{v1/auth => iam/v1}/validation_models/role.py (91%) rename deli/counter/http/mounts/root/routes/{ => iam}/v1/validation_models/service_accounts.py (69%) rename deli/{kubernetes/resources/v1alpha1/service_account => counter/http/mounts/root/routes/location}/__init__.py (100%) rename deli/{menu/models => counter/http/mounts/root/routes/location/v1}/__init__.py (100%) rename deli/counter/http/mounts/root/routes/{ => location}/v1/regions.py (59%) rename deli/{menu/vspc => counter/http/mounts/root/routes/location/v1/validation_models}/__init__.py (100%) rename deli/counter/http/mounts/root/routes/{ => location}/v1/validation_models/regions.py (91%) rename deli/counter/http/mounts/root/routes/{ => location}/v1/validation_models/zones.py (87%) rename deli/counter/http/mounts/root/routes/{ => location}/v1/zones.py (60%) create mode 100644 deli/counter/http/mounts/root/routes/swagger.py delete mode 100644 deli/counter/http/mounts/root/routes/v1/auth/auth.py delete mode 100644 deli/counter/http/mounts/root/routes/v1/auth/policy.py delete mode 100644 deli/counter/http/mounts/root/routes/v1/auth/role.py delete mode 100644 deli/counter/http/mounts/root/routes/v1/auth/tokens.py delete mode 100644 deli/counter/http/mounts/root/routes/v1/auth/validation_models/tokens.py delete mode 100644 deli/counter/http/mounts/root/routes/v1/images.py delete mode 100644 deli/counter/http/mounts/root/routes/v1/project_members.py delete mode 100644 deli/counter/http/mounts/root/routes/v1/projects.py delete mode 100644 deli/counter/http/mounts/root/routes/v1/validation_models/projects.py create mode 100644 deli/counter/http/spec/__init__.py create mode 100644 deli/counter/http/spec/plugins/__init__.py create mode 100644 deli/counter/http/spec/plugins/docstring.py create mode 100644 deli/kubernetes/resources/v1alpha1/iam_group/__init__.py create mode 100644 deli/kubernetes/resources/v1alpha1/iam_group/controller.py create mode 100644 deli/kubernetes/resources/v1alpha1/iam_group/model.py create mode 100644 deli/kubernetes/resources/v1alpha1/iam_policy/__init__.py create mode 100644 deli/kubernetes/resources/v1alpha1/iam_policy/controller.py create mode 100644 deli/kubernetes/resources/v1alpha1/iam_policy/model.py create mode 100644 deli/kubernetes/resources/v1alpha1/iam_role/__init__.py rename deli/kubernetes/resources/v1alpha1/{role => iam_role}/controller.py (68%) create mode 100644 deli/kubernetes/resources/v1alpha1/iam_role/model.py create mode 100644 deli/kubernetes/resources/v1alpha1/iam_service_account/__init__.py rename deli/kubernetes/resources/v1alpha1/{service_account => iam_service_account}/controller.py (64%) create mode 100644 deli/kubernetes/resources/v1alpha1/iam_service_account/model.py delete mode 100644 deli/kubernetes/resources/v1alpha1/project_member/controller.py delete mode 100644 deli/kubernetes/resources/v1alpha1/project_member/model.py delete mode 100644 deli/kubernetes/resources/v1alpha1/role/model.py delete mode 100644 deli/kubernetes/resources/v1alpha1/service_account/model.py create mode 100644 deli/menu/metadata/__init__.py create mode 100644 deli/menu/metadata/driver.py create mode 100644 deli/menu/metadata/vm_client.py delete mode 100644 deli/menu/models/out_of_band.py delete mode 100644 deli/menu/vspc/async_telnet.py delete mode 100644 deli/menu/vspc/server.py delete mode 100644 deli/menu/vspc/vm_client.py diff --git a/deli/cache/__init__.py b/deli/cache/__init__.py new file mode 100644 index 0000000..e8e536e --- /dev/null +++ b/deli/cache/__init__.py @@ -0,0 +1,3 @@ +from deli.cache.client import CacheClient + +cache_client = CacheClient() diff --git a/deli/cache/client.py b/deli/cache/client.py new file mode 100644 index 0000000..30a1b93 --- /dev/null +++ b/deli/cache/client.py @@ -0,0 +1,41 @@ +import json + +import redis + + +class CacheClient(object): + + def __init__(self): + self.redis_client: redis.StrictRedis = None + self.default_cache_time = 600 + + def connect(self, url): + self.redis_client = redis.StrictRedis.from_url(url) + + def get(self, key): + data = self.redis_client.get(key) + if data is None: + return None + return json.loads(data) + + def scan(self, match): + cursor = '0' + while cursor != 0: + cursor, keys = self.redis_client.scan(cursor=cursor, match=match, + count=1000) # Do we keep count hardcoded to 1000? + for key in keys: + item = self.get(key) + if item is None: + continue + yield key, item + + def pipeline(self): + return self.redis_client.pipeline() + + def set(self, key, data, ex=None): + if ex is None: + ex = self.default_cache_time + return self.redis_client.set(key, json.dumps(data), ex=ex) + + def delete(self, key): + return self.redis_client.delete(key) diff --git a/deli/counter/auth/driver.py b/deli/counter/auth/driver.py deleted file mode 100644 index bf72eb7..0000000 --- a/deli/counter/auth/driver.py +++ /dev/null @@ -1,25 +0,0 @@ -import logging -from abc import ABCMeta, abstractmethod -from typing import Dict - -from deli.counter.http.router import SandwichRouter - - -class AuthDriver(object): - __metaclass__ = ABCMeta - - def __init__(self, name): - self.name = name - self.logger = logging.getLogger("%s.%s" % (self.__module__, self.__class__.__name__)) - - @abstractmethod - def discover_options(self) -> Dict: - raise NotImplementedError - - @abstractmethod - def auth_router(self) -> SandwichRouter: - raise NotImplementedError - - @abstractmethod - def health(self): - return None diff --git a/deli/counter/auth/drivers/github/driver.py b/deli/counter/auth/drivers/github/driver.py deleted file mode 100644 index 72b06b2..0000000 --- a/deli/counter/auth/drivers/github/driver.py +++ /dev/null @@ -1,70 +0,0 @@ -from typing import Dict - -import github -from github.NamedUser import NamedUser -from simple_settings import settings - -from deli.counter.auth.driver import AuthDriver -from deli.counter.auth.drivers.github.router import GithubAuthRouter - - -class GithubAuthDriver(AuthDriver): - def __init__(self): - super().__init__('github') - - def auth_router(self) -> GithubAuthRouter: - return GithubAuthRouter(self) - - def discover_options(self) -> Dict: # pragma: no cover - return {} - - def check_in_org(self, github_user) -> bool: - for org in github_user.get_orgs(): - if org.login == settings.GITHUB_ORG: - return True - - return False - - def health(self): - health = { - 'healthy': False, - 'valid_credentials': False - } - try: - client = github.Github(client_id=settings.GITHUB_CLIENT_ID, client_secret=settings.GITHUB_CLIENT_SECRET, - base_url=settings.GITHUB_URL) - remaining, limit = client.rate_limiting - health['valid_credentials'] = True - health['rate'] = { - 'limit': limit, - 'remaining': remaining - } - if remaining > 10: - health['healthy'] = True - except Exception: - self.logger.exception("Error getting auth driver health") - finally: - return health - - def find_roles(self, github_user): - roles = [] - - org = None - for org in github_user.get_orgs(): - if org.login == settings.GITHUB_ORG: - break - - for team in org.get_teams(): - - if team.has_in_members(NamedUser(None, [], {"login": github_user.login}, completed=True)) is False: - continue - - if team.name in settings.GITHUB_TEAM_ROLES: - roles.append(settings.GITHUB_TEAM_ROLES[team.name]) - continue - - if team.name.startswith(settings.GITHUB_TEAM_ROLES_PREFIX): - roles.append(team.name.replace(settings.GITHUB_TEAM_ROLES_PREFIX, "")) - continue - - return roles diff --git a/deli/counter/auth/drivers/github/router.py b/deli/counter/auth/drivers/github/router.py deleted file mode 100644 index 7877dac..0000000 --- a/deli/counter/auth/drivers/github/router.py +++ /dev/null @@ -1,97 +0,0 @@ -import cherrypy -import github -import github.AuthenticatedUser -import requests -import requests.exceptions -from github.GithubException import TwoFactorException, GithubException, BadCredentialsException -from ingredients_http.request_methods import RequestMethods -from ingredients_http.route import Route -from simple_settings import settings -from sqlalchemy_utils.types.json import json - -from deli.counter.auth.token import Token -from deli.counter.auth.validation_models.github import RequestGithubAuthorization, RequestGithubToken -from deli.counter.http.mounts.root.routes.v1.auth.validation_models.tokens import ResponseOAuthToken -from deli.counter.http.router import SandwichRouter - - -class GithubAuthRouter(SandwichRouter): - def __init__(self, driver): - super().__init__(uri_base='github') - self.driver = driver - - def generate_token(self, token_github_client): - github_user = token_github_client.get_user() - if self.driver.check_in_org(github_user) is False: - raise cherrypy.HTTPError(403, "User not a member of GitHub organization: '" + settings.GITHUB_ORG + "'") - - token = Token() - token.driver_name = self.driver.name - token.username = github_user.login - token.set_global_roles(self.driver.find_roles(github_user)) - response = ResponseOAuthToken() - response.access_token = token.marshal(self.mount.fernet) - response.expiry = token.expires_at - return response - - @Route(route='authorization', methods=[RequestMethods.POST]) - @cherrypy.config(**{'tools.authentication.on': False}) - @cherrypy.tools.model_in(cls=RequestGithubAuthorization) - @cherrypy.tools.model_out(cls=ResponseOAuthToken) - def authorization(self): # Used to get token via API (username and password Auth Flow) - request: RequestGithubAuthorization = cherrypy.request.model - - user_github_client = github.Github(request.username, request.password, base_url=settings.GITHUB_URL) - github_user: github.AuthenticatedUser.AuthenticatedUser = user_github_client.get_user() - - try: - authorization = github_user.create_authorization( - scopes=['user:email', 'read:org'], - note='Sandwich Cloud Authorization', - client_id=settings.GITHUB_CLIENT_ID, - client_secret=settings.GITHUB_CLIENT_SECRET, - onetime_password=request.otp_code - ) - except TwoFactorException: - cherrypy.response.headers['X-GitHub-OTP'] = '2fa' - raise cherrypy.HTTPError(401, "OTP Code Required") - except BadCredentialsException: - raise cherrypy.HTTPError(404, "Invalid credentials") - except GithubException as e: - self.logger.exception("Error while validating GitHub authorization") - raise cherrypy.HTTPError(424, "Backend error while talking with GitHub: " + json.dumps(e.data)) - - return self.generate_token(github.Github(authorization.token, base_url=settings.GITHUB_URL)) - - @Route(route='token', methods=[RequestMethods.OPTIONS]) - @cherrypy.config(**{'tools.authentication.on': False}) - @cherrypy.tools.json_out() - def token_options(self): # pragma: no cover - # This is required for EmberJS for some reason? - return {} - - @Route(route='token', methods=[RequestMethods.POST]) - @cherrypy.config(**{'tools.authentication.on': False}) - @cherrypy.tools.model_in(cls=RequestGithubToken) - @cherrypy.tools.model_out(cls=ResponseOAuthToken) - def token(self): # Used to get token via Web UI (Authorization Code Auth Flow) - request: RequestGithubToken = cherrypy.request.model - - # TODO: how to get this url since the settings only has the api url? - r = requests.post('https://github.com/login/oauth/access_token', json={ - 'client_id': settings.GITHUB_CLIENT_ID, - 'client_secret': settings.GITHUB_CLIENT_SECRET, - 'code': request.authorizationCode - }, headers={'Accept': 'application/json'}) - - if r.status_code == 404: - raise cherrypy.HTTPError(404, "Unknown Authorization Code") - elif r.status_code != 200: - try: - r.raise_for_status() - except request.exceptions.RequestException as e: - self.logger.exception("Error while validating GitHub access token") - raise cherrypy.HTTPError(424, "Backend error while talking with GitHub: " + e.response.text) - - access_token_data = r.json() - return self.generate_token(github.Github(access_token_data['access_token'], base_url=settings.GITHUB_URL)) diff --git a/deli/counter/auth/manager.py b/deli/counter/auth/manager.py deleted file mode 100644 index cad6f87..0000000 --- a/deli/counter/auth/manager.py +++ /dev/null @@ -1,37 +0,0 @@ -import importlib -import logging - -from simple_settings import settings - -from deli.counter.auth.driver import AuthDriver - -DRIVERS = {} - - -def load_drivers(): - logger = logging.getLogger("%s.%s" % (load_drivers.__module__, load_drivers.__name__)) - for driver_string in settings.AUTH_DRIVERS: - if ':' not in driver_string: - raise ValueError("AUTH_DRIVER does not contain a module and class. " - "Must be in the following format: 'my.module:MyClass'") - - auth_module, auth_class, *_ = driver_string.split(":") - try: - auth_module = importlib.import_module(auth_module) - except ImportError: - logger.exception("Could not import auth driver's module: " + auth_module) - raise - try: - driver_klass = getattr(auth_module, auth_class) - except AttributeError: - logger.exception("Could not get driver's module class: " + auth_class) - raise - - if not issubclass(driver_klass, AuthDriver): - raise ValueError("AUTH_DRIVER class is not a subclass of '" + AuthDriver.__module__ + ".AuthDriver'") - - driver: AuthDriver = driver_klass() - DRIVERS[driver.name] = driver - - if len(DRIVERS) == 0: - raise ValueError("No auth drivers loaded") diff --git a/deli/counter/auth/policy.py b/deli/counter/auth/policy.py index 97f038c..4d5b5ba 100644 --- a/deli/counter/auth/policy.py +++ b/deli/counter/auth/policy.py @@ -1,108 +1,39 @@ -POLICIES = [ - # Policies - { - "name": "policies:get", - "description": "Ability to get a policy", - - }, - { - "name": "policies:list", - "description": "Ability to list policies", - }, - +SYSTEM_POLICIES = [ # Roles { - "name": "roles:global:create", - "description": "Ability to create a global role" + "name": "roles:system:create", + "description": "Ability to create a system role" }, { - "name": "roles:project:create", - "description": "Ability to create a project role", - "tags": [ - "project", - "default_project_member" - ] + "name": "roles:system:get", + "description": "Ability to get a system role", }, { - "name": "roles:global:get", - "description": "Ability to get a global role", + "name": "roles:system:list", + "description": "Ability to list system roles", }, { - "name": "roles:project:get", - "description": "Ability to get a project role", - "tags": [ - "project", - "default_project_member" - ] - }, - { - "name": "roles:global:list", - "description": "Ability to list global roles", + "name": "roles:system:update", + "description": "Ability to update a system role", }, { - "name": "roles:project:list", - "description": "Ability to list project roles", - "tags": [ - "project", - "default_project_member" - ] + "name": "roles:system:delete", + "description": "Ability to delete a system role" }, - { - "name": "roles:global:update", - "description": "Ability to update a global role", - }, - { - "name": "roles:project:update", - "description": "Ability to update a project role", - "tags": [ - "project", - "default_project_member" - ] - }, - { - "name": "roles:global:delete", - "description": "Ability to delete a global role" - }, - { - "name": "roles:project:delete", - "description": "Ability to delete a project role", - "tags": [ - "project", - "default_project_member" - ] - }, - # Flavors { "name": "flavors:create", "description": "Ability to create a flavor", }, - { - "name": "flavors:get", - "description": "Ability to get a flavor", - }, - { - "name": "flavors:list", - "description": "Ability to list flavors", - }, { "name": "flavors:delete", "description": "Ability to delete a flavor", }, - # Regions { "name": "regions:create", "description": "Ability to create a region" }, - { - "name": "regions:get", - "description": "Ability to get a region", - }, - { - "name": "regions:list", - "description": "Ability to list regions", - }, { "name": "regions:delete", "description": "Ability to delete a region" @@ -111,20 +42,11 @@ "name": "regions:action:schedule", "description": "Ability to change the schedule mode of the region" }, - # Zones { "name": "zones:create", "description": "Ability to create a zone" }, - { - "name": "zones:get", - "description": "Ability to get a zone", - }, - { - "name": "zones:list", - "description": "Ability to list zones", - }, { "name": "zones:delete", "description": "Ability to delete a zone" @@ -133,394 +55,308 @@ "name": "zones:action:schedule", "description": "Ability to change the schedule mode of the zone" }, - + # Networks + { + "name": "networks:create", + "description": "Ability to create a network" + }, + { + "name": "networks:delete", + "description": "Ability to delete a network" + }, # Projects { "name": "projects:create", "description": "Ability to create a project" }, { - "name": "projects:get", - "description": "Ability to get a project", + "name": "projects:quota:modify", + "description": "Ability to modify a project's quota", }, + # Service Accounts { - "name": "projects:get:all", - "description": "Ability to get all projects" + "name": "service_accounts:system:create", + "description": "Ability to create a system service account" }, { - "name": "projects:list", - "description": "Ability to list projects", + "name": "service_accounts:system:get", + "description": "Ability to get a system service account" }, { - "name": "projects:list:all", - "description": "Ability to list all projects" + "name": "service_accounts:system:list", + "description": "Ability to list system service accounts" }, { - "name": "projects:delete", - "description": "Ability to delete a project" + "name": "service_accounts:system:update", + "description": "Ability to update a system service account" }, + { - "name": "projects:scope", - "description": "Ability to scope to projects" + "name": "service_accounts:system:delete", + "description": "Ability to delete a system service account" }, { - "name": "projects:scope:all", - "description": "Ability to scope to all projects" + "name": "service_accounts:system:key:create", + "description": "Ability to create keys for system service accounts" }, { - "name": "projects:members:add", - "description": "Ability to add a member to a project", + "name": "service_accounts:system:key:delete", + "description": "Ability to delete keys from system service accounts" + }, +] + +PROJECT_POLICIES = [ + # Project + { + "name": "projects:get", + "description": "Ability to get a project", "tags": [ - "project", - "default_project_member" + 'viewer', + 'editor' ] }, { - "name": "projects:members:get", - "description": "Ability to get a member in a project", + "name": "projects:delete", + "description": "Ability to delete a project" + }, + { + "name": "projects:quota:get", + "description": "Ability to get a project's quota", "tags": [ - "project", - "default_project_member" + 'viewer', + 'editor' ] }, + # Roles { - "name": "projects:members:list", - "description": "Ability to list members in a project", + "name": "roles:project:create", + "description": "Ability to create a project role", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'editor' ] }, { - "name": "projects:members:modify", - "description": "Ability to modify a project member's roles", + "name": "roles:project:get", + "description": "Ability to get a project role", "tags": [ - "project", - "default_project_member" + 'viewer', + 'editor' ] }, { - "name": "projects:members:remove", - "description": "Ability to remove a member from a project", + "name": "roles:project:list", + "description": "Ability to list project roles", "tags": [ - "project", - "default_project_member" + 'viewer', + 'editor' ] }, { - "name": "projects:quota:get", - "description": "Ability to get a project's quota", + "name": "roles:project:update", + "description": "Ability to update a project role", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'editor' ] }, { - "name": "projects:quota:modify", - "description": "Ability to modify a project's quota", + "name": "roles:project:delete", + "description": "Ability to delete a project role", + "tags": [ + 'editor' + ] }, # Volumes { "name": "volumes:create", "description": "Ability to create a volume", "tags": [ - "project", - "default_project_member", + 'editor' ] }, { "name": "volumes:get", "description": "Ability to get a volume", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'viewer', + 'editor' ] }, { "name": "volumes:list", "description": "Ability to list volumes", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'viewer', + 'editor' ] }, { "name": "volumes:delete", "description": "Ability to delete a volume", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "volumes:action:attach", "description": "Ability to attach a volume to an instance", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "volumes:action:detach", "description": "Ability to detach a volume from an instance", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "volumes:action:grow", "description": "Ability to grow a volume", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "volumes:action:clone", "description": "Ability to clone a volume", "tags": [ - "project", - "default_project_member" + 'editor' ] }, - # Images { "name": "images:create", - "description": "Ability to create an image" + "description": "Ability to create an image", + "tags": [ + 'editor' + ] }, { "name": "images:get", "description": "Ability to get an image", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'viewer', + 'editor' ] }, { "name": "images:list", "description": "Ability to list images", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'viewer', + 'editor' ] }, { "name": "images:delete", "description": "Ability to delete an image", "tags": [ - "project", - "default_project_member" + 'editor' ] }, - { - "name": "images:action:visibility", - "description": "Ability to change the image visibility", - "tags": [ - "project", - "default_project_member" - ] - }, - { - "name": "images:action:visibility:public", - "description": "Ability to change the image visibility to public" - }, { "name": "images:action:lock", "description": "Ability to lock an image", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "images:action:unlock", "description": "Ability to unlock an image", "tags": [ - "project", - "default_project_member" - ] - }, - { - "name": "images:members:add", - "description": "Ability to add a member to an image", - "tags": [ - "project", - "default_project_member" - ] - }, - { - "name": "images:members:list", - "description": "Ability to list image members", - "tags": [ - "project", - "default_project_member" - ] - }, - { - "name": "images:members:delete", - "description": "Ability to delete a member from an image", - "tags": [ - "project", - "default_project_member" + 'editor' ] }, - # Instances { "name": "instances:create", "description": "Ability to create an instance", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "instances:get", "description": "Ability to get an instance", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'viewer', + 'editor' ] }, { "name": "instances:list", "description": "Ability to list instances", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'viewer', + 'editor' ] }, { "name": "instances:delete", "description": "Ability to delete an instance", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "instances:action:stop", "description": "Ability to stop an instance", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "instances:action:start", "description": "Ability to start an instance", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "instances:action:restart", "description": "Ability to restart an instance", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "instances:action:image", "description": "Ability to create an image from an instance", "tags": [ - "project", - "default_project_member" + 'editor' ] }, - - # Networks - { - "name": "networks:create", - "description": "Ability to create a network" - }, - { - "name": "networks:get", - "description": "Ability to get a network", - }, - { - "name": "networks:list", - "description": "Ability to list networks", - }, - { - "name": "networks:delete", - "description": "Ability to delete a network" - }, - # Service Accounts - { - "name": "service_accounts:global:create", - "description": "Ability to create a global service account" - }, - { - "name": "service_accounts:global:get", - "description": "Ability to get a global service account" - }, - { - "name": "service_accounts:global:list", - "description": "Ability to list global service accounts" - }, - { - "name": "service_accounts:global:update", - "description": "Ability to update a global service account" - }, - - { - "name": "service_accounts:global:delete", - "description": "Ability to delete a global service account" - }, - { - "name": "service_accounts:global:key:create", - "description": "Ability to create keys for global service accounts" - }, - { - "name": "service_accounts:global:key:delete", - "description": "Ability to delete keys from global service accounts" - }, { "name": "service_accounts:project:create", "description": "Ability to create a project service account", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "service_accounts:project:get", "description": "Ability to get a project service account", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'viewer', + 'editor' ] }, { "name": "service_accounts:project:list", "description": "Ability to list project service accounts", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'viewer', + 'editor' ] }, { "name": "service_accounts:project:update", "description": "Ability to update a project service account", "tags": [ - "project", - "default_project_member" + 'editor' ] }, @@ -528,113 +364,76 @@ "name": "service_accounts:project:delete", "description": "Ability to delete a project service account", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "service_accounts:project:key:create", "description": "Ability to create keys for project service accounts", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "service_accounts:project:key:delete", "description": "Ability to delete keys from project service accounts", "tags": [ - "project", - "default_project_member" + 'editor' ] }, - # Keypairs { "name": "keypairs:create", "description": "Ability to create a keypair", "tags": [ - "project", - "default_project_member" + 'editor' ] }, { "name": "keypairs:get", "description": "Ability to get a keypair", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'viewer', + 'editor' ] }, { "name": "keypairs:list", "description": "Ability to list keypairs", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'viewer', + 'editor' ] }, { "name": "keypairs:delete", "description": "Ability to delete a keypair", "tags": [ - "project", - "default_project_member" + 'editor' ] }, - # Network Ports { "name": "network_ports:get", "description": "Ability to get a network port", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'viewer', + 'editor' ] }, { "name": "network_ports:list", "description": "Ability to list network ports", "tags": [ - "project", - "default_project_member", - "default_service_account" + 'viewer', + 'editor' ] }, { "name": "network_ports:delete", "description": "Ability to delete a network port", "tags": [ - "project", - "default_project_member" + 'editor' ] }, - - # Database Users - { - "name": "database:users:create", - "description": "Ability to create users", - }, - { - "name": "database:users:get", - "description": "Ability to get a user" - }, - { - "name": "database:users:list", - "description": "Ability to list users" - }, - { - "name": "database:users:delete", - "description": "Ability to delete a user" - }, - { - "name": "database:users:password", - "description": "Ability to change a user's password" - }, - { - "name": "database:users:roles:update", - "description": "Ability to update a user's roles" - }] +] diff --git a/deli/counter/auth/token.py b/deli/counter/auth/token.py index 569bc64..d7f0711 100644 --- a/deli/counter/auth/token.py +++ b/deli/counter/auth/token.py @@ -1,128 +1,221 @@ import json -from typing import Optional +import logging +from typing import List import arrow import cherrypy +import jose +import requests from cryptography.fernet import InvalidToken +from jose import jwt +from simple_settings import settings -from deli.counter.auth import manager -from deli.counter.auth.driver import AuthDriver +from deli.counter.auth.policy import SYSTEM_POLICIES from deli.kubernetes.resources.project import Project -from deli.kubernetes.resources.v1alpha1.role.model import GlobalRole, ProjectRole -from deli.kubernetes.resources.v1alpha1.service_account.model import GlobalServiceAccount, ProjectServiceAccount +from deli.kubernetes.resources.v1alpha1.iam_group.model import IAMSystemGroup +from deli.kubernetes.resources.v1alpha1.iam_policy.model import IAMPolicy +from deli.kubernetes.resources.v1alpha1.iam_role.model import IAMSystemRole, IAMProjectRole +from deli.kubernetes.resources.v1alpha1.iam_service_account.model import SystemServiceAccount, ProjectServiceAccount +from deli.kubernetes.resources.v1alpha1.instance.model import Instance class Token(object): def __init__(self): - self.expires_at = arrow.now().shift(days=+1) - self.driver_name = None - self.project_id = None - self.global_role_ids = [] - self.project_role_ids = [] - self.username = None - self.service_account_id = None - self.service_account_key = None + self.email = None + self.service_account = None + self.metadata = {} # Extra metadata the token contains, currently only used for service accounts + self.expires_at = arrow.now('UTC').shift(days=+1) + + self.system_roles = [] + self.oauth_groups = [] + + self.logger = logging.getLogger("%s.%s" % (self.__module__, self.__class__.__name__)) + + def get_oauth_rsa_key(self, unverified_header): + r = requests.get(settings.OPENID_ISSUER_URL + ".well-known/openid-configuration") + if r.status_code != 200: + try: + r.raise_for_status() + except requests.exceptions.RequestException as e: + self.logger.exception("Backend error while discovering OAuth configuration from provider") + raise cherrypy.HTTPError(424, + "Backend error while discovering OAuth configuration from provider: " + + e.response.text) + + well_known_data = r.json() + r = requests.get(well_known_data['jwks_uri']) + if r.status_code != 200: + try: + r.raise_for_status() + except requests.exceptions.RequestException as e: + self.logger.exception("Backend error while discovering OAuth keys") + raise cherrypy.HTTPError(424, + "Backend error while discovering OAuth keys from provider: " + e.response.text) + + rsa_key = {} + for key in r.json()["keys"]: + if key["kid"] == unverified_header["kid"]: + rsa_key = { + "kty": key["kty"], + "kid": key["kid"], + "use": key["use"], + "n": key["n"], + "e": key["e"] + } + break + + if len(rsa_key) == 0: + # Header has a invalid kid + raise cherrypy.HTTPError(401, 'Invalid Authorization Token.') + + return rsa_key @classmethod def unmarshal(cls, token_string, fernet): + token = cls() + try: token_data_bytes = fernet.decrypt(token_string.encode()) + token_json = json.loads(token_data_bytes.decode()) + token.expires_at = arrow.get(token_json['expires_at']) if token_json['expires_at'] is not None else None + if token.expires_at is not None and token.expires_at <= arrow.now('UTC'): + raise cherrypy.HTTPError(401, 'Invalid Authorization Token.') + token.metadata = token_json['metadata'] + token.email = token_json['email'] except InvalidToken: - raise cherrypy.HTTPError(401, 'Invalid authorization token.') - - token_json = json.loads(token_data_bytes.decode()) - token = cls() - token.expires_at = arrow.get(token_json['expires_at']) - token.driver_name = token_json['driver'] - token.project_id = token_json['project'] - token.global_role_ids = token_json['global_role_ids'] - token.project_role_ids = token_json['project_role_ids'] - token.username = token_json['username'] - token.service_account_id = token_json['service_account_id'] - token.service_account_key = token_json['service_account_key'] - - if token.expires_at is not None and token.expires_at <= arrow.now(): - # Token is expired so it is invalid - # Tokens with expires_at of None are static keys - raise cherrypy.HTTPError(401, 'Invalid Authorization Token.') + try: + token_payload = jwt.decode(token_string, + token.get_oauth_rsa_key(jwt.get_unverified_header(token_string)), + algorithms=['RS256'], audience=settings.OPENID_CLIENT_ID, + issuer=settings.OPENID_ISSUER_URL) + token.expires_at = arrow.get(token_payload['exp']) + token.email = token_payload[settings.OPENID_EMAIL_CLAIM] + token.oauth_groups = token_payload[settings.OPENID_GROUPS_CLAIM] + except jose.JOSEError: + # Unable to decode jwt + raise cherrypy.HTTPError(401, 'Invalid Authorization Token.') - if token.driver_name != "metadata" and token.driver() is None: - raise cherrypy.HTTPError(500, - "Auth driver '%s' is not loaded, cannot validate token." % token.driver_name) + username, domain, *_ = token.email.split("@") + if domain.endswith('sandwich.local'): + type, project_name, *_ = domain.split('.') + system = True if project_name == 'system' else False + project = None + if system is False: + project = Project.get(project_name) + if project is None: + # Email domain contains invalid project + raise cherrypy.HTTPError(401, 'Invalid Authorization Token.') - project = token.project() - if token.project_id is not None: - if project is None: - raise cherrypy.HTTPError(400, 'Current scoped project does not exist.') + if type == 'service-account': + if system: + service_account = SystemServiceAccount.get(username) + else: + service_account = ProjectServiceAccount.get(project, username) - if token.service_account_id is not None: - if project is not None: - service_account = ProjectServiceAccount.get(project, token.service_account_id) if service_account is None: - service_account = GlobalServiceAccount.get(token.service_account_id) - else: - service_account = GlobalServiceAccount.get(token.service_account_id) - - if service_account is None: - raise cherrypy.HTTPError(401, 'Invalid Authorization Token.') - - if token.service_account_key: - if token.service_account_key not in service_account.keys: raise cherrypy.HTTPError(401, 'Invalid Authorization Token.') - # Key is set which means token is static - # Static keys do not have roles set in the key so we set them here - role_ids = service_account.role_ids - if project is None: - token.global_role_ids = role_ids + if token.metadata['key'] not in service_account.keys: + if 'instance' in token.metadata: + if Instance.get(project, token.metadata['instance']) is None: + # Token says it's from an instance but we can't find it + raise cherrypy.HTTPError(401, 'Invalid Authorization Token.') + else: + # Token says it's a service account key but it doesn't exist + raise cherrypy.HTTPError(401, 'Invalid Authorization Token.') else: - token.project_role_ids = role_ids + expire_at = service_account.keys[token.metadata['key']] + if expire_at <= arrow.now('UTC'): + # Service account key is expired + raise cherrypy.HTTPError(401, 'Invalid Authorization Token.') + + token.service_account = service_account + else: + # Invalid email type + raise cherrypy.HTTPError(401, 'Invalid Authorization Token.') + + system_policy = IAMPolicy.get("system") + token.system_roles = token.find_roles(system_policy) return token def marshal(self, fernet): token_data = { + 'email': self.email, + 'metadata': self.metadata, 'expires_at': self.expires_at, - 'driver': self.driver_name, - 'project': self.project_id, - 'global_role_ids': self.global_role_ids, - 'project_role_ids': self.project_role_ids, - 'username': self.username, - 'service_account_id': self.service_account_id, - 'service_account_key': self.service_account_key } return fernet.encrypt(json.dumps(token_data).encode()) - def driver(self) -> Optional[AuthDriver]: - return manager.DRIVERS.get(self.driver_name) - - def project(self) -> Optional[Project]: - return Project.get(self.project_id) - - def set_global_roles(self, role_names): - self.global_role_ids = [] - for role_name in role_names: - role = GlobalRole.get_by_name(role_name) - if role is not None: - self.global_role_ids.append(role.id) + @property + def identity(self): + if self.service_account is not None: + check_email = "serviceAccount:" + self.email + else: + check_email = "user:" + self.email + + return check_email + + def find_roles(self, policy: IAMPolicy) -> List[str]: + roles = [] + + for binding in policy.bindings: + role_name = binding['role'] + members = binding['members'] + + # Check if user is in the members list + if self.identity in members: + roles.append(role_name) + + # If the user has oauth groups + # check for groups in the members listing + # and see if the user is part of them + if len(self.oauth_groups) > 0: + for member in members: + if member.endswith("group.system.sandwich.local"): + group_name = member.split("@")[0] + iam_group: IAMSystemGroup = IAMSystemGroup.get(group_name) + if iam_group.oauth_link in self.oauth_groups: + roles.append(role_name) + continue + + return roles + + def get_projects(self) -> List[Project]: + projects = [] + policies = IAMPolicy.list() + for policy in policies: + if policy.name == "system": + continue + if len(self.find_roles(policy)) > 0: + project = Project.get(policy.name) + if project is not None: + projects.append(project) + + return projects + + def enforce_policy(self, policy, project=None): + if len(self.system_roles) > 0: + if policy in [p['name'] for p in SYSTEM_POLICIES]: + for role_name in self.system_roles: + role = IAMSystemRole.get(role_name) + if role is not None and policy in role.policies: + return + raise cherrypy.HTTPError(403, "Insufficient permissions (%s) to perform the requested action." % policy) - def enforce_policy(self, policy): - project = self.project() if project is not None: - for role_id in self.project_role_ids: - role = ProjectRole.get(project, role_id) - if role is None: - continue - if policy in role.policies: + project_policy = IAMPolicy.get(project.name) + if project_policy is None: + raise cherrypy.HTTPError(500, "Could not find iam policy document for project %s" % project.name) + project_roles = self.find_roles(project_policy) + for role_name in project_roles: + role = IAMProjectRole.get(project, role_name) + if role is not None and policy in role.policies: return - for role_id in self.global_role_ids: - role = GlobalRole.get(role_id) - if role is None: - continue - if policy in role.policies: - return + raise cherrypy.HTTPError(403, "Insufficient permissions (%s) to perform the " + "requested action in the project %s." % (policy, project.name)) - raise cherrypy.HTTPError(403, "Insufficient permissions to perform the requested action.") + raise cherrypy.HTTPError(403, "Insufficient permissions (%s) to perform the requested action." % policy) diff --git a/deli/counter/auth/validation_models/github.py b/deli/counter/auth/validation_models/github.py deleted file mode 100644 index ce3a4b0..0000000 --- a/deli/counter/auth/validation_models/github.py +++ /dev/null @@ -1,12 +0,0 @@ -from schematics import Model -from schematics.types import StringType - - -class RequestGithubToken(Model): - authorizationCode = StringType(required=True) - - -class RequestGithubAuthorization(Model): - username = StringType(required=True) - password = StringType(required=True) - otp_code = StringType() diff --git a/deli/counter/cli/commands/gen_admin.py b/deli/counter/cli/commands/gen_admin.py index 2c7a8cf..a6c8b93 100644 --- a/deli/counter/cli/commands/gen_admin.py +++ b/deli/counter/cli/commands/gen_admin.py @@ -8,8 +8,9 @@ from kubernetes.client import Configuration from simple_settings import settings +from deli.cache import cache_client from deli.counter.auth.token import Token -from deli.kubernetes.resources.v1alpha1.service_account.model import GlobalServiceAccount +from deli.kubernetes.resources.v1alpha1.iam_service_account.model import SystemServiceAccount class GenAdmin(Command): @@ -41,24 +42,23 @@ def json_encoder(self, o): # pragma: no cover else: config.load_incluster_config() + cache_client.connect(url=settings.REDIS_URL) return 0 def run(self, args) -> int: fernet = Fernet(settings.AUTH_FERNET_KEYS[0]) - service_account = GlobalServiceAccount.get_by_name('admin') + service_account = SystemServiceAccount.get('admin') if service_account is None: self.logger.error("Could not find admin service account. Is the manager running?") return 1 - key_name = str(uuid.uuid4()) - service_account.keys = [key_name] + service_account.keys = {"admin": arrow.now('UTC').shift(years=+10)} service_account.save() token = Token() - token.driver_name = 'metadata' - token.service_account_id = service_account.id - token.service_account_key = key_name + token.email = service_account.email + token.metadata['key'] = 'admin' self.logger.info("Old admin keys are now invalid.") self.logger.info("Admin Key: " + token.marshal(fernet).decode()) diff --git a/deli/counter/http/app.py b/deli/counter/http/app.py index d9062d1..8fb0f52 100644 --- a/deli/counter/http/app.py +++ b/deli/counter/http/app.py @@ -1,5 +1,19 @@ +import datetime +import json + from ingredients_http.app import HTTPApplication class Application(HTTPApplication): - pass + + def setup(self): + super().setup() + old_json_encoder = json.JSONEncoder.default + + def json_encoder(self, o): # pragma: no cover + if isinstance(o, datetime.datetime): + return str(o.isoformat()) + + return old_json_encoder(self, o) + + json.JSONEncoder.default = json_encoder diff --git a/deli/counter/auth/drivers/__init__.py b/deli/counter/http/mounts/root/errors/__init__.py similarity index 100% rename from deli/counter/auth/drivers/__init__.py rename to deli/counter/http/mounts/root/errors/__init__.py diff --git a/deli/counter/http/mounts/root/routes/v1/errors/quota.py b/deli/counter/http/mounts/root/errors/quota.py similarity index 100% rename from deli/counter/http/mounts/root/routes/v1/errors/quota.py rename to deli/counter/http/mounts/root/errors/quota.py diff --git a/deli/counter/http/mounts/root/mount.py b/deli/counter/http/mounts/root/mount.py index 1179aaf..a6c4431 100644 --- a/deli/counter/http/mounts/root/mount.py +++ b/deli/counter/http/mounts/root/mount.py @@ -1,49 +1,76 @@ import cherrypy +from apispec import APISpec from cryptography.fernet import Fernet, MultiFernet from ingredients_http.app import HTTPApplication from ingredients_http.app_mount import ApplicationMount from kubernetes import config from kubernetes.client import Configuration +from pbr.version import VersionInfo from simple_settings import settings -from deli.counter.auth.manager import load_drivers +from deli.cache import cache_client from deli.counter.auth.token import Token from deli.kubernetes.resources.model import ProjectResourceModel +from deli.kubernetes.resources.project import Project class RootMount(ApplicationMount): def __init__(self, app: HTTPApplication): super().__init__(app=app, mount_point='/') self.fernet = MultiFernet([Fernet(key) for key in settings.AUTH_FERNET_KEYS]) + self.api_spec = APISpec( + title='Sandwich Cloud API', + openapi_version='3.0.0', + version=VersionInfo('sandwichcloud-deli').semantic_version().release_string(), + plugins=[ + "deli.counter.http.spec.plugins.docstring", + ] + ) def validate_token(self): authorization_header = cherrypy.request.headers.get('Authorization', None) if authorization_header is None: raise cherrypy.HTTPError(400, 'Missing Authorization header.') - method, fernet_token, *_ = authorization_header.split(" ") + method, auth_token, *_ = authorization_header.split(" ") if method != 'Bearer': raise cherrypy.HTTPError(400, 'Only Bearer tokens are allowed.') - token = Token.unmarshal(fernet_token, self.fernet) + token = Token.unmarshal(auth_token, self.fernet) cherrypy.request.token = token - cherrypy.request.project = token.project() + cherrypy.request.login = token.email # This sets the userID field in cherrypy logs - def validate_project_scope(self): - if cherrypy.request.project is None: - raise cherrypy.HTTPError(403, "Token not scoped for a project") + if 'instance' in token.metadata: + # If the token is from an instance include the instance name + # The email will contain the project + cherrypy.request.login = token.email + '/' + token.metadata['instance'] def enforce_policy(self, policy_name): - cherrypy.request.token.enforce_policy(policy_name) + project = None + if hasattr(cherrypy.request, 'project'): + project = cherrypy.request.project + cherrypy.request.token.enforce_policy(policy_name, project=project) + + def validate_project_scope(self, delete_param=False): + if 'project_name' in cherrypy.request.params: + project_name = cherrypy.request.params['project_name'] + if delete_param: + del cherrypy.request.params['project_name'] + project = Project.get(project_name) + cherrypy.request.project = project + if project is None: + raise cherrypy.HTTPError(404, "The project (%s) could not be found." % project_name) + else: + raise cherrypy.HTTPError(500, "Could not infer project from resource URL") def resource_object(self, id_param, cls): - resource_id = cherrypy.request.params[id_param] + resource_name = str(cherrypy.request.params[id_param]) if issubclass(cls, ProjectResourceModel): - resource = cls.get(cherrypy.request.project, resource_id) + resource = cls.get(cherrypy.request.project, resource_name) else: - resource = cls.get(resource_id) + resource = cls.get(resource_name) if resource is None: raise cherrypy.HTTPError(404, "The resource could not be found.") @@ -54,11 +81,8 @@ def __setup_tools(self): cherrypy.tools.authentication = cherrypy.Tool('on_start_resource', self.validate_token, priority=20) cherrypy.tools.project_scope = cherrypy.Tool('on_start_resource', self.validate_project_scope, priority=30) - cherrypy.tools.enforce_policy = cherrypy.Tool('before_request_body', self.enforce_policy, priority=40) - cherrypy.tools.resource_object = cherrypy.Tool('before_request_body', self.resource_object, priority=50) - - def __setup_auth(self): - load_drivers() + cherrypy.tools.resource_object = cherrypy.Tool('before_request_body', self.resource_object, priority=40) + cherrypy.tools.enforce_policy = cherrypy.Tool('before_request_body', self.enforce_policy, priority=50) def __setup_kubernetes(self): if settings.KUBE_CONFIG is not None or settings.KUBE_MASTER is not None: @@ -70,10 +94,13 @@ def __setup_kubernetes(self): else: config.load_incluster_config() + def __setup_redis(self): + cache_client.connect(url=settings.REDIS_URL) + def setup(self): self.__setup_tools() - self.__setup_auth() self.__setup_kubernetes() + self.__setup_redis() super().setup() def mount_config(self): diff --git a/deli/counter/auth/drivers/github/__init__.py b/deli/counter/http/mounts/root/routes/auth/__init__.py similarity index 100% rename from deli/counter/auth/drivers/github/__init__.py rename to deli/counter/http/mounts/root/routes/auth/__init__.py diff --git a/deli/counter/auth/validation_models/__init__.py b/deli/counter/http/mounts/root/routes/auth/v1/__init__.py similarity index 100% rename from deli/counter/auth/validation_models/__init__.py rename to deli/counter/http/mounts/root/routes/auth/v1/__init__.py diff --git a/deli/counter/http/mounts/root/routes/auth/v1/oauth.py b/deli/counter/http/mounts/root/routes/auth/v1/oauth.py new file mode 100644 index 0000000..eba9925 --- /dev/null +++ b/deli/counter/http/mounts/root/routes/auth/v1/oauth.py @@ -0,0 +1,71 @@ +import arrow +import cherrypy +import requests +from ingredients_http.request_methods import RequestMethods +from ingredients_http.route import Route +from simple_settings import settings + +from deli.counter.http.mounts.root.routes.auth.v1.validation_models.oauth import ResponseOAuthToken, RequestOAuthToken +from deli.counter.http.router import SandwichRouter + + +class AuthRouter(SandwichRouter): + def __init__(self): + super().__init__(uri_base='oauth') + + def get_token_url(self): + r = requests.get(settings.OPENID_ISSUER_URL + ".well-known/openid-configuration") + if r.status_code != 200: + try: + r.raise_for_status() + except requests.exceptions.RequestException as e: + self.logger.exception("Backend error while discovering OAuth configuration from provider") + raise cherrypy.HTTPError(424, + "Backend error while discovering OAuth configuration from provider: " + + e.response.text) + + well_known_data = r.json() + return well_known_data['token_endpoint'] + + @Route(route='token', methods=[RequestMethods.POST]) + @cherrypy.config(**{'tools.authentication.on': False}) + @cherrypy.tools.model_in(cls=RequestOAuthToken) + @cherrypy.tools.model_out(cls=ResponseOAuthToken) + def token(self): # Used to get token via API (username and password Auth Flow) + """Auth to the API + --- + post: + description: Auth to the API + tags: + - auth + requestBody: + description: User credentials + responses: + 200: + description: An API Token + """ + request: RequestOAuthToken = cherrypy.request.model + + r = requests.post(self.get_token_url(), json={ + 'grant_type': 'password', + 'username': request.email, + 'password': request.password, + 'client_id': settings.OPENID_CLIENT_ID, + 'client_secret': settings.OPENID_CLIENT_SECRET, + 'scope': 'openid profile email' + }, headers={'Accept': 'application/json'}) + + if r.status_code == 403: + raise cherrypy.HTTPError(403, 'Wrong email or password') + elif r.status_code != 200: + try: + r.raise_for_status() + except requests.exceptions.RequestException as e: + self.logger.exception("Error while validating OAuth access token") + raise cherrypy.HTTPError(424, "Backend error while talking with OAuth Provider: " + e.response.text) + + token_json = r.json() + token_model = ResponseOAuthToken() + token_model.access_token = token_json['id_token'] + token_model.expiry = arrow.now('UTC').shift(seconds=+token_json['expires_in']) + return token_model diff --git a/deli/counter/http/mounts/root/routes/v1/__init__.py b/deli/counter/http/mounts/root/routes/auth/v1/validation_models/__init__.py similarity index 100% rename from deli/counter/http/mounts/root/routes/v1/__init__.py rename to deli/counter/http/mounts/root/routes/auth/v1/validation_models/__init__.py diff --git a/deli/counter/http/mounts/root/routes/auth/v1/validation_models/oauth.py b/deli/counter/http/mounts/root/routes/auth/v1/validation_models/oauth.py new file mode 100644 index 0000000..03b157b --- /dev/null +++ b/deli/counter/http/mounts/root/routes/auth/v1/validation_models/oauth.py @@ -0,0 +1,14 @@ +from ingredients_http.schematics.types import ArrowType +from schematics import Model +from schematics.types import EmailType, StringType + + +class RequestOAuthToken(Model): + email = EmailType(required=True) + password = StringType(min_length=1, required=True) + otp_code = StringType() + + +class ResponseOAuthToken(Model): + access_token = StringType(required=True) + expiry = ArrowType(required=True) diff --git a/deli/counter/http/mounts/root/routes/v1/auth/__init__.py b/deli/counter/http/mounts/root/routes/compute/__init__.py similarity index 100% rename from deli/counter/http/mounts/root/routes/v1/auth/__init__.py rename to deli/counter/http/mounts/root/routes/compute/__init__.py diff --git a/deli/counter/http/mounts/root/routes/v1/auth/validation_models/__init__.py b/deli/counter/http/mounts/root/routes/compute/v1/__init__.py similarity index 100% rename from deli/counter/http/mounts/root/routes/v1/auth/validation_models/__init__.py rename to deli/counter/http/mounts/root/routes/compute/v1/__init__.py diff --git a/deli/counter/http/mounts/root/routes/v1/flavor.py b/deli/counter/http/mounts/root/routes/compute/v1/flavor.py similarity index 57% rename from deli/counter/http/mounts/root/routes/v1/flavor.py rename to deli/counter/http/mounts/root/routes/compute/v1/flavor.py index 75db7d1..25541be 100644 --- a/deli/counter/http/mounts/root/routes/v1/flavor.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/flavor.py @@ -2,7 +2,8 @@ from ingredients_http.request_methods import RequestMethods from ingredients_http.route import Route -from deli.counter.http.mounts.root.routes.v1.validation_models.flavor import RequestCreateFlavor, ResponseFlavor, \ +from deli.counter.http.mounts.root.routes.compute.v1.validation_models.flavor import RequestCreateFlavor, \ + ResponseFlavor, \ ParamsFlavor, ParamsListFlavor from deli.counter.http.router import SandwichRouter from deli.kubernetes.resources.model import ResourceState @@ -18,9 +19,22 @@ def __init__(self): @cherrypy.tools.model_out(cls=ResponseFlavor) @cherrypy.tools.enforce_policy(policy_name="flavors:create") def create(self): + """Create a flavor + --- + post: + description: Create a flavor + tags: + - compute + - flavor + requestBody: + description: Flavor to Create + responses: + 200: + description: The created flavor + """ request: RequestCreateFlavor = cherrypy.request.model - flavor = Flavor.get_by_name(request.name) + flavor = Flavor.get(request.name) if flavor is not None: raise cherrypy.HTTPError(400, 'A flavor with the requested name already exists.') @@ -33,27 +47,57 @@ def create(self): return ResponseFlavor.from_database(flavor) - @Route(route='{flavor_id}') + @Route(route='{flavor_name}') @cherrypy.tools.model_params(cls=ParamsFlavor) @cherrypy.tools.model_out(cls=ResponseFlavor) - @cherrypy.tools.resource_object(id_param="flavor_id", cls=Flavor) - @cherrypy.tools.enforce_policy(policy_name="flavors:get") + @cherrypy.tools.resource_object(id_param="flavor_name", cls=Flavor) def get(self, **_): + """Get a flavor + --- + get: + description: Get a flavor + tags: + - compute + - flavor + responses: + 200: + description: The flavor + """ return ResponseFlavor.from_database(cherrypy.request.resource_object) @Route() - @cherrypy.tools.project_scope() @cherrypy.tools.model_params(cls=ParamsListFlavor) @cherrypy.tools.model_out_pagination(cls=ResponseFlavor) - @cherrypy.tools.enforce_policy(policy_name="flavors:list") def list(self, limit, marker): + """List flavors + --- + get: + description: List flavors + tags: + - compute + - flavor + responses: + 200: + description: List of flavors + """ return self.paginate(Flavor, ResponseFlavor, limit, marker) - @Route(route='{flavor_id}', methods=[RequestMethods.DELETE]) + @Route(route='{flavor_name}', methods=[RequestMethods.DELETE]) @cherrypy.tools.model_params(cls=ParamsFlavor) - @cherrypy.tools.resource_object(id_param="flavor_id", cls=Flavor) + @cherrypy.tools.resource_object(id_param="flavor_name", cls=Flavor) @cherrypy.tools.enforce_policy(policy_name="flavors:delete") def delete(self, **_): + """Delete a Flavor + --- + delete: + description: Delete a flavor + tags: + - compute + - flavor + responses: + 204: + description: Flavor deleted + """ cherrypy.response.status = 204 flavor: Flavor = cherrypy.request.resource_object diff --git a/deli/counter/http/mounts/root/routes/compute/v1/images.py b/deli/counter/http/mounts/root/routes/compute/v1/images.py new file mode 100644 index 0000000..770767b --- /dev/null +++ b/deli/counter/http/mounts/root/routes/compute/v1/images.py @@ -0,0 +1,146 @@ +import uuid + +import cherrypy +from ingredients_http.request_methods import RequestMethods +from ingredients_http.route import Route + +from deli.counter.http.mounts.root.routes.compute.v1.validation_models.images import RequestCreateImage, \ + ResponseImage, ParamsImage, ParamsListImage +from deli.counter.http.router import SandwichProjectRouter +from deli.kubernetes.resources.const import REGION_LABEL +from deli.kubernetes.resources.model import ResourceState +from deli.kubernetes.resources.project import Project +from deli.kubernetes.resources.v1alpha1.image.model import Image, ImageVisibility +from deli.kubernetes.resources.v1alpha1.region.model import Region + + +class ImageRouter(SandwichProjectRouter): + def __init__(self): + super().__init__(uri_base='images') + + @Route(methods=[RequestMethods.POST]) + @cherrypy.tools.model_in(cls=RequestCreateImage) + @cherrypy.tools.model_out(cls=ResponseImage) + @cherrypy.tools.enforce_policy(policy_name="images:create") + def create(self): + """Create an image + --- + post: + description: Create an image + tags: + - compute + - image + requestBody: + description: Image to create + responses: + 200: + description: The created image + """ + request: RequestCreateImage = cherrypy.request.model + project: Project = cherrypy.request.project + + image = Image.get(project, request.name) + if image is not None: + raise cherrypy.HTTPError(400, 'An image with the requested name already exists.') + + region = Region.get(request.region_name) + if region is None: + raise cherrypy.HTTPError(404, 'A region with the requested name does not exist.') + + if region.state != ResourceState.Created: + raise cherrypy.HTTPError(409, 'Can only create a image with a region in the following state: %s'.format( + ResourceState.Created)) + + # TODO: check duplicate file name + + image = Image() + image.name = request.name + image.file_name = request.file_name + image.project = project + image.region = region + image.create() + + return ResponseImage.from_database(image) + + @Route(route='{image_name}') + @cherrypy.tools.model_params(cls=ParamsImage) + @cherrypy.tools.model_out(cls=ResponseImage) + @cherrypy.tools.resource_object(id_param="image_name", cls=Image) + @cherrypy.tools.enforce_policy(policy_name="images:get") + def get(self, **_): + """Get an image + --- + get: + description: Get an image + tags: + - compute + - image + responses: + 200: + description: The image + """ + image: Image = cherrypy.request.resource_object + + return ResponseImage.from_database(image) + + @Route() + @cherrypy.tools.model_params(cls=ParamsListImage) + @cherrypy.tools.model_out_pagination(cls=ResponseImage) + @cherrypy.tools.enforce_policy(policy_name="images:list") + def list(self, region_name, visibility: ImageVisibility, limit: int, marker: uuid.UUID): + """List images + --- + get: + description: List images + tags: + - compute + - image + responses: + 200: + description: List of images + """ + kwargs = { + 'project': cherrypy.request.project, + 'label_selector': [] + } + + if region_name is not None: + region: Region = Region.get(region_name) + if region is None: + raise cherrypy.HTTPError(404, "A region with the requested name does not exist.") + + kwargs['label_selector'].append(REGION_LABEL + '=' + region.name) + + kwargs['label_selector'] = ",".join(kwargs['label_selector']) + + return self.paginate(Image, ResponseImage, limit, marker, **kwargs) + + @Route(route='{image_name}', methods=[RequestMethods.DELETE]) + @cherrypy.tools.model_params(cls=ParamsImage) + @cherrypy.tools.resource_object(id_param="image_name", cls=Image) + @cherrypy.tools.enforce_policy(policy_name="images:delete") + def delete(self, **_): + """Delete an image + --- + delete: + description: Delete an image + tags: + - compute + - image + responses: + 204: + description: Image deleted + """ + cherrypy.response.status = 204 + image: Image = cherrypy.request.resource_object + + if image.project_name != cherrypy.request.project.name: + raise cherrypy.HTTPError(404, "The resource could not be found.") + if image.state == ResourceState.ToDelete or image.state == ResourceState.Deleting: + raise cherrypy.HTTPError(400, "Image is already being deleting") + if image.state == ResourceState.Deleted: + raise cherrypy.HTTPError(400, "Image has already been deleted") + if image.state not in [ResourceState.Created, ResourceState.Error]: + raise cherrypy.HTTPError(400, 'Image cannot be deleted in the current state') + + image.delete() diff --git a/deli/counter/http/mounts/root/routes/v1/instance.py b/deli/counter/http/mounts/root/routes/compute/v1/instance.py similarity index 66% rename from deli/counter/http/mounts/root/routes/v1/instance.py rename to deli/counter/http/mounts/root/routes/compute/v1/instance.py index 1e8b24c..16c7381 100644 --- a/deli/counter/http/mounts/root/routes/v1/instance.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/instance.py @@ -4,108 +4,119 @@ from ingredients_http.request_methods import RequestMethods from ingredients_http.route import Route -from deli.counter.http.mounts.root.routes.v1.errors.quota import QuotaError -from deli.counter.http.mounts.root.routes.v1.validation_models.images import ResponseImage -from deli.counter.http.mounts.root.routes.v1.validation_models.instances import RequestCreateInstance, \ +from deli.counter.http.mounts.root.errors.quota import QuotaError +from deli.counter.http.mounts.root.routes.compute.v1.validation_models.images import ResponseImage +from deli.counter.http.mounts.root.routes.compute.v1.validation_models.instances import RequestCreateInstance, \ ResponseInstance, ParamsInstance, ParamsListInstance, RequestInstancePowerOffRestart, RequestInstanceImage -from deli.counter.http.router import SandwichRouter +from deli.counter.http.router import SandwichProjectRouter from deli.kubernetes.resources.const import REGION_LABEL, IMAGE_LABEL, ZONE_LABEL, ATTACHED_TO_LABEL from deli.kubernetes.resources.model import ResourceState from deli.kubernetes.resources.project import Project from deli.kubernetes.resources.v1alpha1.flavor.model import Flavor -from deli.kubernetes.resources.v1alpha1.image.model import Image, ImageVisibility +from deli.kubernetes.resources.v1alpha1.iam_service_account.model import ProjectServiceAccount +from deli.kubernetes.resources.v1alpha1.image.model import Image from deli.kubernetes.resources.v1alpha1.instance.model import Instance, VMPowerState from deli.kubernetes.resources.v1alpha1.keypair.keypair import Keypair from deli.kubernetes.resources.v1alpha1.network.model import NetworkPort, Network from deli.kubernetes.resources.v1alpha1.project_quota.model import ProjectQuota from deli.kubernetes.resources.v1alpha1.region.model import Region -from deli.kubernetes.resources.v1alpha1.service_account.model import ProjectServiceAccount from deli.kubernetes.resources.v1alpha1.volume.model import Volume from deli.kubernetes.resources.v1alpha1.zone.model import Zone -class InstanceRouter(SandwichRouter): +class InstanceRouter(SandwichProjectRouter): def __init__(self): super().__init__(uri_base='instances') @Route(methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() @cherrypy.tools.model_in(cls=RequestCreateInstance) @cherrypy.tools.model_out(cls=ResponseInstance) @cherrypy.tools.enforce_policy(policy_name="instances:create") def create(self): + """Create an instance + --- + post: + description: Create an instance + tags: + - compute + - instance + requestBody: + description: Instance to create + responses: + 200: + description: The created instance + """ request: RequestCreateInstance = cherrypy.request.model project: Project = cherrypy.request.project - # TODO: do we care about unique instance names in a project? - # instance = Instance.get_by_name(project, request.name) - # if instance is not None: - # raise cherrypy.HTTPError(409, 'An instance with the requested name already exists.') + instance = Instance.get(project, request.name) + if instance is not None: + raise cherrypy.HTTPError(409, 'An instance with the requested name already exists.') - region = Region.get(request.region_id) + region = Region.get(request.region_name) if region is None: - raise cherrypy.HTTPError(404, 'A region with the requested id does not exist.') + raise cherrypy.HTTPError(404, 'A region with the requested name does not exist.') if region.state != ResourceState.Created: raise cherrypy.HTTPError(400, 'Can only create a instance with a region in the following state: %s'.format( ResourceState.Created)) zone = None - if request.zone_id is not None: - zone = Zone.get(request.zone_id) + if request.zone_name is not None: + zone = Zone.get(request.zone_name) if zone is None: - raise cherrypy.HTTPError(404, 'A zone with the requested id does not exist.') - if zone.region.id != region.id: + raise cherrypy.HTTPError(404, 'A zone with the requested name does not exist.') + if zone.region.name != region.name: raise cherrypy.HTTPError(409, 'The requested zone is not within the requested region') if zone.state != ResourceState.Created: raise cherrypy.HTTPError(400, 'Can only create a instance with a zone in the following state: %s'.format( ResourceState.Created)) - network = Network.get(request.network_id) + network = Network.get(request.network_name) if network is None: - raise cherrypy.HTTPError(404, 'A network with the requested id does not exist.') - if network.region.id != region.id: + raise cherrypy.HTTPError(404, 'A network with the requested name does not exist.') + if network.region.name != region.name: raise cherrypy.HTTPError(409, 'The requested network is not within the requested region') if network.state != ResourceState.Created: raise cherrypy.HTTPError(400, 'Can only create a instance with a network in the following state: %s'.format( ResourceState.Created)) - image: Image = Image.get(request.image_id) + # TODO: User inputs image url instead of id + # projectId/imageId + image: Image = Image.get(project, request.image_name) if image is None: - raise cherrypy.HTTPError(404, 'An image with the requested id does not exist.') - if image.visibility == ImageVisibility.PRIVATE: - if image.project_id != project.id: - if image.is_member(project.id) is False: - raise cherrypy.HTTPError(404, 'An image with the requested id does not exist.') - if image.region.id != region.id: + raise cherrypy.HTTPError(404, 'An image with the requested name does not exist.') + if image.region.name != region.name: raise cherrypy.HTTPError(409, 'The requested image is not within the requested region') if image.state != ResourceState.Created: raise cherrypy.HTTPError(400, 'Can only create a instance with a image in the following state: %s'.format( ResourceState.Created)) - flavor: Flavor = Flavor.get(request.flavor_id) + flavor: Flavor = Flavor.get(request.flavor_name) if flavor is None: - raise cherrypy.HTTPError(404, 'A flavor with the requested id does not exist.') + raise cherrypy.HTTPError(404, 'A flavor with the requested name does not exist.') keypairs = [] - for keypair_id in request.keypair_ids: - keypair = Keypair.get(project, keypair_id) + for keypair_name in request.keypair_names: + keypair = Keypair.get(project, keypair_name) if keypair is None: raise cherrypy.HTTPError(404, - 'A keypair with the requested id of %s does not exist.'.format(keypair_id)) + 'A keypair with the requested name of %s does not exist.'.format(keypair_name)) keypairs.append(keypair) - if request.service_account_id is not None: - service_account = ProjectServiceAccount.get(project, request.service_account_id) + # TODO: User inputs service account email instead of id + # Only project service accounts are allowed + if request.service_account_name is not None: + service_account = ProjectServiceAccount.get(project, request.service_account_name) if service_account is None: - raise cherrypy.HTTPError(404, 'A service account with the requested id of %s does not exist.'.format( - request.service_account_id)) + raise cherrypy.HTTPError(404, 'A service account with the requested name of %s does not exist.'.format( + request.service_account_name)) else: - service_account = ProjectServiceAccount.get_by_name(project, 'default') + service_account = ProjectServiceAccount.get(project, 'default') if service_account is None: raise cherrypy.HTTPError(500, 'Could not find a default service account to attach to the instance.') - quota: ProjectQuota = ProjectQuota.get(project, project.id) + quota: ProjectQuota = ProjectQuota.get(project.name) used_vcpu = quota.used_vcpu + flavor.vcpus used_ram = quota.used_ram + flavor.ram requested_disk = flavor.disk @@ -131,6 +142,7 @@ def create(self): quota.save() network_port = NetworkPort() + network_port.name = str(uuid.uuid4()) # We don't care about the network port's name, just that it's unique network_port.project = project network_port.network = network network_port.create() @@ -164,43 +176,63 @@ def create(self): return ResponseInstance.from_database(instance) - @Route(route='{instance_id}') - @cherrypy.tools.project_scope() + @Route(route='{instance_name}') @cherrypy.tools.model_params(cls=ParamsInstance) @cherrypy.tools.model_out(cls=ResponseInstance) - @cherrypy.tools.resource_object(id_param="instance_id", cls=Instance) + @cherrypy.tools.resource_object(id_param="instance_name", cls=Instance) @cherrypy.tools.enforce_policy(policy_name="instances:get") def get(self, **_): + """Get an instance + --- + get: + description: Get an instance + tags: + - compute + - instance + responses: + 200: + description: The instance + """ return ResponseInstance.from_database(cherrypy.request.resource_object) @Route() - @cherrypy.tools.project_scope() @cherrypy.tools.model_params(cls=ParamsListInstance) @cherrypy.tools.model_out_pagination(cls=ResponseInstance) @cherrypy.tools.enforce_policy(policy_name="instances:list") - def list(self, image_id, region_id, zone_id, limit: int, marker: uuid.UUID): + def list(self, image_name, region_name, zone_name, limit: int, marker: uuid.UUID): + """List instances + --- + get: + description: List instances + tags: + - compute + - instance + responses: + 200: + description: List of instances + """ kwargs = { 'project': cherrypy.request.project, 'label_selector': [], } - if image_id is not None: - image: Image = Image.get(cherrypy.request.project, image_id) + if image_name is not None: + image: Image = Image.get(cherrypy.request.project, image_name) if image is None: - raise cherrypy.HTTPError(404, "An image with the requested id does not exist.") - kwargs['label_selector'].append(IMAGE_LABEL + '=' + image.id) + raise cherrypy.HTTPError(404, "An image with the requested name does not exist.") + kwargs['label_selector'].append(IMAGE_LABEL + '=' + image.name) - if region_id is not None: - region: Region = Region.get(region_id) + if region_name is not None: + region: Region = Region.get(region_name) if region is None: - raise cherrypy.HTTPError(404, "A region with the requested id does not exist.") - kwargs['label_selector'].append(REGION_LABEL + '=' + region.id) + raise cherrypy.HTTPError(404, "A region with the requested name does not exist.") + kwargs['label_selector'].append(REGION_LABEL + '=' + region.name) - if zone_id is not None: - zone: Zone = Zone.get(zone_id) + if zone_name is not None: + zone: Zone = Zone.get(zone_name) if zone is None: - raise cherrypy.HTTPError(404, 'A zone with the requested id does not exist.') - kwargs['label_selector'].append(ZONE_LABEL + '=' + zone.id) + raise cherrypy.HTTPError(404, 'A zone with the requested name does not exist.') + kwargs['label_selector'].append(ZONE_LABEL + '=' + zone.name) if len(kwargs['label_selector']) > 0: kwargs['label_selector'] = ",".join(kwargs['label_selector']) @@ -209,12 +241,22 @@ def list(self, image_id, region_id, zone_id, limit: int, marker: uuid.UUID): return self.paginate(Instance, ResponseInstance, limit, marker, **kwargs) - @Route(route='{instance_id}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.project_scope() + @Route(route='{instance_name}', methods=[RequestMethods.DELETE]) @cherrypy.tools.model_params(cls=ParamsInstance) - @cherrypy.tools.resource_object(id_param="instance_id", cls=Instance) + @cherrypy.tools.resource_object(id_param="instance_name", cls=Instance) @cherrypy.tools.enforce_policy(policy_name="instances:delete") def delete(self, **_): + """Delete an instance + --- + delete: + description: Delete an instance + tags: + - compute + - instance + responses: + 204: + description: Instance deleted + """ cherrypy.response.status = 204 instance: Instance = cherrypy.request.resource_object @@ -227,12 +269,22 @@ def delete(self, **_): instance.delete() - @Route(route='{instance_id}/action/start', methods=[RequestMethods.PUT]) - @cherrypy.tools.project_scope() + @Route(route='{instance_name}/action/start', methods=[RequestMethods.PUT]) @cherrypy.tools.model_params(cls=ParamsInstance) - @cherrypy.tools.resource_object(id_param="instance_id", cls=Instance) + @cherrypy.tools.resource_object(id_param="instance_name", cls=Instance) @cherrypy.tools.enforce_policy(policy_name="instances:action:stop") def action_start(self, **_): + """Start an instance + --- + put: + description: Start an instance + tags: + - compute + - instance + responses: + 204: + description: Instance starting + """ cherrypy.response.status = 202 instance: Instance = cherrypy.request.resource_object @@ -246,13 +298,25 @@ def action_start(self, **_): instance.action_start() - @Route(route='{instance_id}/action/stop', methods=[RequestMethods.PUT]) - @cherrypy.tools.project_scope() + @Route(route='{instance_name}/action/stop', methods=[RequestMethods.PUT]) @cherrypy.tools.model_params(cls=ParamsInstance) @cherrypy.tools.model_in(cls=RequestInstancePowerOffRestart) - @cherrypy.tools.resource_object(id_param="instance_id", cls=Instance) + @cherrypy.tools.resource_object(id_param="instance_name", cls=Instance) @cherrypy.tools.enforce_policy(policy_name="instances:action:start") def action_stop(self, **_): + """Stop an instance + --- + put: + description: Stop an instance + tags: + - compute + - instance + requestBody: + description: Stop options + responses: + 204: + description: Instance stopping + """ request: RequestInstancePowerOffRestart = cherrypy.request.model cherrypy.response.status = 202 @@ -267,13 +331,25 @@ def action_stop(self, **_): instance.action_stop(request.hard, request.timeout) - @Route(route='{instance_id}/action/restart', methods=[RequestMethods.PUT]) - @cherrypy.tools.project_scope() + @Route(route='{instance_name}/action/restart', methods=[RequestMethods.PUT]) @cherrypy.tools.model_params(cls=ParamsInstance) @cherrypy.tools.model_in(cls=RequestInstancePowerOffRestart) - @cherrypy.tools.resource_object(id_param="instance_id", cls=Instance) + @cherrypy.tools.resource_object(id_param="instance_name", cls=Instance) @cherrypy.tools.enforce_policy(policy_name="instances:action:restart") def action_restart(self, **_): + """Restart an instance + --- + put: + description: Restart an instance + tags: + - compute + - instance + requestBody: + description: Restart options + responses: + 204: + description: Instance restarting + """ request: RequestInstancePowerOffRestart = cherrypy.request.model cherrypy.response.status = 202 @@ -288,14 +364,26 @@ def action_restart(self, **_): instance.action_restart(request.hard, request.timeout) - @Route(route='{instance_id}/action/image', methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() + @Route(route='{instance_name}/action/image', methods=[RequestMethods.POST]) @cherrypy.tools.model_params(cls=ParamsInstance) @cherrypy.tools.model_in(cls=RequestInstanceImage) @cherrypy.tools.model_out(cls=ResponseImage) - @cherrypy.tools.resource_object(id_param="instance_id", cls=Instance) + @cherrypy.tools.resource_object(id_param="instance_name", cls=Instance) @cherrypy.tools.enforce_policy(policy_name="instances:action:image") def action_image(self, **_): + """Image an instance + --- + put: + description: Image an instance + tags: + - compute + - instance + requestBody: + description: Image to create + responses: + 200: + description: The created image + """ project: Project = cherrypy.request.project request: RequestInstanceImage = cherrypy.request.model @@ -308,10 +396,10 @@ def action_image(self, **_): if instance.power_state != VMPowerState.POWERED_OFF: raise cherrypy.HTTPError(400, 'Instance must be powered off.') - if Image.get_by_name(request.name, project=project) is not None: + if Image.get(project, request.name) is not None: raise cherrypy.HTTPError(400, 'An image with the requested name already exists.') - attached_volumes = Volume.list(project, label_selector=ATTACHED_TO_LABEL + "=" + str(instance.id)) + attached_volumes = Volume.list(project, label_selector=ATTACHED_TO_LABEL + "=" + str(instance.name)) if len(attached_volumes) > 0: raise cherrypy.HTTPError(409, 'Cannot create an image while volumes are attached.') diff --git a/deli/counter/http/mounts/root/routes/v1/keypairs.py b/deli/counter/http/mounts/root/routes/compute/v1/keypairs.py similarity index 57% rename from deli/counter/http/mounts/root/routes/v1/keypairs.py rename to deli/counter/http/mounts/root/routes/compute/v1/keypairs.py index be2a386..e5c9643 100644 --- a/deli/counter/http/mounts/root/routes/v1/keypairs.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/keypairs.py @@ -4,27 +4,40 @@ from ingredients_http.request_methods import RequestMethods from ingredients_http.route import Route -from deli.counter.http.mounts.root.routes.v1.validation_models.keypairs import RequestCreateKeypair, ResponseKeypair, \ +from deli.counter.http.mounts.root.routes.compute.v1.validation_models.keypairs import RequestCreateKeypair, \ + ResponseKeypair, \ ParamsKeypair, ParamsListKeypair -from deli.counter.http.router import SandwichRouter +from deli.counter.http.router import SandwichProjectRouter from deli.kubernetes.resources.model import ResourceState from deli.kubernetes.resources.v1alpha1.keypair.keypair import Keypair -class KeypairsRouter(SandwichRouter): +class KeypairsRouter(SandwichProjectRouter): def __init__(self): super().__init__(uri_base='keypairs') @Route(methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() @cherrypy.tools.model_in(cls=RequestCreateKeypair) @cherrypy.tools.model_out(cls=ResponseKeypair) @cherrypy.tools.enforce_policy(policy_name="keypairs:create") def create(self): + """Create a keypair + --- + post: + description: Create a keypair + tags: + - compute + - keypair + requestBody: + description: Keypair to create + responses: + 200: + description: The created keypair + """ request: RequestCreateKeypair = cherrypy.request.model project = cherrypy.request.project - keypair = Keypair.get_by_name(project, request.name) + keypair = Keypair.get(project, request.name) if keypair is not None: raise cherrypy.HTTPError(409, "A keypair with the requested name already exists.") @@ -35,32 +48,62 @@ def create(self): keypair.create() return ResponseKeypair.from_database(keypair) - @Route(route='{keypair_id}') - @cherrypy.tools.project_scope() + @Route(route='{keypair_name}') @cherrypy.tools.model_params(cls=ParamsKeypair) @cherrypy.tools.model_out(cls=ResponseKeypair) - @cherrypy.tools.resource_object(id_param="keypair_id", cls=Keypair) + @cherrypy.tools.resource_object(id_param="keypair_name", cls=Keypair) @cherrypy.tools.enforce_policy(policy_name="keypairs:get") def get(self, **_): + """Get a keypair + --- + get: + description: Get a keypair + tags: + - compute + - keypair + responses: + 200: + description: The keypair + """ return ResponseKeypair.from_database(cherrypy.request.resource_object) @Route() - @cherrypy.tools.project_scope() @cherrypy.tools.model_params(cls=ParamsListKeypair) @cherrypy.tools.model_out_pagination(cls=ResponseKeypair) @cherrypy.tools.enforce_policy(policy_name="keypairs:list") def list(self, limit: int, marker: uuid.UUID): + """List keypairs + --- + get: + description: List keypairs + tags: + - compute + - keypair + responses: + 200: + description: List of keypairs + """ kwargs = { 'project': cherrypy.request.project } return self.paginate(Keypair, ResponseKeypair, limit, marker, **kwargs) - @Route(route='{keypair_id}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.project_scope() + @Route(route='{keypair_name}', methods=[RequestMethods.DELETE]) @cherrypy.tools.model_params(cls=ParamsKeypair) - @cherrypy.tools.resource_object(id_param="keypair_id", cls=Keypair) + @cherrypy.tools.resource_object(id_param="keypair_name", cls=Keypair) @cherrypy.tools.enforce_policy(policy_name="keypairs:delete") def delete(self, **_): + """Delete a keypair + --- + delete: + description: Delete a keypair + tags: + - compute + - keypair + responses: + 204: + description: Keypair deleted + """ cherrypy.response.status = 204 keypair: Keypair = cherrypy.request.resource_object diff --git a/deli/counter/http/mounts/root/routes/v1/network_ports.py b/deli/counter/http/mounts/root/routes/compute/v1/network_ports.py similarity index 67% rename from deli/counter/http/mounts/root/routes/v1/network_ports.py rename to deli/counter/http/mounts/root/routes/compute/v1/network_ports.py index c72c12e..4126bfe 100644 --- a/deli/counter/http/mounts/root/routes/v1/network_ports.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/network_ports.py @@ -2,45 +2,75 @@ from ingredients_http.request_methods import RequestMethods from ingredients_http.route import Route -from deli.counter.http.mounts.root.routes.v1.validation_models.network_ports import ResponseNetworkPort, \ +from deli.counter.http.mounts.root.routes.compute.v1.validation_models.network_ports import ResponseNetworkPort, \ ParamsNetworkPort, ParamsListNetworkPort -from deli.counter.http.router import SandwichRouter +from deli.counter.http.router import SandwichProjectRouter from deli.kubernetes.resources.const import NETWORK_PORT_LABEL from deli.kubernetes.resources.model import ResourceState from deli.kubernetes.resources.v1alpha1.instance.model import Instance from deli.kubernetes.resources.v1alpha1.network.model import NetworkPort -class NetworkPortsRouter(SandwichRouter): +class NetworkPortsRouter(SandwichProjectRouter): def __init__(self): super().__init__(uri_base='network-ports') @Route(route='{network_port_id}') - @cherrypy.tools.project_scope() @cherrypy.tools.model_params(cls=ParamsNetworkPort) @cherrypy.tools.model_out(cls=ResponseNetworkPort) @cherrypy.tools.resource_object(id_param="network_port_id", cls=NetworkPort) @cherrypy.tools.enforce_policy(policy_name="network_ports:get") def get(self, **_): + """Get a network port + --- + get: + description: Get a network port + tags: + - compute + - network + responses: + 200: + description: The network port + """ return ResponseNetworkPort.from_database(cherrypy.request.resource_object) @Route() - @cherrypy.tools.project_scope() @cherrypy.tools.model_params(cls=ParamsListNetworkPort) @cherrypy.tools.model_out_pagination(cls=ResponseNetworkPort) @cherrypy.tools.enforce_policy(policy_name="network_ports:list") def list(self, limit, marker): + """List network ports + --- + get: + description: List network ports + tags: + - compute + - network + responses: + 200: + description: List of network ports + """ kwargs = { 'project': cherrypy.request.project } return self.paginate(NetworkPort, ResponseNetworkPort, limit, marker, **kwargs) @Route(route='{network_port_id}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.project_scope() @cherrypy.tools.model_params(cls=ParamsNetworkPort) @cherrypy.tools.resource_object(id_param="network_port_id", cls=NetworkPort) @cherrypy.tools.enforce_policy(policy_name="network_ports:delete") def delete(self, **_): + """Delete a network port + --- + delete: + description: Delete a network port + tags: + - compute + - network + responses: + 204: + description: Network port deleted + """ cherrypy.response.status = 204 network_port: NetworkPort = cherrypy.request.resource_object @@ -50,7 +80,7 @@ def delete(self, **_): if network_port.state == ResourceState.Deleted: raise cherrypy.HTTPError(400, "Network Port has already been deleted") - instances = Instance.list_all(label_selector=NETWORK_PORT_LABEL + "=" + str(network_port.id)) + instances = Instance.list_all(label_selector=NETWORK_PORT_LABEL + "=" + network_port.name) if len(instances) > 0: raise cherrypy.HTTPError(409, 'Cannot delete a network port while it is in use by an instance') diff --git a/deli/counter/http/mounts/root/routes/v1/networks.py b/deli/counter/http/mounts/root/routes/compute/v1/networks.py similarity index 63% rename from deli/counter/http/mounts/root/routes/v1/networks.py rename to deli/counter/http/mounts/root/routes/compute/v1/networks.py index 0f37eb3..af12faa 100644 --- a/deli/counter/http/mounts/root/routes/v1/networks.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/networks.py @@ -4,10 +4,11 @@ from ingredients_http.request_methods import RequestMethods from ingredients_http.route import Route -from deli.counter.http.mounts.root.routes.v1.validation_models.networks import RequestCreateNetwork, ResponseNetwork, \ +from deli.counter.http.mounts.root.routes.compute.v1.validation_models.networks import RequestCreateNetwork, \ + ResponseNetwork, \ ParamsNetwork, ParamsListNetwork from deli.counter.http.router import SandwichRouter -from deli.kubernetes.resources.const import REGION_LABEL, NAME_LABEL +from deli.kubernetes.resources.const import REGION_LABEL from deli.kubernetes.resources.model import ResourceState from deli.kubernetes.resources.v1alpha1.network.model import Network from deli.kubernetes.resources.v1alpha1.region.model import Region @@ -22,15 +23,28 @@ def __init__(self): @cherrypy.tools.model_out(cls=ResponseNetwork) @cherrypy.tools.enforce_policy(policy_name="networks:create") def create(self): + """Create a network + --- + post: + description: Create a network + tags: + - compute + - network + requestBody: + description: Network to create + responses: + 200: + description: The created network + """ request: RequestCreateNetwork = cherrypy.request.model - network = Network.get_by_name(request.name) + network = Network.get(request.name) if network is not None: raise cherrypy.HTTPError(409, 'A network with the requested name does already exists.') - region = Region.get(request.region_id) + region = Region.get(request.region_name) if region is None: - raise cherrypy.HTTPError(404, 'A region with the requested id does not exist.') + raise cherrypy.HTTPError(404, 'A region with the requested name does not exist.') if region.state != ResourceState.Created: raise cherrypy.HTTPError(409, 'Can only create a network with a region in the following state: %s'.format( @@ -51,31 +65,48 @@ def create(self): return ResponseNetwork.from_database(network) - @Route(route='{network_id}') + @Route(route='{network_name}') @cherrypy.tools.model_params(cls=ParamsNetwork) @cherrypy.tools.model_out(cls=ResponseNetwork) - @cherrypy.tools.resource_object(id_param="network_id", cls=Network) - @cherrypy.tools.enforce_policy(policy_name="networks:get") + @cherrypy.tools.resource_object(id_param="network_name", cls=Network) def get(self, **_): + """Get a network + --- + get: + description: Get a network + tags: + - compute + - network + responses: + 200: + description: The network + """ return ResponseNetwork.from_database(cherrypy.request.resource_object) @Route() @cherrypy.tools.model_params(cls=ParamsListNetwork) @cherrypy.tools.model_out_pagination(cls=ResponseNetwork) - @cherrypy.tools.enforce_policy(policy_name="networks:list") - def list(self, name, region_id, limit: int, marker: uuid.UUID): + def list(self, region, limit: int, marker: uuid.UUID): + """List networks + --- + get: + description: List networks + tags: + - compute + - network + responses: + 200: + description: List of networks + """ kwargs = { 'label_selector': [] } - if region_id is not None: - region: Region = Region.get(region_id) + if region is not None: + region: Region = Region.get(region) if region is None: raise cherrypy.HTTPError(404, "A region with the requested name does not exist.") - kwargs['label_selector'].append(REGION_LABEL + '=' + str(region.id)) - - if name is not None: - kwargs['label_selector'].append(NAME_LABEL + '=' + name) + kwargs['label_selector'].append(REGION_LABEL + '=' + region.name) if len(kwargs['label_selector']) > 0: kwargs['label_selector'] = ",".join(kwargs['label_selector']) @@ -84,11 +115,22 @@ def list(self, name, region_id, limit: int, marker: uuid.UUID): return self.paginate(Network, ResponseNetwork, limit, marker, **kwargs) - @Route(route='{network_id}', methods=[RequestMethods.DELETE]) + @Route(route='{network_name}', methods=[RequestMethods.DELETE]) @cherrypy.tools.model_params(cls=ParamsNetwork) - @cherrypy.tools.resource_object(id_param="network_id", cls=Network) + @cherrypy.tools.resource_object(id_param="network_name", cls=Network) @cherrypy.tools.enforce_policy(policy_name="networks:delete") def delete(self, **_): + """Delete a network + --- + delete: + description: Delete a network + tags: + - compute + - network + responses: + 204: + description: Network deleted + """ cherrypy.response.status = 204 network: Network = cherrypy.request.resource_object diff --git a/deli/counter/http/mounts/root/routes/v1/errors/__init__.py b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/__init__.py similarity index 100% rename from deli/counter/http/mounts/root/routes/v1/errors/__init__.py rename to deli/counter/http/mounts/root/routes/compute/v1/validation_models/__init__.py diff --git a/deli/counter/http/mounts/root/routes/v1/validation_models/flavor.py b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/flavor.py similarity index 89% rename from deli/counter/http/mounts/root/routes/v1/validation_models/flavor.py rename to deli/counter/http/mounts/root/routes/compute/v1/validation_models/flavor.py index ce0c2ff..dc89576 100644 --- a/deli/counter/http/mounts/root/routes/v1/validation_models/flavor.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/flavor.py @@ -6,7 +6,7 @@ class ParamsFlavor(Model): - flavor_id = UUIDType(required=True) + flavor_name = KubeName(required=True) class ParamsListFlavor(Model): @@ -22,21 +22,21 @@ class RequestCreateFlavor(Model): class ResponseFlavor(Model): - id = UUIDType(required=True) name = KubeName(required=True, min_length=3) vcpus = IntType(required=True) ram = IntType(required=True) disk = IntType(required=True) created_at = ArrowType(required=True) + updated_at = ArrowType(required=True) @classmethod def from_database(cls, flavor: Flavor): model = cls() - model.id = flavor.id model.name = flavor.name model.vcpus = flavor.vcpus model.ram = flavor.ram model.disk = flavor.disk model.created_at = flavor.created_at + model.updated_at = flavor.updated_at return model diff --git a/deli/counter/http/mounts/root/routes/v1/validation_models/images.py b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/images.py similarity index 58% rename from deli/counter/http/mounts/root/routes/v1/validation_models/images.py rename to deli/counter/http/mounts/root/routes/compute/v1/validation_models/images.py index c672823..923628d 100644 --- a/deli/counter/http/mounts/root/routes/v1/validation_models/images.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/images.py @@ -1,23 +1,18 @@ from ingredients_http.schematics.types import EnumType, KubeName, ArrowType from schematics import Model -from schematics.types import UUIDType, StringType, IntType, BooleanType +from schematics.types import UUIDType, StringType, IntType from deli.kubernetes.resources.model import ResourceState from deli.kubernetes.resources.v1alpha1.image.model import Image, ImageVisibility class ParamsImage(Model): - image_id = UUIDType(required=True) - - -class ParamsImageMember(Model): - image_id = UUIDType(required=True) - project_id = UUIDType(required=True) + image_name = StringType(required=True) class ParamsListImage(Model): visibility = EnumType(ImageVisibility, default=ImageVisibility.PRIVATE) - region_id = UUIDType() + region_name = StringType() limit = IntType(default=100, max_value=100, min_value=1) marker = UUIDType() @@ -25,46 +20,32 @@ class ParamsListImage(Model): class RequestCreateImage(Model): name = KubeName(required=True, min_length=3) file_name = StringType(required=True) - region_id = KubeName(required=True) - - -class RequestAddMember(Model): - project_id = UUIDType(required=True) - - -class ResponseImageMember(Model): - project_id = UUIDType(required=True) - - -class RequestImageVisibility(Model): - public = BooleanType(required=True) + region_name = KubeName(required=True) class ResponseImage(Model): - id = UUIDType(required=True) - project_id = UUIDType(required=True) + project_name = StringType(required=True) name = KubeName(required=True, min_length=3) file_name = StringType() - region_id = UUIDType(required=True) - visibility = EnumType(ImageVisibility) + region_name = StringType(required=True) state = EnumType(ResourceState, required=True) error_message = StringType() created_at = ArrowType(required=True) + updated_at = ArrowType(required=True) @classmethod def from_database(cls, image: Image): image_model = cls() - image_model.id = image.id - image_model.project_id = image.project_id + image_model.project_name = image.project_name image_model.name = image.name image_model.file_name = image.file_name - image_model.region_id = image.region_id - image_model.visibility = image.visibility + image_model.region_name = image.region_name image_model.state = image.state if image.error_message != "": image_model.error_message = image.error_message image_model.created_at = image.created_at + image_model.updated_at = image.updated_at return image_model diff --git a/deli/counter/http/mounts/root/routes/v1/validation_models/instances.py b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/instances.py similarity index 65% rename from deli/counter/http/mounts/root/routes/v1/validation_models/instances.py rename to deli/counter/http/mounts/root/routes/compute/v1/validation_models/instances.py index 9a5320e..2fa4c57 100644 --- a/deli/counter/http/mounts/root/routes/v1/validation_models/instances.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/instances.py @@ -3,7 +3,6 @@ from schematics.types import UUIDType, IntType, DictType, ListType, BooleanType, StringType, ModelType from deli.kubernetes.resources.model import ResourceState -from deli.kubernetes.resources.v1alpha1.image.model import ImageVisibility from deli.kubernetes.resources.v1alpha1.instance.model import Instance, VMPowerState, VMTask @@ -14,16 +13,16 @@ class RequestInitialVolumes(Model): class RequestCreateInstance(Model): name = KubeName(required=True, min_length=3) - image_id = UUIDType(required=True) - service_account_id = UUIDType() - network_id = UUIDType(required=True) - region_id = UUIDType(required=True) - zone_id = UUIDType() - keypair_ids = ListType(UUIDType, default=list) + image_name = KubeName(required=True) + service_account_name = KubeName() + network_name = KubeName(required=True) + region_name = KubeName(required=True) + zone_name = KubeName() + keypair_names = ListType(KubeName, default=list) tags = DictType(KubeString, default=dict) user_data = StringType() - flavor_id = UUIDType(required=True) + flavor_name = KubeName(required=True) disk = IntType() initial_volumes = ListType(ModelType(RequestInitialVolumes), default=list) @@ -33,18 +32,17 @@ def validate_disk(self, data, value): class ResponseInstance(Model): - id = UUIDType(required=True) name = KubeName(required=True, min_length=3) - image_id = UUIDType() - network_port_id = UUIDType(required=True) - region_id = UUIDType(required=True) - zone_id = UUIDType() - service_account_id = UUIDType() - keypair_ids = ListType(UUIDType, default=list) + image_name = KubeName() + network_port_id = KubeName(required=True) + region_name = KubeName(required=True) + zone_name = KubeName() + service_account_name = KubeName() + keypair_names = ListType(KubeName, default=list) state = EnumType(ResourceState, required=True) power_state = EnumType(VMPowerState, required=True) - flavor_id = UUIDType(required=True) + flavor_name = KubeName(required=True) vcpus = IntType(required=True) ram = IntType(required=True) disk = IntType(required=True) @@ -54,23 +52,23 @@ class ResponseInstance(Model): user_data = StringType() error_message = StringType() created_at = ArrowType(required=True) + updated_at = ArrowType(required=True) @classmethod def from_database(cls, instance: Instance): instance_model = cls() - instance_model.id = instance.id instance_model.name = instance.name - instance_model.image_id = instance.image_id + instance_model.image_name = instance.image_name instance_model.network_port_id = instance.network_port_id - instance_model.service_account_id = instance.service_account_id + instance_model.service_account_name = instance.service_account_name - instance_model.region_id = instance.region_id - if instance.zone_id is not None: - instance_model.zone_id = instance.zone_id + instance_model.region_name = instance.region_name + if instance.zone_name is not None: + instance_model.zone_name = instance.zone_name - for keypair_id in instance.keypair_ids: - instance_model.keypair_ids.append(keypair_id) + for keypair_name in instance.keypair_names: + instance_model.keypair_names.append(keypair_name) instance_model.state = instance.state if instance.error_message != "": @@ -78,8 +76,8 @@ def from_database(cls, instance: Instance): instance_model.power_state = instance.power_state - if instance.flavor_id is not None: - instance_model.flavor_id = instance.flavor_id + if instance.flavor_name is not None: + instance_model.flavor_name = instance.flavor_name instance_model.vcpus = instance.vcpus instance_model.ram = instance.ram @@ -90,24 +88,24 @@ def from_database(cls, instance: Instance): instance_model.tags = instance.tags instance_model.user_data = instance.user_data instance_model.created_at = instance.created_at + instance_model.updated_at = instance.updated_at return instance_model class ParamsInstance(Model): - instance_id = UUIDType(required=True) + instance_name = KubeName(required=True) class ParamsListInstance(Model): - image_id = UUIDType() - zone_id = UUIDType() - region_id = UUIDType() + image_name = KubeName() + zone_name = KubeName() + region_name = KubeName() limit = IntType(default=100, max_value=100, min_value=1) marker = UUIDType() class RequestInstanceImage(Model): name = KubeName(required=True) - visibility = EnumType(ImageVisibility, default=ImageVisibility.PRIVATE) class RequestInstancePowerOffRestart(Model): diff --git a/deli/counter/http/mounts/root/routes/v1/validation_models/keypairs.py b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/keypairs.py similarity index 92% rename from deli/counter/http/mounts/root/routes/v1/validation_models/keypairs.py rename to deli/counter/http/mounts/root/routes/compute/v1/validation_models/keypairs.py index a9f3acf..7bda3ef 100644 --- a/deli/counter/http/mounts/root/routes/v1/validation_models/keypairs.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/keypairs.py @@ -10,7 +10,7 @@ class ParamsKeypair(Model): - keypair_id = UUIDType(required=True) + keypair_name = KubeName(required=True) class ParamsListKeypair(Model): @@ -34,17 +34,17 @@ def validate_public_key(self, data, value): class ResponseKeypair(Model): - id = UUIDType(required=True) name = KubeName(required=True, min_length=3) public_key = StringType(required=True) created_at = ArrowType(required=True) + updated_at = ArrowType(required=True) @classmethod def from_database(cls, keypair: Keypair): model = cls() - model.id = keypair.id model.name = keypair.name model.public_key = keypair.public_key model.created_at = keypair.created_at + model.updated_at = keypair.updated_at return model diff --git a/deli/counter/http/mounts/root/routes/v1/validation_models/network_ports.py b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/network_ports.py similarity index 78% rename from deli/counter/http/mounts/root/routes/v1/validation_models/network_ports.py rename to deli/counter/http/mounts/root/routes/compute/v1/validation_models/network_ports.py index e40a13c..3fe773d 100644 --- a/deli/counter/http/mounts/root/routes/v1/validation_models/network_ports.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/network_ports.py @@ -1,4 +1,4 @@ -from ingredients_http.schematics.types import IPv4AddressType, EnumType, ArrowType +from ingredients_http.schematics.types import IPv4AddressType, EnumType, ArrowType, KubeName from schematics import Model from schematics.types import UUIDType, IntType @@ -17,18 +17,20 @@ class ParamsListNetworkPort(Model): class ResponseNetworkPort(Model): id = UUIDType(required=True) - network_id = UUIDType(required=True) + network_name = KubeName(required=True) ip_address = IPv4AddressType(required=True) state = EnumType(ResourceState, required=True) created_at = ArrowType(required=True) + updated_at = ArrowType(required=True) @classmethod def from_database(cls, network_port: NetworkPort): model = cls() - model.id = network_port.id - model.network_id = network_port.network_id + model.id = network_port.name + model.network_name = network_port.network_name model.ip_address = network_port.ip_address model.state = network_port.state model.created_at = network_port.created_at + model.updated_at = network_port.updated_at return model diff --git a/deli/counter/http/mounts/root/routes/v1/validation_models/networks.py b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/networks.py similarity index 90% rename from deli/counter/http/mounts/root/routes/v1/validation_models/networks.py rename to deli/counter/http/mounts/root/routes/compute/v1/validation_models/networks.py index 832adba..391e1ae 100644 --- a/deli/counter/http/mounts/root/routes/v1/validation_models/networks.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/networks.py @@ -17,7 +17,7 @@ class RequestCreateNetwork(Model): dns_servers = ListType(IPv4AddressType, min_size=1, required=True) pool_start = IPv4AddressType(required=True) pool_end = IPv4AddressType(required=True) - region_id = UUIDType(required=True) + region_name = KubeName(required=True) def validate_gateway(self, data, value): cidr: ipaddress.IPv4Network = data['cidr'] @@ -48,7 +48,6 @@ def validate_pool_end(self, data, value): class ResponseNetwork(Model): - id = UUIDType(required=True) name = KubeName(required=True, min_length=3) port_group = StringType(required=True) cidr = IPv4NetworkType(required=True) @@ -56,15 +55,15 @@ class ResponseNetwork(Model): dns_servers = ListType(IPv4AddressType, min_size=1, required=True) pool_start = IPv4AddressType(required=True) pool_end = IPv4AddressType(required=True) - region_id = UUIDType() + region_name = KubeName() state = EnumType(ResourceState, required=True) error_message = StringType() created_at = ArrowType(required=True) + updated_at = ArrowType(required=True) @classmethod def from_database(cls, network: Network): network_model = cls() - network_model.id = network.id network_model.name = network.name network_model.port_group = network.port_group @@ -73,22 +72,23 @@ def from_database(cls, network: Network): network_model.dns_servers = network.dns_servers network_model.pool_start = network.pool_start network_model.pool_end = network.pool_end - network_model.region_id = network.region_id + network_model.region_name = network.region_name network_model.state = network.state if network.error_message != "": network_model.error_message = network.error_message network_model.created_at = network.created_at + network_model.updated_at = network.updated_at return network_model class ParamsNetwork(Model): - network_id = UUIDType(required=True) + network_name = KubeName(required=True) class ParamsListNetwork(Model): name = KubeName() - region_id = UUIDType() + region = KubeName() limit = IntType(default=100, max_value=100, min_value=1) marker = UUIDType() diff --git a/deli/counter/http/mounts/root/routes/v1/validation_models/volume.py b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/volume.py similarity index 80% rename from deli/counter/http/mounts/root/routes/v1/validation_models/volume.py rename to deli/counter/http/mounts/root/routes/compute/v1/validation_models/volume.py index a6c884b..933eaeb 100644 --- a/deli/counter/http/mounts/root/routes/v1/validation_models/volume.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/validation_models/volume.py @@ -7,7 +7,7 @@ class ParamsVolume(Model): - volume_id = UUIDType(required=True) + volume_name = UUIDType(required=True) class ParamsListVolume(Model): @@ -17,7 +17,7 @@ class ParamsListVolume(Model): class RequestCreateVolume(Model): name = KubeName(required=True, min_length=3) - zone_id = UUIDType(required=True) + zone_name = UUIDType(required=True) size = IntType(required=True, min_value=5) @@ -26,7 +26,7 @@ class RequestCloneVolume(Model): class RequestAttachVolume(Model): - instance_id = UUIDType(required=True) + instance_name = UUIDType(required=True) class RequestGrowVolume(Model): @@ -34,27 +34,27 @@ class RequestGrowVolume(Model): class ResponseVolume(Model): - id = UUIDType(required=True) name = KubeName(required=True, min_length=3) - zone_id = UUIDType(required=True) + zone_name = UUIDType(required=True) size = IntType(required=True) attached_to = UUIDType() state = EnumType(ResourceState, required=True) task = EnumType(VolumeTask) error_message = StringType() created_at = ArrowType(required=True) + updated_at = ArrowType(required=True) @classmethod def from_database(cls, volume: Volume): model = cls() - model.id = volume.id model.name = volume.name - model.zone_id = volume.zone_id + model.zone_name = volume.zone_name model.size = volume.size model.state = volume.state - model.attached_to = volume.attached_to_id + model.attached_to = volume.attached_to_name model.task = volume.task model.error_message = volume.error_message model.created_at = volume.created_at + model.updated_at = volume.updated_at return model diff --git a/deli/counter/http/mounts/root/routes/v1/volume.py b/deli/counter/http/mounts/root/routes/compute/v1/volume.py similarity index 66% rename from deli/counter/http/mounts/root/routes/v1/volume.py rename to deli/counter/http/mounts/root/routes/compute/v1/volume.py index f997d35..797027f 100644 --- a/deli/counter/http/mounts/root/routes/v1/volume.py +++ b/deli/counter/http/mounts/root/routes/compute/v1/volume.py @@ -4,10 +4,10 @@ from ingredients_http.request_methods import RequestMethods from ingredients_http.route import Route -from deli.counter.http.mounts.root.routes.v1.errors.quota import QuotaError -from deli.counter.http.mounts.root.routes.v1.validation_models.volume import RequestCreateVolume, ResponseVolume, \ - ParamsVolume, ParamsListVolume, RequestCloneVolume, RequestAttachVolume, RequestGrowVolume -from deli.counter.http.router import SandwichRouter +from deli.counter.http.mounts.root.errors.quota import QuotaError +from deli.counter.http.mounts.root.routes.compute.v1.validation_models.volume import RequestCreateVolume, \ + ResponseVolume, ParamsVolume, ParamsListVolume, RequestCloneVolume, RequestAttachVolume, RequestGrowVolume +from deli.counter.http.router import SandwichProjectRouter from deli.kubernetes.resources.model import ResourceState from deli.kubernetes.resources.project import Project from deli.kubernetes.resources.v1alpha1.instance.model import Instance @@ -16,32 +16,44 @@ from deli.kubernetes.resources.v1alpha1.zone.model import Zone -class VolumeRouter(SandwichRouter): +class VolumeRouter(SandwichProjectRouter): def __init__(self): super().__init__(uri_base='volumes') @Route(methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() @cherrypy.tools.model_in(cls=RequestCreateVolume) @cherrypy.tools.model_out(cls=ResponseVolume) @cherrypy.tools.enforce_policy(policy_name="volumes:create") def create(self): + """Create a volume + --- + post: + description: Create a volume + tags: + - compute + - volume + requestBody: + description: Volume to create + responses: + 200: + description: The created volume + """ request: RequestCreateVolume = cherrypy.request.model project: Project = cherrypy.request.project - volume = Volume.get_by_name(project, request.name) + volume = Volume.get(project, request.name) if volume is not None: raise cherrypy.HTTPError(409, 'A volume with the requested name already exists.') - zone = Zone.get(request.zone_id) + zone = Zone.get(request.zone_name) if zone is None: - raise cherrypy.HTTPError(404, 'A zone with the requested id does not exist.') + raise cherrypy.HTTPError(404, 'A zone with the requested name does not exist.') if zone.state != ResourceState.Created: raise cherrypy.HTTPError(400, 'Can only create a volume with a zone in the following state: %s'.format( ResourceState.Created)) - quota: ProjectQuota = ProjectQuota.get(project, project.id) + quota: ProjectQuota = ProjectQuota.get(project.name) used_disk = quota.used_disk + request.size if quota.disk != -1: if used_disk > quota.disk: @@ -59,21 +71,41 @@ def create(self): return ResponseVolume.from_database(volume) - @Route(route='{volume_id}') - @cherrypy.tools.project_scope() + @Route(route='{volume_name}') @cherrypy.tools.model_params(cls=ParamsVolume) @cherrypy.tools.model_out(cls=ResponseVolume) - @cherrypy.tools.resource_object(id_param="volume_id", cls=Volume) + @cherrypy.tools.resource_object(id_param="volume_name", cls=Volume) @cherrypy.tools.enforce_policy(policy_name="volumes:get") def get(self, **_): + """Get a volume + --- + get: + description: Get a volume + tags: + - compute + - volume + responses: + 200: + description: The volume + """ return ResponseVolume.from_database(cherrypy.request.resource_object) @Route() - @cherrypy.tools.project_scope() @cherrypy.tools.model_params(cls=ParamsListVolume) @cherrypy.tools.model_out_pagination(cls=ResponseVolume) @cherrypy.tools.enforce_policy(policy_name="volumes:list") def list(self, limit: int, marker: uuid.UUID): + """List volumes + --- + get: + description: List volumes + tags: + - compute + - volume + responses: + 200: + description: List of volumes + """ kwargs = { 'project': cherrypy.request.project, 'label_selector': [], @@ -84,12 +116,22 @@ def list(self, limit: int, marker: uuid.UUID): del kwargs['label_selector'] return self.paginate(Volume, ResponseVolume, limit, marker, **kwargs) - @Route(route='{volume_id}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.project_scope() + @Route(route='{volume_name}', methods=[RequestMethods.DELETE]) @cherrypy.tools.model_params(cls=ParamsVolume) - @cherrypy.tools.resource_object(id_param="volume_id", cls=Volume) + @cherrypy.tools.resource_object(id_param="volume_name", cls=Volume) @cherrypy.tools.enforce_policy(policy_name="volumes:delete") def delete(self, **_): + """Delete a volume + --- + delete: + description: Delete a volume + tags: + - compute + - volume + responses: + 204: + description: Volume deleted + """ cherrypy.response.status = 204 volume: Volume = cherrypy.request.resource_object @@ -100,18 +142,30 @@ def delete(self, **_): if volume.state == ResourceState.Deleted: raise cherrypy.HTTPError(400, "Volume has already been deleted") - if volume.attached_to_id is not None: + if volume.attached_to_name is not None: raise cherrypy.HTTPError(409, 'Cannot delete when attached to an instance.') volume.delete() - @Route(route='{volume_id}/action/attach', methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() + @Route(route='{volume_name}/action/attach', methods=[RequestMethods.POST]) @cherrypy.tools.model_params(cls=ParamsVolume) @cherrypy.tools.model_in(cls=RequestAttachVolume) - @cherrypy.tools.resource_object(id_param="volume_id", cls=Volume) + @cherrypy.tools.resource_object(id_param="volume_name", cls=Volume) @cherrypy.tools.enforce_policy(policy_name="volumes:action:attach") def action_attach(self, **_): + """Attach a volume + --- + put: + description: Attach a volume + tags: + - compute + - volume + requestBody: + description: Attach options + responses: + 204: + description: Volume attaching + """ request: RequestAttachVolume = cherrypy.request.model cherrypy.response.status = 204 @@ -121,19 +175,19 @@ def action_attach(self, **_): if volume.task is not None: raise cherrypy.HTTPError(400, "Please wait for the current task to finish.") - if volume.attached_to_id is not None: + if volume.attached_to_name is not None: raise cherrypy.HTTPError(409, 'Volume is already attached to an instance.') - instance: Instance = Instance.get(cherrypy.request.project, request.instance_id) + instance: Instance = Instance.get(cherrypy.request.project, request.instance_name) if instance is None: raise cherrypy.HTTPError(404, 'Could not find the requested instance.') if instance.state != ResourceState.Created: raise cherrypy.HTTPError(400, 'The requested instance is not in the following state: ' + ResourceState.Created.value) - if instance.region_id != volume.region_id: + if instance.region_name != volume.region_name: raise cherrypy.HTTPError(400, 'The requested instance is not in the same region as the volume.') - if instance.zone_id != volume.zone_id: + if instance.zone_name != volume.zone_name: raise cherrypy.HTTPError(400, 'The requested instance is not in the same zone as the volume.') if instance.task is not None: raise cherrypy.HTTPError(400, 'Please wait for the current task on the instance to finish.') @@ -141,12 +195,22 @@ def action_attach(self, **_): volume.attach(instance) volume.save() - @Route(route='{volume_id}/action/detach', methods=[RequestMethods.PUT]) - @cherrypy.tools.project_scope() + @Route(route='{volume_name}/action/detach', methods=[RequestMethods.PUT]) @cherrypy.tools.model_params(cls=ParamsVolume) - @cherrypy.tools.resource_object(id_param="volume_id", cls=Volume) + @cherrypy.tools.resource_object(id_param="volume_name", cls=Volume) @cherrypy.tools.enforce_policy(policy_name="volumes:action:detach") def action_detach(self, **_): + """Detach a volume + --- + put: + description: Detach a volume + tags: + - compute + - volume + responses: + 204: + description: Volume detaching + """ cherrypy.response.status = 204 volume: Volume = cherrypy.request.resource_object @@ -155,19 +219,31 @@ def action_detach(self, **_): if volume.task is not None: raise cherrypy.HTTPError(400, "Please wait for the current task to finish.") - if volume.attached_to_id is None: + if volume.attached_to_name is None: raise cherrypy.HTTPError(409, 'Volume is not attached to an instance.') volume.task = VolumeTask.DETACHING volume.save() - @Route(route='{volume_id}/action/grow', methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() + @Route(route='{volume_name}/action/grow', methods=[RequestMethods.POST]) @cherrypy.tools.model_params(cls=ParamsVolume) @cherrypy.tools.model_in(cls=RequestGrowVolume) - @cherrypy.tools.resource_object(id_param="volume_id", cls=Volume) + @cherrypy.tools.resource_object(id_param="volume_name", cls=Volume) @cherrypy.tools.enforce_policy(policy_name="volumes:action:grow") def action_grow(self, **_): + """Grow a volume + --- + put: + description: Grow a volume + tags: + - compute + - volume + requestBody: + description: Grow options + responses: + 204: + description: Volume growing + """ project: Project = cherrypy.request.project request: RequestGrowVolume = cherrypy.request.model cherrypy.response.status = 204 @@ -178,13 +254,13 @@ def action_grow(self, **_): if volume.task is not None: raise cherrypy.HTTPError(400, "Please wait for the current task to finish.") - if volume.attached_to_id is not None: + if volume.attached_to_name is not None: raise cherrypy.HTTPError(409, 'Cannot grow while attached to an instance') if request.size <= volume.size: raise cherrypy.HTTPError(400, 'Size must be bigger than the current volume size.') - quota: ProjectQuota = ProjectQuota.get(project, project.id) + quota: ProjectQuota = ProjectQuota.get(project.name) used_disk = quota.used_disk + request.size if quota.disk != -1: if used_disk > quota.disk: @@ -197,14 +273,26 @@ def action_grow(self, **_): volume.task_kwargs = {"size": request.size} volume.save() - @Route(route='{volume_id}/action/clone', methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() + @Route(route='{volume_name}/action/clone', methods=[RequestMethods.POST]) @cherrypy.tools.model_params(cls=ParamsVolume) @cherrypy.tools.model_in(cls=RequestCloneVolume) @cherrypy.tools.model_out(cls=ResponseVolume) - @cherrypy.tools.resource_object(id_param="volume_id", cls=Volume) + @cherrypy.tools.resource_object(id_param="volume_name", cls=Volume) @cherrypy.tools.enforce_policy(policy_name="volumes:action:grow") def action_clone(self, **_): + """Clone a volume + --- + put: + description: Clone a volume + tags: + - compute + - volume + requestBody: + description: Clone options + responses: + 200: + description: The created volume + """ request: RequestCloneVolume = cherrypy.request.model project: Project = cherrypy.request.project volume: Volume = cherrypy.request.resource_object @@ -212,12 +300,12 @@ def action_clone(self, **_): raise cherrypy.HTTPError(400, 'Volume is not in the following state: ' + ResourceState.Created.value) if volume.task is not None: raise cherrypy.HTTPError(400, "Please wait for the current task to finish.") - if volume.attached_to_id is not None: + if volume.attached_to_name is not None: raise cherrypy.HTTPError(409, 'Cannot clone while attached to an instance') - if Volume.get_by_name(cherrypy.request.project, request.name) is not None: + if Volume.get(cherrypy.request.project, request.name) is not None: raise cherrypy.HTTPError(409, 'A volume with the requested name already exists.') - quota: ProjectQuota = ProjectQuota.get(project, project.id) + quota: ProjectQuota = ProjectQuota.get(project.name) used_disk = quota.used_disk + volume.size if quota.disk != -1: if used_disk > quota.disk: @@ -235,7 +323,7 @@ def action_clone(self, **_): new_volume.create() volume.task = VolumeTask.CLONING - volume.task_kwargs = {'volume_id': str(new_volume.id)} + volume.task_kwargs = {'volume_name': str(new_volume.name)} volume.save() return ResponseVolume.from_database(new_volume) diff --git a/deli/counter/http/mounts/root/routes/health.py b/deli/counter/http/mounts/root/routes/health.py index 35e244a..7b018d2 100644 --- a/deli/counter/http/mounts/root/routes/health.py +++ b/deli/counter/http/mounts/root/routes/health.py @@ -14,8 +14,7 @@ def __init__(self): @cherrypy.tools.json_out() def get(self): data = { - 'kubernetes_version': None, - 'auth': {} + 'kubernetes_version': None } try: @@ -25,10 +24,4 @@ def get(self): cherrypy.response.status = 503 self.logger.exception("Error getting kubernetes version") - for driver in self.mount.auth_manager.drivers.values(): - driver_health = driver.health() - if driver_health['healthy'] is False: - cherrypy.response.status = 503 - data['auth'][driver.name] = driver_health - return data diff --git a/deli/counter/http/mounts/root/routes/v1/validation_models/__init__.py b/deli/counter/http/mounts/root/routes/iam/__init__.py similarity index 100% rename from deli/counter/http/mounts/root/routes/v1/validation_models/__init__.py rename to deli/counter/http/mounts/root/routes/iam/__init__.py diff --git a/deli/kubernetes/resources/v1alpha1/project_member/__init__.py b/deli/counter/http/mounts/root/routes/iam/v1/__init__.py similarity index 100% rename from deli/kubernetes/resources/v1alpha1/project_member/__init__.py rename to deli/counter/http/mounts/root/routes/iam/v1/__init__.py diff --git a/deli/counter/http/mounts/root/routes/iam/v1/policy.py b/deli/counter/http/mounts/root/routes/iam/v1/policy.py new file mode 100644 index 0000000..96d298b --- /dev/null +++ b/deli/counter/http/mounts/root/routes/iam/v1/policy.py @@ -0,0 +1,118 @@ +import uuid + +import cherrypy +from ingredients_http.route import Route + +from deli.counter.auth.policy import SYSTEM_POLICIES, PROJECT_POLICIES +from deli.counter.http.mounts.root.routes.iam.v1.validation_models.policy import ParamsPolicy, ResponsePolicy, \ + ParamsListPolicy +from deli.counter.http.router import SandwichProjectRouter, SandwichSystemRouter + + +class IAMSystemPolicyRouter(SandwichSystemRouter): + def __init__(self): + super().__init__('policies') + + @Route(route='{policy_name}') + @cherrypy.tools.model_params(cls=ParamsPolicy) + @cherrypy.tools.model_out(cls=ResponsePolicy) + def get(self, policy_name): + """Get a system policy + --- + get: + description: Get a system policy + tags: + - iam + - policy + responses: + 200: + description: The policy + """ + policy = None + + for p in SYSTEM_POLICIES: + if p['name'] == policy_name: + policy = p + break + + if policy is None: + raise cherrypy.HTTPError(404, "The resource could not be found.") + + return ResponsePolicy(policy) + + @Route() + @cherrypy.tools.model_params(cls=ParamsListPolicy) + @cherrypy.tools.model_out_pagination(cls=ResponsePolicy) + def list(self, limit: int, marker: uuid.UUID): + """List system policies + --- + get: + description: List system policies + tags: + - iam + - policy + responses: + 200: + description: List of system policies + """ + policies = [] + + for p in SYSTEM_POLICIES: + policies.append(ResponsePolicy(p)) + + return policies, False + + +class IAMProjectPolicyRouter(SandwichProjectRouter): + def __init__(self): + super().__init__('policies') + + @Route(route='{policy_name}') + @cherrypy.tools.model_params(cls=ParamsPolicy) + @cherrypy.tools.model_out(cls=ResponsePolicy) + def get(self, policy_name): + """Get a project policy + --- + get: + description: Get a project policy + tags: + - iam + - policy + responses: + 200: + description: The policy + """ + + policy = None + + for p in PROJECT_POLICIES: + if p['name'] == policy_name: + policy = p + break + + if policy is None: + raise cherrypy.HTTPError(404, "The resource could not be found.") + + return ResponsePolicy(policy) + + @Route() + @cherrypy.tools.model_params(cls=ParamsListPolicy) + @cherrypy.tools.model_out_pagination(cls=ResponsePolicy) + def list(self, limit: int, marker: uuid.UUID): + """List projectpolicies + --- + get: + description: List project policies + tags: + - iam + - policy + responses: + 200: + description: List of project policies + """ + policies = [] + + for p in PROJECT_POLICIES: + policies.append(ResponsePolicy(p)) + + return policies, False diff --git a/deli/counter/http/mounts/root/routes/v1/project_quota.py b/deli/counter/http/mounts/root/routes/iam/v1/project_quota.py similarity index 52% rename from deli/counter/http/mounts/root/routes/v1/project_quota.py rename to deli/counter/http/mounts/root/routes/iam/v1/project_quota.py index 243f7d1..f06d94e 100644 --- a/deli/counter/http/mounts/root/routes/v1/project_quota.py +++ b/deli/counter/http/mounts/root/routes/iam/v1/project_quota.py @@ -2,35 +2,57 @@ from ingredients_http.request_methods import RequestMethods from ingredients_http.route import Route -from deli.counter.http.mounts.root.routes.v1.validation_models.projects import ResponseProjectQuota, \ +from deli.counter.http.mounts.root.routes.iam.v1.validation_models.projects import ResponseProjectQuota, \ RequestProjectModifyQuota -from deli.counter.http.router import SandwichRouter +from deli.counter.http.router import SandwichProjectRouter from deli.kubernetes.resources.project import Project from deli.kubernetes.resources.v1alpha1.project_quota.model import ProjectQuota -class ProjectQuotaRouter(SandwichRouter): +class ProjectQuotaRouter(SandwichProjectRouter): def __init__(self): - super().__init__(uri_base='project-quota') + super().__init__(uri_base='quota') @Route(methods=[RequestMethods.GET]) - @cherrypy.tools.project_scope() @cherrypy.tools.model_out(cls=ResponseProjectQuota) @cherrypy.tools.enforce_policy(policy_name="projects:quota:get") def get(self): + """Get a project's quota + --- + get: + description: Get a project's quota + tags: + - iam + - project + responses: + 200: + description: The project's quota + """ project: Project = cherrypy.request.project - quota = ProjectQuota.get(project, project.id) + quota = ProjectQuota.get(project.name) return ResponseProjectQuota.from_database(quota) @Route(methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() @cherrypy.tools.model_in(cls=RequestProjectModifyQuota) @cherrypy.tools.enforce_policy(policy_name="projects:quota:modify") def modify(self): + """Modify a project's quota + --- + post: + description: Modify a project's quota + tags: + - iam + - project + requestBody: + description: Quota options + responses: + 204: + description: Quota set + """ cherrypy.response.status = 204 request: RequestProjectModifyQuota = cherrypy.request.model project: Project = cherrypy.request.project - quota: ProjectQuota = ProjectQuota.get(project, project.id) + quota: ProjectQuota = ProjectQuota.get(project.name) quota.vcpu = request.vcpu quota.ram = request.ram diff --git a/deli/counter/http/mounts/root/routes/iam/v1/projects.py b/deli/counter/http/mounts/root/routes/iam/v1/projects.py new file mode 100644 index 0000000..b7b9d49 --- /dev/null +++ b/deli/counter/http/mounts/root/routes/iam/v1/projects.py @@ -0,0 +1,128 @@ +import uuid + +import cherrypy +from ingredients_http.request_methods import RequestMethods +from ingredients_http.route import Route + +from deli.counter.auth.token import Token +from deli.counter.http.mounts.root.routes.iam.v1.validation_models.projects import ResponseProject, \ + RequestCreateProject, \ + ParamsProject, ParamsListProject +from deli.counter.http.router import SandwichRouter +from deli.kubernetes.resources.project import Project +from deli.kubernetes.resources.v1alpha1.iam_policy.model import IAMPolicy +from deli.kubernetes.resources.v1alpha1.iam_role.model import IAMProjectRole +from deli.kubernetes.resources.v1alpha1.iam_service_account.model import ProjectServiceAccount +from deli.kubernetes.resources.v1alpha1.project_quota.model import ProjectQuota + + +class ProjectRouter(SandwichRouter): + def __init__(self): + super().__init__(uri_base='projects') + + @Route(methods=[RequestMethods.POST]) + @cherrypy.tools.model_in(cls=RequestCreateProject) + @cherrypy.tools.model_out(cls=ResponseProject) + @cherrypy.tools.enforce_policy(policy_name="projects:create") + def create(self): + """Create a project + --- + post: + description: Create a project + tags: + - iam + - project + requestBody: + description: Project to create + responses: + 200: + description: The created project + """ + request: RequestCreateProject = cherrypy.request.model + + project = Project.get(request.name) + if project is not None: + raise cherrypy.HTTPError(409, 'A project with the requested name already exists.') + + if request.name == "system": + raise cherrypy.HTTPError(409, 'Cannot use a reserved name as the project name') + + project = Project() + project.name = request.name + project.create() + + IAMProjectRole.create_default_roles(project) + ProjectServiceAccount.create_default_service_account(project) + + quota = ProjectQuota() + quota.name = project.name + quota.create() + + IAMPolicy.create_project_policy(project, cherrypy.request.token) + + return ResponseProject.from_database(project) + + @Route(route='{project_name}') + @cherrypy.tools.model_params(cls=ParamsProject) + @cherrypy.tools.model_out(cls=ResponseProject) + @cherrypy.tools.project_scope() + @cherrypy.tools.resource_object(id_param="project_name", cls=Project) + @cherrypy.tools.enforce_policy(policy_name="projects:get") + def get(self, **_): + """Get a project + --- + get: + description: Get a project + tags: + - iam + - project + responses: + 200: + description: The project + """ + project: Project = cherrypy.request.resource_object + return ResponseProject.from_database(project) + + @Route() + @cherrypy.tools.model_params(cls=ParamsListProject) + @cherrypy.tools.model_out_pagination(cls=ResponseProject) + def list(self, limit: int, marker: uuid.UUID): + """List projects + --- + get: + description: List projects + tags: + - iam + - project + responses: + 200: + description: List of projects + """ + token: Token = cherrypy.request.token + + resp_models = [] + for project in token.get_projects(): + resp_models.append(ResponseProject.from_database(project)) + return resp_models, False + + @Route(route='{project_name}', methods=[RequestMethods.DELETE]) + @cherrypy.tools.model_params(cls=ParamsProject) + @cherrypy.tools.project_scope() + @cherrypy.tools.resource_object(id_param="project_name", cls=Project) + @cherrypy.tools.enforce_policy(policy_name="projects:delete") + def delete(self, **_): + """Delete a project + --- + delete: + description: Delete a project + tags: + - iam + - project + responses: + 204: + description: Project deleted + """ + cherrypy.response.status = 204 + + project: Project = cherrypy.request.resource_object + project.delete() diff --git a/deli/counter/http/mounts/root/routes/iam/v1/role.py b/deli/counter/http/mounts/root/routes/iam/v1/role.py new file mode 100644 index 0000000..9acfd37 --- /dev/null +++ b/deli/counter/http/mounts/root/routes/iam/v1/role.py @@ -0,0 +1,290 @@ +from typing import Optional + +import cherrypy +from ingredients_http.request_methods import RequestMethods +from ingredients_http.route import Route + +from deli.counter.auth.policy import PROJECT_POLICIES, SYSTEM_POLICIES +from deli.counter.http.mounts.root.routes.iam.v1.validation_models.role import RequestCreateRole, ResponseRole, \ + RequestRoleUpdate, ParamsRole, ParamsListRoles +from deli.counter.http.router import SandwichProjectRouter, SandwichSystemRouter +from deli.kubernetes.resources.model import ResourceState +from deli.kubernetes.resources.project import Project +from deli.kubernetes.resources.v1alpha1.iam_role.model import IAMSystemRole, IAMProjectRole + + +class RoleHelper(object): + + def helper_create(self, project: Optional[Project]): + request: RequestCreateRole = cherrypy.request.model + + if project is None: + if IAMSystemRole.get(request.name) is not None: + raise cherrypy.HTTPError(400, 'A system role with the requested name already exists.') + role = IAMSystemRole() + else: + if IAMProjectRole.get(project, request.name) is not None: + raise cherrypy.HTTPError(400, 'A project role with the requested name already exists.') + role = IAMProjectRole() + role.project = project + + if project is not None: + policy_names = [p['name'] for p in PROJECT_POLICIES] + else: + policy_names = [p['name'] for p in SYSTEM_POLICIES] + + for policy in request.policies: + if policy not in policy_names: + raise cherrypy.HTTPError(404, 'Unknown policy %s' % policy) + + role.name = request.name + role.policies = request.policies + role.create() + + return ResponseRole.from_database(role) + + def helper_get(self): + return ResponseRole.from_database(cherrypy.request.resource_object) + + def helper_list(self, project: Optional[Project], limit, marker): + if project is None: + return self.paginate(IAMSystemRole, ResponseRole, limit, marker) + else: + return self.paginate(IAMProjectRole, ResponseRole, limit, marker, project=project) + + def helper_update(self, project: Optional[Project]): + cherrypy.response.status = 204 + request: RequestRoleUpdate = cherrypy.request.model + role = cherrypy.request.resource_object + + if role.name in ['admin', 'viewer', 'editor', 'owner']: + raise cherrypy.HTTPError(409, 'Cannot update the default roles') + + if role.state != ResourceState.Created: + raise cherrypy.HTTPError(400, 'Role is not in the following state: ' + ResourceState.Created.value) + + if project is not None: + policy_names = [p['name'] for p in PROJECT_POLICIES] + else: + policy_names = [p['name'] for p in SYSTEM_POLICIES] + + for policy in request.policies: + if policy not in policy_names: + raise cherrypy.HTTPError(404, 'Unknown policy %s' % policy) + + role.policies = request.policies + role.save() + + def helper_delete(self): + cherrypy.response.status = 204 + role = cherrypy.request.resource_object + + if role.name in ['admin', 'viewer', 'editor', 'owner']: + raise cherrypy.HTTPError(409, 'Cannot delete the default roles') + + if role.state != ResourceState.Created: + raise cherrypy.HTTPError(400, 'Role is not in the following state: ' + ResourceState.Created.value) + + role.delete() + + +class IAMSystemRolesRouter(SandwichSystemRouter, RoleHelper): + def __init__(self): + super().__init__('roles') + + @Route(methods=[RequestMethods.POST]) + @cherrypy.tools.model_in(cls=RequestCreateRole) + @cherrypy.tools.model_out(cls=ResponseRole) + @cherrypy.tools.enforce_policy(policy_name="roles:system:create") + def create(self): + """Create a system role + --- + post: + description: Create a system role + tags: + - iam + - role + requestBody: + description: Role to create + responses: + 200: + description: The created role + """ + return self.helper_create(None) + + @Route(route='{role_name}') + @cherrypy.tools.model_params(cls=ParamsRole) + @cherrypy.tools.model_out(cls=ResponseRole) + @cherrypy.tools.resource_object(id_param="role_name", cls=IAMSystemRole) + @cherrypy.tools.enforce_policy(policy_name="roles:system:get") + def get(self, **_): + """Get a system role + --- + get: + description: Get a system role + tags: + - iam + - role + responses: + 200: + description: The role + """ + return self.helper_get() + + @Route() + @cherrypy.tools.model_params(cls=ParamsListRoles) + @cherrypy.tools.model_out_pagination(cls=ResponseRole) + @cherrypy.tools.enforce_policy(policy_name="roles:system:list") + def list(self, limit, marker): + """List system roles + --- + get: + description: List system roles + tags: + - iam + - image + responses: + 200: + description: List of system roles + """ + return self.helper_list(None, limit, marker) + + @Route(route='{role_name}', methods=[RequestMethods.POST]) + @cherrypy.tools.model_params(cls=ParamsRole) + @cherrypy.tools.model_in(cls=RequestRoleUpdate) + @cherrypy.tools.resource_object(id_param="role_name", cls=IAMSystemRole) + @cherrypy.tools.enforce_policy(policy_name="roles:system:update") + def update(self, **_): + """Update a system role + --- + post: + description: Update a system role + tags: + - iam + - role + requestBody: + description: Role options + responses: + 204: + description: Role updated + """ + return self.helper_update(None) + + @Route(route='{role_name}', methods=[RequestMethods.DELETE]) + @cherrypy.tools.model_params(cls=ParamsRole) + @cherrypy.tools.resource_object(id_param="role_name", cls=IAMSystemRole) + @cherrypy.tools.enforce_policy(policy_name="roles:system:delete") + def delete(self, **_): + """Delete a system role + --- + delete: + description: Delete a system role + tags: + - iam + - image + responses: + 204: + description: Role deleted + """ + return self.helper_delete() + + +class IAMProjectRolesRouter(SandwichProjectRouter, RoleHelper): + def __init__(self): + super().__init__('roles') + + @Route(methods=[RequestMethods.POST]) + @cherrypy.tools.model_in(cls=RequestCreateRole) + @cherrypy.tools.model_out(cls=ResponseRole) + @cherrypy.tools.enforce_policy(policy_name="roles:project:create") + def create(self): + """Create a project role + --- + post: + description: Create a project role + tags: + - iam + - role + requestBody: + description: Role to create + responses: + 200: + description: The created role + """ + return self.helper_create(cherrypy.request.project) + + @Route(route='{role_name}') + @cherrypy.tools.model_params(cls=ParamsRole) + @cherrypy.tools.model_out(cls=ResponseRole) + @cherrypy.tools.resource_object(id_param="role_name", cls=IAMProjectRole) + @cherrypy.tools.enforce_policy(policy_name="roles:project:get") + def get(self, **_): + """Get a project role + --- + get: + description: Get a project role + tags: + - iam + - role + responses: + 200: + description: The role + """ + return self.helper_get() + + @Route() + @cherrypy.tools.model_params(cls=ParamsListRoles) + @cherrypy.tools.model_out_pagination(cls=ResponseRole) + @cherrypy.tools.enforce_policy(policy_name="roles:project:list") + def list(self, limit, marker): + """List project roles + --- + get: + description: List project roles + tags: + - iam + - image + responses: + 200: + description: List of project roles + """ + return self.helper_list(cherrypy.request.project, limit, marker) + + @Route(route='{role_name}', methods=[RequestMethods.POST]) + @cherrypy.tools.model_params(cls=ParamsRole) + @cherrypy.tools.model_in(cls=RequestRoleUpdate) + @cherrypy.tools.resource_object(id_param="role_name", cls=IAMProjectRole) + @cherrypy.tools.enforce_policy(policy_name="roles:project:update") + def update(self, **_): + """Update a project role + --- + post: + description: Update a project role + tags: + - iam + - role + requestBody: + description: Role options + responses: + 204: + description: Role updated + """ + return self.helper_update(cherrypy.request.project) + + @Route(route='{role_name}', methods=[RequestMethods.DELETE]) + @cherrypy.tools.model_params(cls=ParamsRole) + @cherrypy.tools.resource_object(id_param="role_name", cls=IAMProjectRole) + @cherrypy.tools.enforce_policy(policy_name="roles:project:delete") + def delete(self, **_): + """Delete a project role + --- + delete: + description: Delete a project role + tags: + - iam + - image + responses: + 204: + description: Role deleted + """ + return self.helper_delete() diff --git a/deli/counter/http/mounts/root/routes/v1/service_accounts.py b/deli/counter/http/mounts/root/routes/iam/v1/service_accounts.py similarity index 52% rename from deli/counter/http/mounts/root/routes/v1/service_accounts.py rename to deli/counter/http/mounts/root/routes/iam/v1/service_accounts.py index 675cf6f..d0534e5 100644 --- a/deli/counter/http/mounts/root/routes/v1/service_accounts.py +++ b/deli/counter/http/mounts/root/routes/iam/v1/service_accounts.py @@ -1,21 +1,21 @@ from typing import Optional +import arrow import cherrypy from ingredients_http.request_methods import RequestMethods from ingredients_http.route import Route from deli.counter.auth.token import Token -from deli.counter.http.mounts.root.routes.v1.auth.validation_models.tokens import ResponseOAuthToken -from deli.counter.http.mounts.root.routes.v1.validation_models.service_accounts import ResponseServiceAccount, \ - RequestCreateServiceAccount, ParamsServiceAccount, ParamsListServiceAccount, RequestUpdateServiceAccount, \ - RequestCreateServiceAccountKey, ParamsServiceAccountKey -from deli.counter.http.router import SandwichRouter +from deli.counter.http.mounts.root.routes.auth.v1.validation_models.oauth import ResponseOAuthToken +from deli.counter.http.mounts.root.routes.iam.v1.validation_models.service_accounts import ResponseServiceAccount, \ + RequestCreateServiceAccount, ParamsServiceAccount, ParamsListServiceAccount, RequestCreateServiceAccountKey, \ + ParamsServiceAccountKey +from deli.counter.http.router import SandwichProjectRouter, SandwichSystemRouter from deli.kubernetes.resources.const import SERVICE_ACCOUNT_LABEL from deli.kubernetes.resources.model import ResourceState from deli.kubernetes.resources.project import Project +from deli.kubernetes.resources.v1alpha1.iam_service_account.model import ProjectServiceAccount, SystemServiceAccount from deli.kubernetes.resources.v1alpha1.instance.model import Instance -from deli.kubernetes.resources.v1alpha1.role.model import ProjectRole, GlobalRole -from deli.kubernetes.resources.v1alpha1.service_account.model import ProjectServiceAccount, GlobalServiceAccount class ServiceAccountHelper(object): @@ -24,17 +24,17 @@ def helper_create(self, project: Optional[Project]): request: RequestCreateServiceAccount = cherrypy.request.model if project is None: - service_account = GlobalServiceAccount.get_by_name(request.name) + service_account = SystemServiceAccount.get(request.name) if service_account is not None: - raise cherrypy.HTTPError(400, 'A global service account with the requested name already exists.') - service_account = GlobalServiceAccount() + raise cherrypy.HTTPError(400, 'A system service account with the requested name already exists.') + service_account = SystemServiceAccount() else: - service_account = ProjectServiceAccount.get_by_name(project, request.name) + service_account = ProjectServiceAccount.get(project, request.name) if service_account is not None: raise cherrypy.HTTPError(400, 'A project service account with the requested name already exists.') service_account = ProjectServiceAccount() service_account.project = project - service_account.roles = [ProjectRole.get_by_name(project, 'default-service-account')] + service_account.roles = [] service_account.name = request.name service_account.create() @@ -45,42 +45,10 @@ def helper_get(self): def helper_list(self, project: Optional[Project], limit, marker): if project is None: - return self.paginate(GlobalServiceAccount, ResponseServiceAccount, limit, marker) + return self.paginate(SystemServiceAccount, ResponseServiceAccount, limit, marker) else: return self.paginate(ProjectServiceAccount, ResponseServiceAccount, limit, marker, project=project) - def helper_update(self, project: Optional[Project]): - cherrypy.response.status = 204 - request: RequestUpdateServiceAccount = cherrypy.request.model - service_account = cherrypy.request.resource_object - - if project is not None and service_account.name == "default": - raise cherrypy.HTTPError(409, 'Cannot update the default service account.') - - if project is None and service_account.name == "admin": - raise cherrypy.HTTPError(409, 'Cannot update the admin service account.') - - if service_account.state != ResourceState.Created: - raise cherrypy.HTTPError(400, - 'Service Account member is not in the following state: ' - + ResourceState.Created.value) - - roles = [] - for role_id in request.roles: - if project is None: - role = GlobalRole.get(project, role_id) - if role is None: - raise cherrypy.HTTPError(404, 'A global role with the requested id of %s does not exist.' % role_id) - else: - role = ProjectRole.get(project, role_id) - if role is None: - raise cherrypy.HTTPError(404, - 'A project role with the requested id of %s does not exist.' % role_id) - roles.append(role) - - service_account.roles = roles - service_account.save() - def helper_delete(self, project: Optional[Project]): cherrypy.response.status = 204 service_account = cherrypy.request.resource_object @@ -98,7 +66,7 @@ def helper_delete(self, project: Optional[Project]): raise cherrypy.HTTPError(400, "Service Account has already been deleted") if project is not None: - instances = Instance.list_all(label_selector=SERVICE_ACCOUNT_LABEL + "=" + str(service_account.id)) + instances = Instance.list(project, label_selector=SERVICE_ACCOUNT_LABEL + "=" + str(service_account.name)) if len(instances) > 0: raise cherrypy.HTTPError(409, 'Cannot delete a service account while it is in use by an instance') @@ -120,19 +88,18 @@ def helper_create_key(self, project: Optional[Project]): 'Service Account member is not in the following state: ' + ResourceState.Created.value) - service_account.keys = service_account.keys + [request.name] + keys = service_account.keys + keys[request.name] = arrow.now('UTC').shift(years=+10) + service_account.keys = keys service_account.save() token = Token() - token.driver_name = 'metadata' - token.service_account_id = service_account.id - token.service_account_key = request.name - if project is not None: - token.project_id = project.id + token.email = service_account.email + token.metadata['key'] = request.name response = ResponseOAuthToken() response.access_token = token.marshal(self.mount.fernet) - response.expiry = None + response.expiry = keys[request.name] return response def helper_delete_key(self, name, project: Optional[Project]): @@ -148,129 +115,246 @@ def helper_delete_key(self, name, project: Optional[Project]): raise cherrypy.HTTPError(409, 'Cannot delete keys for the admin service account.') keys = service_account.keys - keys.remove(name) + del keys[name] service_account.keys = keys service_account.save() -class GlobalServiceAccountsRouter(SandwichRouter, ServiceAccountHelper): +class SystemServiceAccountsRouter(SandwichSystemRouter, ServiceAccountHelper): def __init__(self): - super().__init__(uri_base='global-service-accounts') + super().__init__(uri_base='service-accounts') @Route(methods=[RequestMethods.POST]) @cherrypy.tools.model_in(cls=RequestCreateServiceAccount) @cherrypy.tools.model_out(cls=ResponseServiceAccount) - @cherrypy.tools.enforce_policy(policy_name="service_accounts:global:create") + @cherrypy.tools.enforce_policy(policy_name="service_accounts:system:create") def create(self): + """Create a system service account + --- + post: + description: Create a system service account + tags: + - iam + - service-account + requestBody: + description: Service Account to create + responses: + 200: + description: The created service account + """ return self.helper_create(None) - @Route(route='{service_account_id}') + @Route(route='{service_account_name}') @cherrypy.tools.model_params(cls=ParamsServiceAccount) @cherrypy.tools.model_out(cls=ResponseServiceAccount) - @cherrypy.tools.resource_object(id_param="service_account_id", cls=GlobalServiceAccount) - @cherrypy.tools.enforce_policy(policy_name="service_accounts:global:get") + @cherrypy.tools.resource_object(id_param="service_account_name", cls=SystemServiceAccount) + @cherrypy.tools.enforce_policy(policy_name="service_accounts:system:get") def get(self, **_): + """Get a system service account + --- + get: + description: Get a system service account + tags: + - iam + - service-account + responses: + 200: + description: The service account + """ return self.helper_get() @Route() @cherrypy.tools.model_params(cls=ParamsListServiceAccount) @cherrypy.tools.model_out_pagination(cls=ResponseServiceAccount) - @cherrypy.tools.enforce_policy(policy_name="service_accounts:global:list") + @cherrypy.tools.enforce_policy(policy_name="service_accounts:system:list") def list(self, limit, marker): + """List system service accounts + --- + get: + description: List system service accounts + tags: + - iam + - service-account + responses: + 200: + description: List of service accounts + """ return self.helper_list(None, limit, marker) - @Route(route='{service_account_id}', methods=[RequestMethods.POST]) + @Route(route='{service_account_name}', methods=[RequestMethods.DELETE]) @cherrypy.tools.model_params(cls=ParamsServiceAccount) - @cherrypy.tools.model_in(cls=RequestUpdateServiceAccount) - @cherrypy.tools.resource_object(id_param="service_account_id", cls=GlobalServiceAccount) - @cherrypy.tools.enforce_policy(policy_name="service_accounts:global:update") - def update(self, **_): - self.helper_update(None) - - @Route(route='{service_account_id}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.model_params(cls=ParamsServiceAccount) - @cherrypy.tools.resource_object(id_param="service_account_id", cls=GlobalServiceAccount) - @cherrypy.tools.enforce_policy(policy_name="service_accounts:global:delete") + @cherrypy.tools.resource_object(id_param="service_account_name", cls=SystemServiceAccount) + @cherrypy.tools.enforce_policy(policy_name="service_accounts:system:delete") def delete(self, **_): + """Delete a system service account + --- + delete: + description: Delete a system service account + tags: + - iam + - service-account + responses: + 204: + description: Service Account deleted + """ self.helper_delete(None) - @Route(route='{service_account_id}/keys', methods=[RequestMethods.POST]) + @Route(route='{service_account_name}/keys', methods=[RequestMethods.POST]) @cherrypy.tools.model_params(cls=ParamsServiceAccount) @cherrypy.tools.model_in(cls=RequestCreateServiceAccountKey) @cherrypy.tools.model_out(cls=ResponseOAuthToken) - @cherrypy.tools.resource_object(id_param="service_account_id", cls=GlobalServiceAccount) - @cherrypy.tools.enforce_policy(policy_name="service_accounts:global:key:create") + @cherrypy.tools.resource_object(id_param="service_account_name", cls=SystemServiceAccount) + @cherrypy.tools.enforce_policy(policy_name="service_accounts:system:key:create") def create_key(self, **_): + """Create a system service account key + --- + post: + description: Create a system service account key + tags: + - iam + - service-account + requestBody: + description: Service Account to create + responses: + 200: + description: The created key + """ return self.helper_create_key(None) - @Route(route='{service_account_id}/keys/{name}', methods=[RequestMethods.DELETE]) + @Route(route='{service_account_name}/keys/{name}', methods=[RequestMethods.DELETE]) @cherrypy.tools.model_params(cls=ParamsServiceAccountKey) - @cherrypy.tools.resource_object(id_param="service_account_id", cls=GlobalServiceAccount) - @cherrypy.tools.enforce_policy(policy_name="service_accounts:global:key:delete") + @cherrypy.tools.resource_object(id_param="service_account_name", cls=SystemServiceAccount) + @cherrypy.tools.enforce_policy(policy_name="service_accounts:system:key:delete") def delete_key(self, **kwargs): + """Delete a system service account key + --- + delete: + description: Delete a system service account key + tags: + - iam + - service-account + responses: + 204: + description: Service Account key deleted + """ return self.helper_delete_key(name=kwargs['name'], project=None) -class ProjectServiceAccountsRouter(SandwichRouter, ServiceAccountHelper): +class ProjectServiceAccountsRouter(SandwichProjectRouter, ServiceAccountHelper): def __init__(self): - super().__init__(uri_base='project-service-accounts') + super().__init__(uri_base='service-accounts') @Route(methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() @cherrypy.tools.model_in(cls=RequestCreateServiceAccount) @cherrypy.tools.model_out(cls=ResponseServiceAccount) @cherrypy.tools.enforce_policy(policy_name="service_accounts:project:create") def create(self): + """Create a project service account + --- + post: + description: Create a project service account + tags: + - iam + - service-account + requestBody: + description: Service Account to create + responses: + 200: + description: The created service account + """ return self.helper_create(cherrypy.request.project) - @Route(route='{service_account_id}') - @cherrypy.tools.project_scope() + @Route(route='{service_account_name}') @cherrypy.tools.model_params(cls=ParamsServiceAccount) @cherrypy.tools.model_out(cls=ResponseServiceAccount) - @cherrypy.tools.resource_object(id_param="service_account_id", cls=ProjectServiceAccount) + @cherrypy.tools.resource_object(id_param="service_account_name", cls=ProjectServiceAccount) @cherrypy.tools.enforce_policy(policy_name="service_accounts:project:get") def get(self, **_): + """Get a project service account + --- + get: + description: Get a project service account + tags: + - iam + - service-account + responses: + 200: + description: The service account + """ return self.helper_get() @Route() - @cherrypy.tools.project_scope() @cherrypy.tools.model_params(cls=ParamsListServiceAccount) @cherrypy.tools.model_out_pagination(cls=ResponseServiceAccount) @cherrypy.tools.enforce_policy(policy_name="service_accounts:project:list") def list(self, limit, marker): + """List project service accounts + --- + get: + description: List project service accounts + tags: + - iam + - service-account + responses: + 200: + description: List of service accounts + """ return self.helper_list(cherrypy.request.project, limit, marker) - @Route(route='{service_account_id}', methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsServiceAccount) - @cherrypy.tools.model_in(cls=RequestUpdateServiceAccount) - @cherrypy.tools.resource_object(id_param="service_account_id", cls=ProjectServiceAccount) - @cherrypy.tools.enforce_policy(policy_name="service_accounts:project:update") - def update(self, **_): - self.helper_update(cherrypy.request.project) - - @Route(route='{service_account_id}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.project_scope() + @Route(route='{service_account_name}', methods=[RequestMethods.DELETE]) @cherrypy.tools.model_params(cls=ParamsServiceAccount) - @cherrypy.tools.resource_object(id_param="service_account_id", cls=ProjectServiceAccount) + @cherrypy.tools.resource_object(id_param="service_account_name", cls=ProjectServiceAccount) @cherrypy.tools.enforce_policy(policy_name="service_accounts:project:delete") def delete(self, **_): + """Delete a project service account + --- + delete: + description: Delete a project service account + tags: + - iam + - service-account + responses: + 204: + description: Service Account deleted + """ self.helper_delete(cherrypy.request.project) - @Route(route='{service_account_id}/keys', methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() + @Route(route='{service_account_name}/keys', methods=[RequestMethods.POST]) @cherrypy.tools.model_params(cls=ParamsServiceAccount) @cherrypy.tools.model_in(cls=RequestCreateServiceAccountKey) @cherrypy.tools.model_out(cls=ResponseOAuthToken) - @cherrypy.tools.resource_object(id_param="service_account_id", cls=ProjectServiceAccount) + @cherrypy.tools.resource_object(id_param="service_account_name", cls=ProjectServiceAccount) @cherrypy.tools.enforce_policy(policy_name="service_accounts:project:key:create") def create_key(self, **_): + """Create a project service account key + --- + post: + description: Create a project service account key + tags: + - iam + - service-account + requestBody: + description: Service Account to create + responses: + 200: + description: The created key + """ return self.helper_create_key(cherrypy.request.project) - @Route(route='{service_account_id}/keys/{name}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.project_scope() + @Route(route='{service_account_name}/keys/{name}', methods=[RequestMethods.DELETE]) @cherrypy.tools.model_params(cls=ParamsServiceAccountKey) - @cherrypy.tools.resource_object(id_param="service_account_id", cls=ProjectServiceAccount) + @cherrypy.tools.resource_object(id_param="service_account_name", cls=ProjectServiceAccount) @cherrypy.tools.enforce_policy(policy_name="service_accounts:project:key:delete") def delete_key(self, **kwargs): + """Delete a project service account key + --- + delete: + description: Delete a project service account key + tags: + - iam + - service-account + responses: + 204: + description: Service Account key deleted + """ return self.helper_delete_key(name=kwargs['name'], project=cherrypy.request.project) diff --git a/deli/kubernetes/resources/v1alpha1/role/__init__.py b/deli/counter/http/mounts/root/routes/iam/v1/validation_models/__init__.py similarity index 100% rename from deli/kubernetes/resources/v1alpha1/role/__init__.py rename to deli/counter/http/mounts/root/routes/iam/v1/validation_models/__init__.py diff --git a/deli/counter/http/mounts/root/routes/v1/auth/validation_models/policy.py b/deli/counter/http/mounts/root/routes/iam/v1/validation_models/policy.py similarity index 100% rename from deli/counter/http/mounts/root/routes/v1/auth/validation_models/policy.py rename to deli/counter/http/mounts/root/routes/iam/v1/validation_models/policy.py diff --git a/deli/counter/http/mounts/root/routes/iam/v1/validation_models/projects.py b/deli/counter/http/mounts/root/routes/iam/v1/validation_models/projects.py new file mode 100644 index 0000000..a2dc70d --- /dev/null +++ b/deli/counter/http/mounts/root/routes/iam/v1/validation_models/projects.py @@ -0,0 +1,76 @@ +import re + +from ingredients_http.schematics.types import ArrowType +from schematics import Model +from schematics.exceptions import ValidationError +from schematics.types import UUIDType, IntType, StringType + +from deli.kubernetes.resources.project import Project +from deli.kubernetes.resources.v1alpha1.project_quota.model import ProjectQuota + + +class ProjectName(StringType): + + def __init__(self, **kwargs): + super().__init__(max_length=54, **kwargs) + self.k8s_reg = re.compile('^(([A-Za-z0-9][-A-Za-z0-9_]*)?[A-Za-z0-9])?$') + + def validate_kube(self, value, context=None): + if self.k8s_reg.match(value) is None: + raise ValidationError( + "must consist of lower case alphanumeric characters, the optional character '-', and must start and " + "end with an alphanumeric character (e.g. 'example', regex used for validation " + "is '(([A-Za-z0-9][-A-Za-z0-9_]*)?[A-Za-z0-9])?')") + + +class RequestCreateProject(Model): + name = ProjectName(required=True, min_length=3) + + +class ParamsProject(Model): + project_name = ProjectName(required=True) + + +class ParamsListProject(Model): + limit = IntType(default=100, max_value=100, min_value=1) + marker = UUIDType() + + +class ResponseProject(Model): + name = ProjectName(required=True, min_length=3) + created_at = ArrowType(required=True) + + @classmethod + def from_database(cls, project: Project): + project_model = cls() + project_model.name = project.name + project_model.created_at = project.created_at + + return project_model + + +class RequestProjectModifyQuota(Model): + vcpu = IntType(required=True, min_value=-1) + ram = IntType(required=True, min_value=-1) + disk = IntType(required=True, min_value=-1) + + +class ResponseProjectQuota(Model): + vcpu = IntType(required=True) + ram = IntType(required=True) + disk = IntType(required=True) + used_vcpu = IntType(required=True) + used_ram = IntType(required=True) + used_disk = IntType(required=True) + + @classmethod + def from_database(cls, quota: ProjectQuota): + model = cls() + model.vcpu = quota.vcpu + model.ram = quota.ram + model.disk = quota.disk + model.used_vcpu = quota.used_vcpu + model.used_ram = quota.used_ram + model.used_disk = quota.used_disk + + return model diff --git a/deli/counter/http/mounts/root/routes/v1/auth/validation_models/role.py b/deli/counter/http/mounts/root/routes/iam/v1/validation_models/role.py similarity index 91% rename from deli/counter/http/mounts/root/routes/v1/auth/validation_models/role.py rename to deli/counter/http/mounts/root/routes/iam/v1/validation_models/role.py index 6f96b1b..acda168 100644 --- a/deli/counter/http/mounts/root/routes/v1/auth/validation_models/role.py +++ b/deli/counter/http/mounts/root/routes/iam/v1/validation_models/role.py @@ -11,7 +11,7 @@ class RequestCreateRole(Model): class ParamsRole(Model): - role_id = UUIDType(required=True) + role_name = UUIDType(required=True) class ParamsListRoles(Model): @@ -24,7 +24,6 @@ class RequestRoleUpdate(Model): class ResponseRole(Model): - id = UUIDType(required=True) name = KubeName(required=True, min_length=3) policies = ListType(StringType, default=list) state = EnumType(ResourceState, required=True) @@ -33,7 +32,6 @@ class ResponseRole(Model): @classmethod def from_database(cls, role): model = cls() - model.id = role.id model.name = role.name model.policies = role.policies model.state = role.state diff --git a/deli/counter/http/mounts/root/routes/v1/validation_models/service_accounts.py b/deli/counter/http/mounts/root/routes/iam/v1/validation_models/service_accounts.py similarity index 69% rename from deli/counter/http/mounts/root/routes/v1/validation_models/service_accounts.py rename to deli/counter/http/mounts/root/routes/iam/v1/validation_models/service_accounts.py index b72e26c..fddf9cb 100644 --- a/deli/counter/http/mounts/root/routes/v1/validation_models/service_accounts.py +++ b/deli/counter/http/mounts/root/routes/iam/v1/validation_models/service_accounts.py @@ -1,6 +1,6 @@ from ingredients_http.schematics.types import KubeName, ArrowType, EnumType from schematics import Model -from schematics.types import UUIDType, ListType, StringType, IntType +from schematics.types import UUIDType, StringType, IntType, EmailType, DictType from deli.kubernetes.resources.model import ResourceState @@ -9,12 +9,8 @@ class RequestCreateServiceAccount(Model): name = KubeName(required=True, min_length=3) -class RequestUpdateServiceAccount(Model): - roles = ListType(StringType, required=True) - - class ParamsServiceAccount(Model): - service_account_id = UUIDType(required=True) + service_account_name = KubeName(required=True) class ParamsListServiceAccount(Model): @@ -23,19 +19,17 @@ class ParamsListServiceAccount(Model): class ResponseServiceAccount(Model): - id = UUIDType(required=True) name = KubeName(required=True, min_length=3) - roles = ListType(UUIDType, required=True) - keys = ListType(StringType, required=True) + email = EmailType(required=True) + keys = DictType(ArrowType, required=True) state = EnumType(ResourceState, required=True) created_at = ArrowType(required=True) @classmethod def from_database(cls, service_account): model = cls() - model.id = service_account.id model.name = service_account.name - model.roles = service_account.role_ids + model.email = service_account.email model.keys = service_account.keys model.state = service_account.state model.created_at = service_account.created_at @@ -48,5 +42,5 @@ class RequestCreateServiceAccountKey(Model): class ParamsServiceAccountKey(Model): - service_account_id = UUIDType(required=True) + service_account_name = KubeName(required=True) name = StringType(required=True, min_length=3) diff --git a/deli/kubernetes/resources/v1alpha1/service_account/__init__.py b/deli/counter/http/mounts/root/routes/location/__init__.py similarity index 100% rename from deli/kubernetes/resources/v1alpha1/service_account/__init__.py rename to deli/counter/http/mounts/root/routes/location/__init__.py diff --git a/deli/menu/models/__init__.py b/deli/counter/http/mounts/root/routes/location/v1/__init__.py similarity index 100% rename from deli/menu/models/__init__.py rename to deli/counter/http/mounts/root/routes/location/v1/__init__.py diff --git a/deli/counter/http/mounts/root/routes/v1/regions.py b/deli/counter/http/mounts/root/routes/location/v1/regions.py similarity index 59% rename from deli/counter/http/mounts/root/routes/v1/regions.py rename to deli/counter/http/mounts/root/routes/location/v1/regions.py index e039d34..541ae6e 100644 --- a/deli/counter/http/mounts/root/routes/v1/regions.py +++ b/deli/counter/http/mounts/root/routes/location/v1/regions.py @@ -2,10 +2,10 @@ from ingredients_http.request_methods import RequestMethods from ingredients_http.route import Route -from deli.counter.http.mounts.root.routes.v1.validation_models.regions import ResponseRegion, RequestCreateRegion, \ +from deli.counter.http.mounts.root.routes.location.v1.validation_models.regions import ResponseRegion, \ + RequestCreateRegion, \ ParamsRegion, ParamsListRegion, RequestRegionSchedule from deli.counter.http.router import SandwichRouter -from deli.kubernetes.resources.const import NAME_LABEL from deli.kubernetes.resources.model import ResourceState from deli.kubernetes.resources.v1alpha1.region.model import Region @@ -19,9 +19,22 @@ def __init__(self): @cherrypy.tools.model_out(cls=ResponseRegion) @cherrypy.tools.enforce_policy(policy_name="regions:create") def create(self): + """Create a region + --- + post: + description: Create a region + tags: + - location + - region + requestBody: + description: Region to create + responses: + 200: + description: The created region + """ request: RequestCreateRegion = cherrypy.request.model - region = Region.get_by_name(request.name) + region = Region.get(request.name) if region is not None: raise cherrypy.HTTPError(409, 'A region with the requested name already exists.') @@ -38,37 +51,65 @@ def create(self): return ResponseRegion.from_database(region) - @Route(route='{region_id}') + @Route(route='{region_name}') @cherrypy.tools.model_params(cls=ParamsRegion) @cherrypy.tools.model_out(cls=ResponseRegion) - @cherrypy.tools.resource_object(id_param="region_id", cls=Region) - @cherrypy.tools.enforce_policy(policy_name="regions:get") + @cherrypy.tools.resource_object(id_param="region_name", cls=Region) def get(self, **_): + """Get a region + --- + get: + description: Get a region + tags: + - location + - region + responses: + 200: + description: The region + """ return ResponseRegion.from_database(cherrypy.request.resource_object) @Route() @cherrypy.tools.model_params(cls=ParamsListRegion) @cherrypy.tools.model_out_pagination(cls=ResponseRegion) - @cherrypy.tools.enforce_policy(policy_name="regions:list") - def list(self, name, limit, marker): + def list(self, limit, marker): + """List regions + --- + get: + description: List regions + tags: + - location + - region + responses: + 200: + description: List of regions + """ kwargs = { 'label_selector': [] } - if name is not None: - kwargs['label_selector'].append(NAME_LABEL + '=' + name) - if len(kwargs['label_selector']) > 0: kwargs['label_selector'] = ",".join(kwargs['label_selector']) else: del kwargs['label_selector'] return self.paginate(Region, ResponseRegion, limit, marker, **kwargs) - @Route(route='{region_id}', methods=[RequestMethods.DELETE]) + @Route(route='{region_name}', methods=[RequestMethods.DELETE]) @cherrypy.tools.model_params(cls=ParamsRegion) - @cherrypy.tools.resource_object(id_param="region_id", cls=Region) + @cherrypy.tools.resource_object(id_param="region_name", cls=Region) @cherrypy.tools.enforce_policy(policy_name="regions:delete") def delete(self, **_): + """Delete a region + --- + delete: + description: Delete a region + tags: + - location + - region + responses: + 204: + description: Region deleted + """ cherrypy.response.status = 204 region: Region = cherrypy.request.resource_object @@ -84,12 +125,25 @@ def delete(self, **_): region.delete() - @Route(route='{region_id}/action/schedule', methods=[RequestMethods.PUT]) + @Route(route='{region_name}/action/schedule', methods=[RequestMethods.PUT]) @cherrypy.tools.model_params(cls=ParamsRegion) @cherrypy.tools.model_in(cls=RequestRegionSchedule) - @cherrypy.tools.resource_object(id_param="region_id", cls=Region) + @cherrypy.tools.resource_object(id_param="region_name", cls=Region) @cherrypy.tools.enforce_policy(policy_name="regions:action:schedule") def action_schedule(self, **_): + """Allow or disallow a region to be scheduled + --- + put: + description: Allow or disallow a region to be scheduled + tags: + - location + - region + requestBody: + description: Region schedule options + responses: + 204: + description: Region schedule changed + """ cherrypy.response.status = 204 request: RequestRegionSchedule = cherrypy.request.model diff --git a/deli/menu/vspc/__init__.py b/deli/counter/http/mounts/root/routes/location/v1/validation_models/__init__.py similarity index 100% rename from deli/menu/vspc/__init__.py rename to deli/counter/http/mounts/root/routes/location/v1/validation_models/__init__.py diff --git a/deli/counter/http/mounts/root/routes/v1/validation_models/regions.py b/deli/counter/http/mounts/root/routes/location/v1/validation_models/regions.py similarity index 91% rename from deli/counter/http/mounts/root/routes/v1/validation_models/regions.py rename to deli/counter/http/mounts/root/routes/location/v1/validation_models/regions.py index a2e6a15..c582d1c 100644 --- a/deli/counter/http/mounts/root/routes/v1/validation_models/regions.py +++ b/deli/counter/http/mounts/root/routes/location/v1/validation_models/regions.py @@ -14,11 +14,11 @@ class RequestCreateRegion(Model): class ParamsRegion(Model): - region_id = UUIDType(required=True) + region_name = KubeName(required=True) class ParamsListRegion(Model): - name = KubeName() + region_name = KubeName() limit = IntType(default=100, max_value=100, min_value=1) marker = UUIDType() @@ -28,7 +28,6 @@ class RequestRegionSchedule(Model): class ResponseRegion(Model): - id = UUIDType(required=True) name = KubeName(required=True, min_length=3) datacenter = StringType(required=True, ) image_datastore = StringType(required=True) @@ -37,11 +36,11 @@ class ResponseRegion(Model): state = EnumType(ResourceState, required=True) error_message = StringType() created_at = ArrowType(required=True) + updated_at = ArrowType(required=True) @classmethod def from_database(cls, region: Region): region_model = cls() - region_model.id = region.id region_model.name = region.name region_model.datacenter = region.datacenter region_model.image_datastore = region.image_datastore @@ -53,5 +52,6 @@ def from_database(cls, region: Region): region_model.error_message = region.error_message region_model.created_at = region.created_at + region_model.updated_at = region.updated_at return region_model diff --git a/deli/counter/http/mounts/root/routes/v1/validation_models/zones.py b/deli/counter/http/mounts/root/routes/location/v1/validation_models/zones.py similarity index 87% rename from deli/counter/http/mounts/root/routes/v1/validation_models/zones.py rename to deli/counter/http/mounts/root/routes/location/v1/validation_models/zones.py index a030908..6c6eb13 100644 --- a/deli/counter/http/mounts/root/routes/v1/validation_models/zones.py +++ b/deli/counter/http/mounts/root/routes/location/v1/validation_models/zones.py @@ -8,7 +8,7 @@ class RequestCreateZone(Model): name = KubeName(required=True, min_length=3) - region_id = UUIDType(required=True) + region_name = KubeName(required=True) vm_cluster = StringType(required=True) vm_datastore = StringType(required=True) vm_folder = StringType() @@ -17,11 +17,11 @@ class RequestCreateZone(Model): class ParamsZone(Model): - zone_id = UUIDType(required=True) + zone_name = KubeName(required=True) class ParamsListZone(Model): - region_id = KubeName() + region_name = KubeName() limit = IntType(default=100, max_value=100, min_value=1) marker = UUIDType() @@ -31,9 +31,8 @@ class RequestZoneSchedule(Model): class ResponseZone(Model): - id = UUIDType(required=True) name = KubeName(required=True, min_length=3) - region_id = UUIDType() + region_name = KubeName() vm_cluster = StringType(required=True) vm_datastore = StringType(required=True) vm_folder = StringType() @@ -43,13 +42,13 @@ class ResponseZone(Model): state = EnumType(ResourceState, required=True) error_message = StringType() created_at = ArrowType(required=True) + updated_at = ArrowType(required=True) @classmethod def from_database(cls, zone: Zone): zone_model = cls() - zone_model.id = zone.id zone_model.name = zone.name - zone_model.region_id = zone.region_id + zone_model.region_name = zone.region_name zone_model.vm_cluster = zone.vm_cluster zone_model.vm_datastore = zone.vm_datastore zone_model.vm_folder = zone.vm_folder @@ -61,5 +60,6 @@ def from_database(cls, zone: Zone): if zone.error_message != "": zone_model.error_message = zone.error_message zone_model.created_at = zone.created_at + zone_model.updated_at = zone.updated_at return zone_model diff --git a/deli/counter/http/mounts/root/routes/v1/zones.py b/deli/counter/http/mounts/root/routes/location/v1/zones.py similarity index 60% rename from deli/counter/http/mounts/root/routes/v1/zones.py rename to deli/counter/http/mounts/root/routes/location/v1/zones.py index 3a9b368..cadb11d 100644 --- a/deli/counter/http/mounts/root/routes/v1/zones.py +++ b/deli/counter/http/mounts/root/routes/location/v1/zones.py @@ -2,7 +2,7 @@ from ingredients_http.request_methods import RequestMethods from ingredients_http.route import Route -from deli.counter.http.mounts.root.routes.v1.validation_models.zones import RequestCreateZone, ResponseZone, \ +from deli.counter.http.mounts.root.routes.location.v1.validation_models.zones import RequestCreateZone, ResponseZone, \ ParamsZone, ParamsListZone, RequestZoneSchedule from deli.counter.http.router import SandwichRouter from deli.kubernetes.resources.model import ResourceState @@ -19,15 +19,28 @@ def __init__(self): @cherrypy.tools.model_out(cls=ResponseZone) @cherrypy.tools.enforce_policy(policy_name="zones:create") def create(self): + """Create a zone + --- + post: + description: Create a zone + tags: + - location + - zone + requestBody: + description: Zone to create + responses: + 200: + description: The created zone + """ request: RequestCreateZone = cherrypy.request.model - zone = Zone.get_by_name(request.name) + zone = Zone.get(request.name) if zone is not None: raise cherrypy.HTTPError(409, 'A zone with the requested name already exists.') - region = Region.get(request.region_id) + region = Region.get(request.region_name) if region is None: - raise cherrypy.HTTPError(404, 'A region with the requested id does not exist.') + raise cherrypy.HTTPError(404, 'A region with the requested name does not exist.') if region.state != ResourceState.Created: raise cherrypy.HTTPError(400, 'Can only create a zone with a region in the following state: %s'.format( @@ -49,35 +62,65 @@ def create(self): return ResponseZone.from_database(zone) - @Route(route='{zone_id}') + @Route(route='{zone_name}') @cherrypy.tools.model_params(cls=ParamsZone) @cherrypy.tools.model_out(cls=ResponseZone) - @cherrypy.tools.resource_object(id_param="zone_id", cls=Zone) - @cherrypy.tools.enforce_policy(policy_name="zones:get") + @cherrypy.tools.resource_object(id_param="zone_name", cls=Zone) def get(self, **_): + """Get a zone + --- + get: + description: Get a zone + tags: + - location + - zone + responses: + 200: + description: The zone + """ return ResponseZone.from_database(cherrypy.request.resource_object) @Route() @cherrypy.tools.model_params(cls=ParamsListZone) @cherrypy.tools.model_out_pagination(cls=ResponseZone) - @cherrypy.tools.enforce_policy(policy_name="zones:list") - def list(self, region_id, limit, marker): - + def list(self, region_name, limit, marker): + """List zones + --- + get: + description: List zones + tags: + - location + - zone + responses: + 200: + description: List of zones + """ kwargs = {} - if region_id is not None: - region: Region = Region.get(region_id) + if region_name is not None: + region: Region = Region.get(region_name) if region is None: - raise cherrypy.HTTPError(404, "A region with the requested id does not exist.") + raise cherrypy.HTTPError(404, "A region with the requested name does not exist.") kwargs['label_selector'] = 'sandwichcloud.io/region=' + region.name return self.paginate(Zone, ResponseZone, limit, marker, **kwargs) - @Route(route='{zone_id}', methods=[RequestMethods.DELETE]) + @Route(route='{zone_name}', methods=[RequestMethods.DELETE]) @cherrypy.tools.model_params(cls=ParamsZone) - @cherrypy.tools.resource_object(id_param="zone_id", cls=Zone) + @cherrypy.tools.resource_object(id_param="zone_name", cls=Zone) @cherrypy.tools.enforce_policy(policy_name="zones:delete") def delete(self, **_): + """Delete a zone + --- + delete: + description: Delete a zone + tags: + - location + - zone + responses: + 204: + description: Zone deleted + """ cherrypy.response.status = 204 zone: Zone = cherrypy.request.resource_object @@ -92,12 +135,25 @@ def delete(self, **_): zone.delete() - @Route(route='{zone_id}/action/schedule', methods=[RequestMethods.PUT]) + @Route(route='{zone_name}/action/schedule', methods=[RequestMethods.PUT]) @cherrypy.tools.model_params(cls=ParamsZone) @cherrypy.tools.model_in(cls=RequestZoneSchedule) - @cherrypy.tools.resource_object(id_param="zone_id", cls=Zone) + @cherrypy.tools.resource_object(id_param="zone_name", cls=Zone) @cherrypy.tools.enforce_policy(policy_name="zones:action:schedule") def action_schedule(self, **_): + """Allow or disallow a zone to be scheduled + --- + put: + description: Allow or disallow a zone to be scheduled + tags: + - location + - zone + requestBody: + description: Zone schedule options + responses: + 204: + description: Zone schedule changed + """ cherrypy.response.status = 204 request: RequestZoneSchedule = cherrypy.request.model diff --git a/deli/counter/http/mounts/root/routes/swagger.py b/deli/counter/http/mounts/root/routes/swagger.py new file mode 100644 index 0000000..c52c270 --- /dev/null +++ b/deli/counter/http/mounts/root/routes/swagger.py @@ -0,0 +1,83 @@ +import cherrypy +from ingredients_http.route import Route + +from deli.counter.http.router import SandwichRouter + + +class SwaggerRouter(SandwichRouter): + def __init__(self): + super().__init__(uri_base='swagger') + + @Route('json') + @cherrypy.config(**{'tools.authentication.on': False}) + @cherrypy.tools.json_out() + def get(self): + api_spec_dict = self.mount.api_spec.to_dict() + api_spec_dict['components']['securitySchemes'] = { + 'Bearer': {'type': 'apiKey', 'name': 'Authorization', 'in': 'header'}} + + return api_spec_dict + + @Route('ui') + @cherrypy.config(**{'tools.authentication.on': False}) + def ui(self): + ui_html = """ + + + + + + Swagger UI + + + + + + +
+ + + + + + + + """ % cherrypy.url('/swagger/json') + return ui_html diff --git a/deli/counter/http/mounts/root/routes/v1/auth/auth.py b/deli/counter/http/mounts/root/routes/v1/auth/auth.py deleted file mode 100644 index cae67e4..0000000 --- a/deli/counter/http/mounts/root/routes/v1/auth/auth.py +++ /dev/null @@ -1,34 +0,0 @@ -import cherrypy -from ingredients_http.route import Route - -from deli.counter.auth.manager import DRIVERS -from deli.counter.http.router import SandwichRouter - - -class AuthRouter(SandwichRouter): - def __init__(self): - super().__init__() - - def setup_routes(self, dispatcher: cherrypy.dispatch.RoutesDispatcher, uri_prefix: str): - for _, driver in DRIVERS.items(): - try: - driver_router: SandwichRouter = driver.auth_router() - except NotImplementedError: - continue - driver_router.mount = self.mount - driver_router.setup_routes(dispatcher, uri_prefix) - - super().setup_routes(dispatcher, uri_prefix) - - @Route(route='discover') - @cherrypy.config(**{'tools.authentication.on': False}) - @cherrypy.tools.json_out() - def discover(self): - data = { - "default": list(DRIVERS.keys())[0] - } - - for _, driver in DRIVERS.items(): - data[driver.name] = driver.discover_options() - - return data diff --git a/deli/counter/http/mounts/root/routes/v1/auth/policy.py b/deli/counter/http/mounts/root/routes/v1/auth/policy.py deleted file mode 100644 index 16d5101..0000000 --- a/deli/counter/http/mounts/root/routes/v1/auth/policy.py +++ /dev/null @@ -1,44 +0,0 @@ -import uuid - -import cherrypy -from ingredients_http.route import Route - -from deli.counter.auth.policy import POLICIES -from deli.counter.http.mounts.root.routes.v1.auth.validation_models.policy import ParamsPolicy, ResponsePolicy, \ - ParamsListPolicy -from deli.counter.http.router import SandwichRouter - - -class AuthPolicyRouter(SandwichRouter): - def __init__(self): - super().__init__('policies') - - @Route(route='{policy_name}') - @cherrypy.tools.model_params(cls=ParamsPolicy) - @cherrypy.tools.model_out(cls=ResponsePolicy) - @cherrypy.tools.enforce_policy(policy_name="policies:get") - def get(self, policy_name): - - policy = None - - for p in POLICIES: - if p['name'] == policy_name: - policy = p - break - - if policy is None: - raise cherrypy.HTTPError(404, "The resource could not be found.") - - return ResponsePolicy(policy) - - @Route() - @cherrypy.tools.model_params(cls=ParamsListPolicy) - @cherrypy.tools.model_out_pagination(cls=ResponsePolicy) - @cherrypy.tools.enforce_policy(policy_name="policies:list") - def list(self, limit: int, marker: uuid.UUID): - policies = [] - - for p in POLICIES: - policies.append(ResponsePolicy(p)) - - return policies, False diff --git a/deli/counter/http/mounts/root/routes/v1/auth/role.py b/deli/counter/http/mounts/root/routes/v1/auth/role.py deleted file mode 100644 index 219cf7e..0000000 --- a/deli/counter/http/mounts/root/routes/v1/auth/role.py +++ /dev/null @@ -1,178 +0,0 @@ -from typing import Optional - -import cherrypy -from ingredients_http.request_methods import RequestMethods -from ingredients_http.route import Route - -from deli.counter.auth.policy import POLICIES -from deli.counter.http.mounts.root.routes.v1.auth.validation_models.role import RequestCreateRole, ResponseRole, \ - ParamsRole, ParamsListRoles, RequestRoleUpdate -from deli.counter.http.router import SandwichRouter -from deli.kubernetes.resources.model import ResourceState -from deli.kubernetes.resources.project import Project -from deli.kubernetes.resources.v1alpha1.role.model import GlobalRole, ProjectRole - - -class RoleHelper(object): - - def helper_create(self, project: Optional[Project]): - request: RequestCreateRole = cherrypy.request.model - - if project is None: - if GlobalRole.get_by_name(request.name) is not None: - raise cherrypy.HTTPError(400, 'A global role with the requested name already exists.') - role = GlobalRole() - else: - if ProjectRole.get_by_name(project, request.name) is not None: - raise cherrypy.HTTPError(400, 'A project role with the requested name already exists.') - role = ProjectRole() - role.project = project - - policy_names = [p['name'] for p in POLICIES] - - for policy in request.policies: - if policy not in policy_names: - raise cherrypy.HTTPError(404, 'Unknown policy %s' % policy) - - if project is not None: - i = policy_names.index(policy) - if 'project' not in POLICIES[i].get('tags', []): - raise cherrypy.HTTPError(409, 'Cannot add non-project policy %s to project role' % policy) - - role.name = request.name - role.policies = request.policies - role.create() - - return ResponseRole.from_database(role) - - def helper_get(self): - return ResponseRole.from_database(cherrypy.request.resource_object) - - def helper_list(self, project: Optional[Project], limit, marker): - if project is None: - return self.paginate(GlobalRole, ResponseRole, limit, marker) - else: - return self.paginate(ProjectRole, ResponseRole, limit, marker, project=project) - - def helper_update(self, project: Optional[Project]): - cherrypy.response.status = 204 - request: RequestRoleUpdate = cherrypy.request.model - role = cherrypy.request.resource_object - - if role.name in ['admin', 'default-member', 'default-service-account']: - raise cherrypy.HTTPError(409, 'Cannot update the default roles') - - if role.state != ResourceState.Created: - raise cherrypy.HTTPError(400, 'Role is not in the following state: ' + ResourceState.Created.value) - - policy_names = [p['name'] for p in POLICIES] - - for policy in request.policies: - if policy not in policy_names: - raise cherrypy.HTTPError(404, 'Unknown policy %s' % policy) - if project is not None: - i = policy_names.index(policy) - if 'project' not in POLICIES[i].get('tags', []): - raise cherrypy.HTTPError(409, 'Cannot add non-project policy %s to project role' % policy) - - role.policies = request.policies - role.save() - - def helper_delete(self): - cherrypy.response.status = 204 - role = cherrypy.request.resource_object - - if role.name in ['admin', 'default-member', 'default-service-account']: - raise cherrypy.HTTPError(409, 'Cannot delete the default roles') - - if role.state != ResourceState.Created: - raise cherrypy.HTTPError(400, 'Role is not in the following state: ' + ResourceState.Created.value) - - role.delete() - - -class AuthGlobalRolesRouter(SandwichRouter, RoleHelper): - def __init__(self): - super().__init__('global-roles') - - @Route(methods=[RequestMethods.POST]) - @cherrypy.tools.model_in(cls=RequestCreateRole) - @cherrypy.tools.model_out(cls=ResponseRole) - @cherrypy.tools.enforce_policy(policy_name="roles:global:create") - def create(self): - return self.helper_create(None) - - @Route(route='{role_id}') - @cherrypy.tools.model_params(cls=ParamsRole) - @cherrypy.tools.model_out(cls=ResponseRole) - @cherrypy.tools.resource_object(id_param="role_id", cls=GlobalRole) - @cherrypy.tools.enforce_policy(policy_name="roles:global:get") - def get(self, **_): - return self.helper_get() - - @Route() - @cherrypy.tools.model_params(cls=ParamsListRoles) - @cherrypy.tools.model_out_pagination(cls=ResponseRole) - @cherrypy.tools.enforce_policy(policy_name="roles:global:list") - def list(self, limit, marker): - return self.helper_list(None, limit, marker) - - @Route(route='{role_id}', methods=[RequestMethods.POST]) - @cherrypy.tools.model_in(cls=RequestRoleUpdate) - @cherrypy.tools.resource_object(id_param="role_id", cls=GlobalRole) - @cherrypy.tools.enforce_policy(policy_name="roles:global:update") - def update(self, **_): - return self.helper_update(None) - - @Route(route='{role_id}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.model_params(cls=ParamsRole) - @cherrypy.tools.resource_object(id_param="role_id", cls=GlobalRole) - @cherrypy.tools.enforce_policy(policy_name="roles:global:delete") - def delete(self, **_): - return self.helper_delete() - - -class AuthProjectRolesRouter(SandwichRouter, RoleHelper): - def __init__(self): - super().__init__('project-roles') - - @Route(methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() - @cherrypy.tools.model_in(cls=RequestCreateRole) - @cherrypy.tools.model_out(cls=ResponseRole) - @cherrypy.tools.enforce_policy(policy_name="roles:project:create") - def create(self): - return self.helper_create(cherrypy.request.project) - - @Route(route='{role_id}') - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsRole) - @cherrypy.tools.model_out(cls=ResponseRole) - @cherrypy.tools.resource_object(id_param="role_id", cls=ProjectRole) - @cherrypy.tools.enforce_policy(policy_name="roles:project:get") - def get(self, **_): - return self.helper_get() - - @Route() - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsListRoles) - @cherrypy.tools.model_out_pagination(cls=ResponseRole) - @cherrypy.tools.enforce_policy(policy_name="roles:project:list") - def list(self, limit, marker): - return self.helper_list(cherrypy.request.project, limit, marker) - - @Route(route='{role_id}', methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() - @cherrypy.tools.model_in(cls=RequestRoleUpdate) - @cherrypy.tools.resource_object(id_param="role_id", cls=ProjectRole) - @cherrypy.tools.enforce_policy(policy_name="roles:project:update") - def update(self, **_): - return self.helper_update(cherrypy.request.project) - - @Route(route='{role_id}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsRole) - @cherrypy.tools.resource_object(id_param="role_id", cls=ProjectRole) - @cherrypy.tools.enforce_policy(policy_name="roles:project:delete") - def delete(self, **_): - return self.helper_delete() diff --git a/deli/counter/http/mounts/root/routes/v1/auth/tokens.py b/deli/counter/http/mounts/root/routes/v1/auth/tokens.py deleted file mode 100644 index 8acfd0f..0000000 --- a/deli/counter/http/mounts/root/routes/v1/auth/tokens.py +++ /dev/null @@ -1,97 +0,0 @@ -import arrow -import cherrypy -from ingredients_http.request_methods import RequestMethods -from ingredients_http.route import Route -from kubernetes.client.rest import ApiException - -from deli.counter.http.mounts.root.routes.v1.auth.validation_models.tokens import ResponseVerifyToken, \ - RequestScopeToken, ResponseOAuthToken -from deli.counter.http.router import SandwichRouter -from deli.kubernetes.resources.project import Project -from deli.kubernetes.resources.v1alpha1.project_member.model import ProjectMember -from deli.kubernetes.resources.v1alpha1.role.model import GlobalRole, ProjectRole - - -class AuthTokenRouter(SandwichRouter): - def __init__(self): - super().__init__('tokens') - - @Route() - @cherrypy.tools.model_out(cls=ResponseVerifyToken) - def get(self): - token = cherrypy.request.token - project = token.project() - - response = ResponseVerifyToken() - response.driver = token.driver_name - - global_role_names = [] - for role_id in token.global_role_ids: - try: - role: GlobalRole = GlobalRole.get(role_id) - global_role_names.append(role.name) - except ApiException as e: - if e.status != 404: - raise - response.global_roles = global_role_names - - if token.service_account_id is not None: - response.service_account_id = token.service_account_id - else: - response.username = token.username - - if project is not None: - response.project_id = project.id - project_role_names = [] - - for role_id in token.project_role_ids: - try: - role: ProjectRole = ProjectRole.get(project, role_id) - project_role_names.append(role.name) - except ApiException as e: - if e.status != 404: - raise - response.project_roles = project_role_names - - return response - - @Route(methods=[RequestMethods.HEAD]) - def head(self): - cherrypy.response.status = 204 - # Fix for https://github.com/cherrypy/cherrypy/issues/1657 - del cherrypy.response.headers['Content-Type'] - - # Generate a new token scoped for the requested project - @Route(route='scope', methods=[RequestMethods.POST]) - @cherrypy.tools.model_in(cls=RequestScopeToken) - @cherrypy.tools.model_out(cls=ResponseOAuthToken) - @cherrypy.tools.enforce_policy(policy_name="projects:scope") - def scope_token(self): - token = cherrypy.request.token - - if token.project() is not None: - raise cherrypy.HTTPError(403, "Cannot scope an already scoped token.") - - request: RequestScopeToken = cherrypy.request.model - - project = Project.get(request.project_id) - if project is None: - raise cherrypy.HTTPError(404, 'A project with the requested id does not exist.') - - if token.service_account_id is None and project.is_member(token.username, token.driver_name): - member_id = project.get_member_id(token.username, token.driver_name) - project_member: ProjectMember = ProjectMember.get(project, member_id) - token.project_role_ids = project_member.roles - else: - try: - token.enforce_policy("projects:scope:all") - except cherrypy.HTTPError: - raise cherrypy.HTTPError(400, "Only project members can scope to this project") - - token.expires_at = arrow.now().shift(days=+1) - token.project_id = project.id - - response = ResponseOAuthToken() - response.access_token = token.marshal(self.mount.fernet) - response.expiry = token.expires_at - return response diff --git a/deli/counter/http/mounts/root/routes/v1/auth/validation_models/tokens.py b/deli/counter/http/mounts/root/routes/v1/auth/validation_models/tokens.py deleted file mode 100644 index b1a3774..0000000 --- a/deli/counter/http/mounts/root/routes/v1/auth/validation_models/tokens.py +++ /dev/null @@ -1,22 +0,0 @@ -from ingredients_http.schematics.types import ArrowType -from schematics import Model -from schematics.types import StringType, ListType, UUIDType - - -class RequestScopeToken(Model): - project_id = UUIDType(required=True) - - -class ResponseVerifyToken(Model): - username = StringType() - driver = StringType() - service_account_id = UUIDType() - service_account_name = StringType() - project_id = UUIDType() - global_roles = ListType(StringType(), default=list) - project_roles = ListType(StringType(), default=list) - - -class ResponseOAuthToken(Model): - access_token = StringType(required=True) - expiry = ArrowType() diff --git a/deli/counter/http/mounts/root/routes/v1/images.py b/deli/counter/http/mounts/root/routes/v1/images.py deleted file mode 100644 index b82d4c8..0000000 --- a/deli/counter/http/mounts/root/routes/v1/images.py +++ /dev/null @@ -1,219 +0,0 @@ -import uuid - -import cherrypy -from ingredients_http.request_methods import RequestMethods -from ingredients_http.route import Route - -from deli.counter.http.mounts.root.routes.v1.validation_models.images import RequestCreateImage, ResponseImage, \ - ParamsImage, ParamsListImage, ParamsImageMember, RequestAddMember, ResponseImageMember, RequestImageVisibility -from deli.counter.http.router import SandwichRouter -from deli.kubernetes.resources.const import REGION_LABEL, IMAGE_VISIBILITY_LABEL, \ - MEMBER_LABEL -from deli.kubernetes.resources.model import ResourceState -from deli.kubernetes.resources.project import Project -from deli.kubernetes.resources.v1alpha1.image.model import Image, ImageVisibility -from deli.kubernetes.resources.v1alpha1.region.model import Region - - -class ImageRouter(SandwichRouter): - def __init__(self): - super().__init__(uri_base='images') - - @Route(methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() - @cherrypy.tools.model_in(cls=RequestCreateImage) - @cherrypy.tools.model_out(cls=ResponseImage) - @cherrypy.tools.enforce_policy(policy_name="images:create") - def create(self): - request: RequestCreateImage = cherrypy.request.model - project: Project = cherrypy.request.project - - image = Image.get_by_name(request.name, project=project) - if image is not None: - raise cherrypy.HTTPError(400, 'An image with the requested name already exists.') - - region = Region.get(request.region_id) - if region is None: - raise cherrypy.HTTPError(404, 'A region with the requested id does not exist.') - - if region.state != ResourceState.Created: - raise cherrypy.HTTPError(409, 'Can only create a image with a region in the following state: %s'.format( - ResourceState.Created)) - - # TODO: check duplicate file name - - image = Image() - image.name = request.name - image.file_name = request.file_name - image.project = project - image.region = region - image.create() - - return ResponseImage.from_database(image) - - @Route(route='{image_id}') - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsImage) - @cherrypy.tools.model_out(cls=ResponseImage) - @cherrypy.tools.resource_object(id_param="image_id", cls=Image) - @cherrypy.tools.enforce_policy(policy_name="images:get") - def get(self, **_): - image: Image = cherrypy.request.resource_object - - if image.visibility == ImageVisibility.PRIVATE: - if image.project_id != cherrypy.request.project.id: - if image.is_member(cherrypy.request.project.id) is False: - raise cherrypy.HTTPError(404, "The resource could not be found.") - - return ResponseImage.from_database(image) - - @Route() - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsListImage) - @cherrypy.tools.model_out_pagination(cls=ResponseImage) - @cherrypy.tools.enforce_policy(policy_name="images:list") - def list(self, region_id, visibility: ImageVisibility, limit: int, marker: uuid.UUID): - kwargs = { - 'label_selector': [] - } - - if visibility == ImageVisibility.PRIVATE: - kwargs['label_selector'].append(IMAGE_VISIBILITY_LABEL + '=' + ImageVisibility.PRIVATE.value) - kwargs['label_selector'].append(MEMBER_LABEL + "/" + str(cherrypy.request.project.id) + "=1") - else: - kwargs['label_selector'].append(IMAGE_VISIBILITY_LABEL + '=' + ImageVisibility.PUBLIC.value) - - if region_id is not None: - region: Region = Region.get(region_id) - if region is None: - raise cherrypy.HTTPError(404, "A region with the requested id does not exist.") - - kwargs['label_selector'].append(REGION_LABEL + '=' + region.id) - - kwargs['label_selector'] = ",".join(kwargs['label_selector']) - - return self.paginate(Image, ResponseImage, limit, marker, **kwargs) - - @Route(route='{image_id}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsImage) - @cherrypy.tools.resource_object(id_param="image_id", cls=Image) - @cherrypy.tools.enforce_policy(policy_name="images:delete") - def delete(self, **_): - cherrypy.response.status = 204 - image: Image = cherrypy.request.resource_object - - if image.project_id != cherrypy.request.project.id: - raise cherrypy.HTTPError(404, "The resource could not be found.") - if image.state == ResourceState.ToDelete or image.state == ResourceState.Deleting: - raise cherrypy.HTTPError(400, "Image is already being deleting") - if image.state == ResourceState.Deleted: - raise cherrypy.HTTPError(400, "Image has already been deleted") - if image.state not in [ResourceState.Created, ResourceState.Error]: - raise cherrypy.HTTPError(400, 'Image cannot be deleted in the current state') - - image.delete() - - @Route(route='{image_id}/action/visibility', methods=[RequestMethods.PUT]) - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsImage) - @cherrypy.tools.model_in(cls=RequestImageVisibility) - @cherrypy.tools.resource_object(id_param="image_id", cls=Image) - @cherrypy.tools.enforce_policy(policy_name="images:action:visibility") - def action_visibility(self, **_): - cherrypy.response.status = 204 - image: Image = cherrypy.request.resource_object - request: RequestImageVisibility = cherrypy.request.model - - if image.project_id != cherrypy.request.project.id: - raise cherrypy.HTTPError(404, "The resource could not be found.") - - if request.public: - self.mount.enforce_policy("images:action:visibility:public") - if image.visibility == ImageVisibility.PUBLIC: - raise cherrypy.HTTPError(409, 'The requested image is already public') - else: - if image.visibility == ImageVisibility.PRIVATE: - raise cherrypy.HTTPError(409, 'The requested image is already private') - - image.visibility = ImageVisibility.PUBLIC if request.public else ImageVisibility.PRIVATE - image.save() - - @Route(route='{image_id}/members', methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsImage) - @cherrypy.tools.model_in(cls=RequestAddMember) - @cherrypy.tools.resource_object(id_param="image_id", cls=Image) - @cherrypy.tools.enforce_policy(policy_name="images:members:add") - def add_member(self, **_): - cherrypy.response.status = 204 - request: RequestAddMember = cherrypy.request.model - image: Image = cherrypy.request.resource_object - - if image.project_id != cherrypy.request.project.id: - raise cherrypy.HTTPError(404, "The resource could not be found.") - - if image.visibility == ImageVisibility.PUBLIC: - raise cherrypy.HTTPError(409, 'Cannot add a member to a public image') - - project = Project.get(request.project_id) - if project is None: - raise cherrypy.HTTPError(404, 'A project with the requested id does not exist.') - - if image.project_id == project.id: - raise cherrypy.HTTPError(409, 'Cannot add the owning project as a member.') - - if image.is_member(request.project_id): - raise cherrypy.HTTPError(409, 'A project with the requested id is already a member.') - - image.add_member(request.project_id) - image.save() - - @Route(route='{image_id}/members') - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsImage) - @cherrypy.tools.model_out_pagination(cls=ResponseImageMember) - @cherrypy.tools.resource_object(id_param="image_id", cls=Image) - @cherrypy.tools.enforce_policy(policy_name="images:members:list") - def list_members(self, **_): - image: Image = cherrypy.request.resource_object - - if image.project_id != cherrypy.request.project.id: - raise cherrypy.HTTPError(404, "The resource could not be found.") - - if image.visibility == ImageVisibility.PUBLIC: - raise cherrypy.HTTPError(409, 'Cannot list members of a public image') - - members = [] - - for member_id in image.member_ids(): - member = ResponseImageMember() - member.project_id = member_id - members.append(member) - - return members, False - - @Route(route='{image_id}/members/{project_id}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsImageMember) - @cherrypy.tools.resource_object(id_param="image_id", cls=Image) - @cherrypy.tools.enforce_policy(policy_name="images:members:delete") - def delete_member(self, project_id, **_): - cherrypy.response.status = 204 - image: Image = cherrypy.request.resource_object - - if image.project_id != cherrypy.request.project.id: - raise cherrypy.HTTPError(404, "The resource could not be found.") - - if image.visibility == ImageVisibility.PUBLIC: - raise cherrypy.HTTPError(409, 'Cannot delete a member from a public image') - - project = Project.get(project_id) - if project is None: - raise cherrypy.HTTPError(404, 'A project with the requested id does not exist.') - - if image.is_member(project_id) is False: - raise cherrypy.HTTPError(409, 'A project with the requested id is not a member.') - - image.remove_member(project_id) - image.save() diff --git a/deli/counter/http/mounts/root/routes/v1/project_members.py b/deli/counter/http/mounts/root/routes/v1/project_members.py deleted file mode 100644 index 591a373..0000000 --- a/deli/counter/http/mounts/root/routes/v1/project_members.py +++ /dev/null @@ -1,108 +0,0 @@ -import uuid - -import cherrypy -from ingredients_http.request_methods import RequestMethods -from ingredients_http.route import Route - -from deli.counter.http.mounts.root.routes.v1.validation_models.projects import ResponseProjectMember, \ - RequestProjectAddMember, ParamsProjectMember, ParamsListProjectMember, RequestProjectUpdateMember -from deli.counter.http.router import SandwichRouter -from deli.kubernetes.resources.model import ResourceState -from deli.kubernetes.resources.project import Project -from deli.kubernetes.resources.v1alpha1.project_member.model import ProjectMember -from deli.kubernetes.resources.v1alpha1.role.model import ProjectRole - - -class ProjectMemberRouter(SandwichRouter): - def __init__(self): - super().__init__(uri_base='project-members') - - @Route(methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() - @cherrypy.tools.model_in(cls=RequestProjectAddMember) - @cherrypy.tools.model_out(cls=ResponseProjectMember) - @cherrypy.tools.enforce_policy(policy_name="projects:members:add") - def create(self): - request: RequestProjectAddMember = cherrypy.request.model - project: Project = cherrypy.request.project - - if project.is_member(request.username, request.driver): - raise cherrypy.HTTPError(400, 'The requested user is already a member of the project.') - - default_role = ProjectRole.get_by_name(project, "default-member") - - member = ProjectMember() - member.project = project - member.username = request.username - member.driver = request.driver - member.roles = [default_role] - member.create() - - project.add_member(member) - project.save() - - return ResponseProjectMember.from_database(member) - - @Route(route='{member_id}') - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsProjectMember) - @cherrypy.tools.model_out(cls=ResponseProjectMember) - @cherrypy.tools.resource_object(id_param="member_id", cls=ProjectMember) - @cherrypy.tools.enforce_policy(policy_name="projects:members:get") - def get(self, **_): - return ResponseProjectMember.from_database(cherrypy.request.resource_object) - - @Route() - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsListProjectMember) - @cherrypy.tools.model_out_pagination(cls=ResponseProjectMember) - @cherrypy.tools.enforce_policy(policy_name="projects:members:list") - def list(self, limit: int, marker: uuid.UUID): - return self.paginate(ProjectMember, ResponseProjectMember, limit, marker, project=cherrypy.request.project) - - @Route(route='{member_id}', methods=[RequestMethods.POST]) - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsProjectMember) - @cherrypy.tools.model_in(cls=RequestProjectUpdateMember) - @cherrypy.tools.resource_object(id_param="member_id", cls=ProjectMember) - @cherrypy.tools.enforce_policy(policy_name="projects:members:update") - def update(self, **_): - cherrypy.response.status = 204 - request: RequestProjectUpdateMember = cherrypy.request.model - project: Project = cherrypy.request.project - member: ProjectMember = cherrypy.request.resource_object - - roles = [] - - if member.state != ResourceState.Created: - raise cherrypy.HTTPError(400, - 'Project member is not in the following state: ' + ResourceState.Created.value) - - for role_id in request.roles: - role: ProjectRole = ProjectRole.get(project, str(role_id)) - if role is None: - raise cherrypy.HTTPError(404, - 'A project role with the requested id of %s does not exist.' % role_id) - roles.append(role) - - member.roles = roles - member.save() - - @Route(route='{member_id}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.project_scope() - @cherrypy.tools.model_params(cls=ParamsProjectMember) - @cherrypy.tools.resource_object(id_param="member_id", cls=ProjectMember) - @cherrypy.tools.enforce_policy(policy_name="projects:members:remove") - def delete(self, **_): - cherrypy.response.status = 204 - project: Project = cherrypy.request.project - member: ProjectMember = cherrypy.request.resource_object - - if member.state != ResourceState.Created: - raise cherrypy.HTTPError(400, - 'Project member is not in the following state: ' + ResourceState.Created.value) - - project.remove_member(member) - project.save() - - member.delete() diff --git a/deli/counter/http/mounts/root/routes/v1/projects.py b/deli/counter/http/mounts/root/routes/v1/projects.py deleted file mode 100644 index 6530f48..0000000 --- a/deli/counter/http/mounts/root/routes/v1/projects.py +++ /dev/null @@ -1,95 +0,0 @@ -import uuid - -import cherrypy -from ingredients_http.request_methods import RequestMethods -from ingredients_http.route import Route -from kubernetes.client.rest import ApiException - -from deli.counter.auth.token import Token -from deli.counter.http.mounts.root.routes.v1.validation_models.projects import ResponseProject, RequestCreateProject, \ - ParamsProject, ParamsListProject -from deli.counter.http.router import SandwichRouter -from deli.kubernetes.resources.const import MEMBER_LABEL -from deli.kubernetes.resources.project import Project -from deli.kubernetes.resources.v1alpha1.project_quota.model import ProjectQuota -from deli.kubernetes.resources.v1alpha1.role.model import ProjectRole -from deli.kubernetes.resources.v1alpha1.service_account.model import ProjectServiceAccount - - -class ProjectRouter(SandwichRouter): - def __init__(self): - super().__init__(uri_base='projects') - - @Route(methods=[RequestMethods.POST]) - @cherrypy.tools.model_in(cls=RequestCreateProject) - @cherrypy.tools.model_out(cls=ResponseProject) - @cherrypy.tools.enforce_policy(policy_name="projects:create") - def create(self): - request: RequestCreateProject = cherrypy.request.model - - project = Project.get_by_name(request.name) - if project is not None: - raise cherrypy.HTTPError(409, 'A project with the requested name already exists.') - - project = Project() - project.name = request.name - - try: - project.create() - except ApiException as e: - if e.status == 409: - raise cherrypy.HTTPError(409, 'Cannot create a project with the requested name, it is reserved.') - raise - - ProjectRole.create_default_roles(project) - ProjectServiceAccount.create_default_service_account(project) - quota = ProjectQuota() - # Set the quota id to the project id so we know how to get it back - quota._raw['metadata']['name'] = str(project.id) - quota.project = project - quota.create() - - return ResponseProject.from_database(project) - - @Route(route='{project_id}') - @cherrypy.tools.model_params(cls=ParamsProject) - @cherrypy.tools.model_out(cls=ResponseProject) - @cherrypy.tools.resource_object(id_param="project_id", cls=Project) - @cherrypy.tools.enforce_policy(policy_name="projects:get") - def get(self, **_): - token: Token = cherrypy.request.token - project: Project = cherrypy.request.resource_object - - if token.service_account_id is None and project.is_member(token.username, token.driver_name) is False: - self.mount.enforce_policy("projects:get:all") - - return ResponseProject.from_database(cherrypy.request.resource_object) - - @Route() - @cherrypy.tools.model_params(cls=ParamsListProject) - @cherrypy.tools.model_out_pagination(cls=ResponseProject) - @cherrypy.tools.enforce_policy(policy_name="projects:list") - def list(self, all: bool, limit: int, marker: uuid.UUID): - token: Token = cherrypy.request.token - kwargs = { - "label_selector": [] - } - - if token.service_account_id is None and all is False: - kwargs['label_selector'].append(token.driver_name + "." + MEMBER_LABEL + "/" + token.username) - else: - self.mount.enforce_policy("projects:list:all") - - kwargs['label_selector'] = ",".join(kwargs['label_selector']) - - return self.paginate(Project, ResponseProject, limit, marker, **kwargs) - - @Route(route='{project_id}', methods=[RequestMethods.DELETE]) - @cherrypy.tools.model_params(cls=ParamsProject) - @cherrypy.tools.resource_object(id_param="project_id", cls=Project) - @cherrypy.tools.enforce_policy(policy_name="projects:delete") - def delete(self, **_): - cherrypy.response.status = 204 - - project: Project = cherrypy.request.resource_object - project.delete() diff --git a/deli/counter/http/mounts/root/routes/v1/validation_models/projects.py b/deli/counter/http/mounts/root/routes/v1/validation_models/projects.py deleted file mode 100644 index 7adb8bb..0000000 --- a/deli/counter/http/mounts/root/routes/v1/validation_models/projects.py +++ /dev/null @@ -1,103 +0,0 @@ -from ingredients_http.schematics.types import KubeName, ArrowType, EnumType -from schematics import Model -from schematics.types import UUIDType, IntType, BooleanType, ListType, StringType - -from deli.kubernetes.resources.model import ResourceState -from deli.kubernetes.resources.project import Project -from deli.kubernetes.resources.v1alpha1.project_member.model import ProjectMember -from deli.kubernetes.resources.v1alpha1.project_quota.model import ProjectQuota - - -class RequestCreateProject(Model): - name = KubeName(required=True, min_length=3) - - -class ParamsProject(Model): - project_id = UUIDType(required=True) - - -class ParamsListProject(Model): - all = BooleanType(default=False) - limit = IntType(default=100, max_value=100, min_value=1) - marker = UUIDType() - - -class ResponseProject(Model): - id = UUIDType(required=True) - name = KubeName(required=True, min_length=3) - created_at = ArrowType(required=True) - - @classmethod - def from_database(cls, project: Project): - project_model = cls() - project_model.id = project.id - project_model.name = project.name - project_model.created_at = project.created_at - - return project_model - - -class RequestProjectModifyQuota(Model): - vcpu = IntType(required=True, min_value=-1) - ram = IntType(required=True, min_value=-1) - disk = IntType(required=True, min_value=-1) - - -class ResponseProjectQuota(Model): - vcpu = IntType(required=True) - ram = IntType(required=True) - disk = IntType(required=True) - used_vcpu = IntType(required=True) - used_ram = IntType(required=True) - used_disk = IntType(required=True) - - @classmethod - def from_database(cls, quota: ProjectQuota): - model = cls() - model.vcpu = quota.vcpu - model.ram = quota.ram - model.disk = quota.disk - model.used_vcpu = quota.used_vcpu - model.used_ram = quota.used_ram - model.used_disk = quota.used_disk - - return model - - -class RequestProjectAddMember(Model): - username = StringType(required=True) - driver = StringType(required=True, choices=['github', 'database']) - - -class ParamsProjectMember(Model): - member_id = UUIDType(required=True) - - -class ParamsListProjectMember(Model): - limit = IntType(default=100, max_value=100, min_value=1) - marker = UUIDType() - - -class RequestProjectUpdateMember(Model): - roles = ListType(UUIDType, required=True, min_size=1) - - -class ResponseProjectMember(Model): - id = UUIDType(required=True) - username = StringType(required=True) - driver = StringType(required=True) - roles = ListType(UUIDType, default=list) - state = EnumType(ResourceState, required=True) - created_at = ArrowType(required=True) - - @classmethod - def from_database(cls, project_member: ProjectMember): - model = cls() - model.id = project_member.id - model.username = project_member.username - model.driver = project_member.driver - model.roles = project_member.roles - model.state = project_member.state - model.created_at = project_member.created_at - - return model diff --git a/deli/counter/http/router.py b/deli/counter/http/router.py index 3b32af2..04bf442 100644 --- a/deli/counter/http/router.py +++ b/deli/counter/http/router.py @@ -1,3 +1,8 @@ +import inspect +from typing import Callable, List + +import cherrypy +from ingredients_http.request_methods import RequestMethods from ingredients_http.router import Router @@ -9,3 +14,35 @@ def paginate(self, db_cls, response_cls, limit, marker, **kwargs): resp_models.append(response_cls.from_database(obj)) return resp_models, False + + def on_register(self, uri: str, action: Callable, methods: List[RequestMethods]): + self.mount.api_spec.add_path(path=uri, router=self, func=action) + + +class SandwichSystemRouter(SandwichRouter): + + def __init__(self, uri_base=None): + if uri_base is None: + uri_base = 'system' + else: + uri_base = 'system/' + uri_base + super().__init__(uri_base=uri_base) + + +class SandwichProjectRouter(SandwichRouter): + + def __init__(self, uri_base=None): + if uri_base is None: + uri_base = 'projects/{project_name}' + else: + uri_base = 'projects/{project_name}/' + uri_base + super().__init__(uri_base=uri_base) + + def setup_routes(self, dispatcher: cherrypy.dispatch.RoutesDispatcher, uri_prefix: str): + for member in [getattr(self, attr) for attr in dir(self)]: + if inspect.ismethod(member) and hasattr(member, '_route'): + # Enable project scope checking + self.__class__.__dict__[member.__name__]._cp_config['tools.project_scope.on'] = True + self.__class__.__dict__[member.__name__]._cp_config['tools.project_scope.delete_param'] = True + + super().setup_routes(dispatcher, uri_prefix) diff --git a/deli/counter/http/spec/__init__.py b/deli/counter/http/spec/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/deli/counter/http/spec/plugins/__init__.py b/deli/counter/http/spec/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/deli/counter/http/spec/plugins/docstring.py b/deli/counter/http/spec/plugins/docstring.py new file mode 100644 index 0000000..f92f797 --- /dev/null +++ b/deli/counter/http/spec/plugins/docstring.py @@ -0,0 +1,112 @@ +from apispec import Path +from apispec.utils import load_operations_from_docstring +from schematics.models import FieldDescriptor + +from deli.counter.http.router import SandwichProjectRouter + + +def docstring_path_helper(spec, path, router, func, **kwargs): + operations = load_operations_from_docstring(func.__doc__) + + cp_config = func._cp_config + + if operations is not None: + for method, data in operations.items(): + + if cp_config.get('tools.authentication.on', True): + data['security'] = [ + {'Bearer': []} + ] + + if 'tools.model_in.cls' in cp_config: + model_cls = cp_config['tools.model_in.cls'] + spec.definition(model_cls.__name__, **parse_model(model_cls)) + + data['requestBody']['required'] = True + data['requestBody']['content'] = { + 'application/json': { + 'schema': {'$ref': '#/components/schemas/' + model_cls.__name__} + } + } + + if 'tools.model_params.cls' in cp_config: + model_cls = cp_config['tools.model_params.cls'] + data['parameters'] = data.get('parameters', []) + + # In query vs in path + for key, obj in model_cls.__dict__.items(): + inn = 'query' + if '{' + key + '}' in path.path: + inn = 'path' + if isinstance(obj, FieldDescriptor): + data['parameters'].append({ + 'name': key, + 'in': inn, + 'required': model_cls._fields[key].required, + 'schema': { + 'type': 'string' + } + }) + + if 'tools.model_out.cls' in cp_config: + model_cls = cp_config['tools.model_out.cls'] + spec.definition(model_cls.__name__, **parse_model(model_cls)) + data['responses'][200]['content'] = { + 'application/json': { + 'schema': {'$ref': '#/components/schemas/' + model_cls.__name__} + } + } + + if 'tools.model_out_pagination.cls' in cp_config: + model_cls = cp_config['tools.model_out_pagination.cls'] + spec.definition(model_cls.__name__, **parse_model(model_cls)) + data['responses'][200]['content'] = { + 'application/json': { + 'schema': { + 'type': 'array', + 'items': {'$ref': '#/components/schemas/' + model_cls.__name__} + } + } + } + + if isinstance(router, SandwichProjectRouter): + data['parameters'] = data.get('parameters', []) + data['parameters'].append({ + 'name': 'project_name', + 'in': 'path', + 'required': True, + 'schema': { + 'type': 'string' + } + }) + + if 'tools.enforce_policy.policy_name' in cp_config: + data['x-required-policy'] = cp_config['tools.enforce_policy.policy_name'] + + return Path(path=path.path, operations=operations) + + +def setup(spec): + spec.register_path_helper(docstring_path_helper) + + +def parse_model(model_cls): + kwargs = { + 'properties': {}, + 'extra_fields': { + 'type': 'object', + 'required': [] + } + } + for key, obj in model_cls.__dict__.items(): + if isinstance(obj, FieldDescriptor): + kwargs['properties'][key] = { + "type": "string" + } + if model_cls._fields[key].required: + kwargs['extra_fields']['required'].append(key) + + if len(kwargs['extra_fields']['required']) == 0: + del kwargs['extra_fields']['required'] + + return kwargs diff --git a/deli/counter/settings.py b/deli/counter/settings.py index 3a9722e..fcf4b8f 100644 --- a/deli/counter/settings.py +++ b/deli/counter/settings.py @@ -64,6 +64,12 @@ KUBE_CONFIG = os.environ.get("KUBECONFIG") KUBE_MASTER = os.environ.get("KUBEMASTER") +#################### +# REDIS # +#################### + +REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379") + #################### # Auth # #################### @@ -71,16 +77,15 @@ AUTH_DRIVERS = os.environ.get('AUTH_DRIVERS', "").split(",") AUTH_FERNET_KEYS = os.environ['AUTH_FERNET_KEYS'].split(",") -#################### -# GITHUB AUTH # -#################### +# URL of the OpenID Provider +OPENID_ISSUER_URL = os.environ['OPENID_ISSUER_URL'] + +# Client crendentials to auth with the OpenID Prover +OPENID_CLIENT_ID = os.environ['OPENID_CLIENT_ID'] +OPENID_CLIENT_SECRET = os.environ['OPENID_CLIENT_SECRET'] -GITHUB_URL = os.environ.get('GITHUB_URL', 'https://api.github.com') -GITHUB_CLIENT_ID = os.environ.get('GITHUB_CLIENT_ID') -GITHUB_CLIENT_SECRET = os.environ.get('GITHUB_CLIENT_SECRET') -GITHUB_ORG = os.environ.get('GITHUB_ORG') -GITHUB_TEAM_ROLES_PREFIX = os.environ.get("GITHUB_TEAM_ROLES_PREFIX", "sandwich-") +# JWT Claim to use as the user's email +OPENID_EMAIL_CLAIM = os.environ.get('OPENID_EMAIL_CLAIM', 'email') -# Split the env var into a dict because it is faster to search -_github_team_roles = os.environ.get('GITHUB_TEAM_ROLES', 'sandwich-admin:admin') -GITHUB_TEAM_ROLES = dict(item.split(":") for item in _github_team_roles.split(",")) +# JWT claim to use as the user's groups +OPENID_GROUPS_CLAIM = os.environ.get('OPENID_GROUPS_CLAIM', 'groups') diff --git a/deli/kubernetes/resources/const.py b/deli/kubernetes/resources/const.py index 35ff521..dbf465b 100644 --- a/deli/kubernetes/resources/const.py +++ b/deli/kubernetes/resources/const.py @@ -1,6 +1,6 @@ GROUP = "sandwichcloud.com" -ID_LABEL = GROUP + '/id' +# LABELS NAME_LABEL = GROUP + '/name' PROJECT_LABEL = GROUP + '/project' REGION_LABEL = GROUP + '/region' @@ -10,6 +10,9 @@ NETWORK_PORT_LABEL = GROUP + '/network_port' SERVICE_ACCOUNT_LABEL = GROUP + '/service_account' TAG_LABEL = 'tag.' + GROUP -IMAGE_VISIBILITY_LABEL = GROUP + '/visibility' -MEMBER_LABEL = 'member.' + GROUP ATTACHED_TO_LABEL = GROUP + '/attached_to' +MEMBER_NAME_LABEL = GROUP + '/member_name' +MEMBER_DOMAIN_LABEL = GROUP + '/member_domain' +VM_ID_LABEL = GROUP + '/vm_id' +# Annotations +UPDATED_AT_ANNOTATION = GROUP + '/updated_at' diff --git a/deli/kubernetes/resources/model.py b/deli/kubernetes/resources/model.py index fc865ea..bd54529 100644 --- a/deli/kubernetes/resources/model.py +++ b/deli/kubernetes/resources/model.py @@ -1,14 +1,15 @@ import enum +import json import re import time -import uuid import arrow from kubernetes import client from kubernetes.client import V1DeleteOptions, V1beta1CustomResourceDefinitionStatus from kubernetes.client.rest import ApiException -from deli.kubernetes.resources.const import GROUP, NAME_LABEL, ID_LABEL +from deli.cache import cache_client +from deli.kubernetes.resources.const import GROUP, UPDATED_AT_ANNOTATION, NAME_LABEL from deli.kubernetes.resources.project import Project @@ -25,16 +26,17 @@ class ResourceState(enum.Enum): class ResourceModel(object): def __init__(self, raw=None): if raw is None: - id = str(uuid.uuid4()) raw = { "apiVersion": GROUP + "/" + self.version(), "kind": self.kind(), "metadata": { - "name": id, + "name": '', "labels": { - ID_LABEL: id, NAME_LABEL: '' }, + 'annotations': { + UPDATED_AT_ANNOTATION: arrow.now('UTC').isoformat(), + }, "finalizers": [ 'delete.sandwichcloud.com' ] @@ -47,16 +49,13 @@ def __init__(self, raw=None): } self._raw = raw - @property - def id(self): - return uuid.UUID(self._raw['metadata']['name']) - @property def name(self): - return self._raw['metadata']['labels'][NAME_LABEL] + return self._raw['metadata']['name'] @name.setter def name(self, value): + self._raw['metadata']['name'] = value self._raw['metadata']['labels'][NAME_LABEL] = value @property @@ -85,6 +84,14 @@ def created_at(self): def resource_version(self): return self._raw['resourceVersion'] + @property + def updated_at(self): + return arrow.get(self._raw['metadata']['annotations'][UPDATED_AT_ANNOTATION]) + + @updated_at.setter + def updated_at(self, value): + self._raw['metadata']['annotations'][UPDATED_AT_ANNOTATION] = value.isoformat() + @classmethod def name_plural(cls): return cls.name_singular() + "s" @@ -118,7 +125,7 @@ def create_crd(cls): "spec": { "group": GROUP, "version": cls.version(), - "scope": "Cluster" if issubclass(cls, GlobalResourceModel) else "Namespaced", + "scope": "Cluster" if issubclass(cls, SystemResourceModel) else "Namespaced", "names": { "plural": cls.name_plural(), "singular": cls.name_singular(), @@ -176,44 +183,50 @@ def wait_for_crd(cls): time.sleep(1) -class GlobalResourceModel(ResourceModel): +class SystemResourceModel(ResourceModel): + def create(self): crd_api = client.CustomObjectsApi() + self._raw = crd_api.create_cluster_custom_object(GROUP, self.version(), self.name_plural(), self._raw) + cache_client.set(self.name_plural() + "_" + self.name, self._raw) @classmethod - def get(cls, id, safe=True): - crd_api = client.CustomObjectsApi() - try: - resp = crd_api.get_cluster_custom_object(GROUP, cls.version(), cls.name_plural(), str(id)) - if resp is None: - return None - except ApiException as e: - if e.status == 404 and safe: - return None - raise - return cls(resp) + def get(cls, name, safe=True, from_cache=True): + resp = None + if from_cache: + resp = cache_client.get(cls.name_plural() + "_" + name) + if resp is None: + crd_api = client.CustomObjectsApi() + try: + resp = crd_api.get_cluster_custom_object(GROUP, cls.version(), cls.name_plural(), name) + if resp is None: + return None + o = cls(resp) + cache_client.set(cls.name_plural() + "_" + o.name, o._raw) + except ApiException as e: + if e.status == 404: + cache_client.delete(cls.name_plural() + "_" + name) + if safe: + return None + raise + else: + o = cls(resp) - @classmethod - def get_by_name(cls, name, safe=True): - try: - objs = cls.list(label_selector=NAME_LABEL + "=" + name) - if len(objs) == 0: - return None - return objs[0] - except ApiException as e: - if e.status == 404 and safe: - return None - raise + return o def save(self, ignore=False): crd_api = client.CustomObjectsApi() try: - self._raw = crd_api.replace_cluster_custom_object(GROUP, self.version(), self.name_plural(), str(self.id), + self.updated_at = arrow.now('UTC') + self._raw = crd_api.replace_cluster_custom_object(GROUP, self.version(), self.name_plural(), self.name, self._raw) + cache_client.set(self.name_plural() + "_" + self.name, self._raw) except ApiException as e: - if e.status == 409 and ignore: - return + cache_client.delete(self.name_plural() + "_" + self.name) + if e.status == 409: + if ignore: + return raise e @classmethod @@ -222,12 +235,18 @@ def list_sig(cls): @classmethod def list(cls, **kwargs): + items = [] + crd_api = client.CustomObjectsApi() args, sig_kwargs = cls.list_sig() raw_list = crd_api.list_cluster_custom_object(*args, **{**sig_kwargs, **kwargs}) - items = [] + pipe = cache_client.pipeline() for item in raw_list['items']: - items.append(cls(item)) + o = cls(item) + items.append(o) + pipe.set(cls.name_plural() + "_" + o.name, json.dumps(item), ex=cache_client.default_cache_time) + pipe.execute() + return items def delete(self, force=False): @@ -235,10 +254,12 @@ def delete(self, force=False): if 'delete.sandwichcloud.com' in self._raw['metadata']['finalizers']: self._raw['metadata']['finalizers'].remove('delete.sandwichcloud.com') self.save() + cache_client.delete(self.name_plural() + "_" + self.name) crd_api = client.CustomObjectsApi() try: - crd_api.delete_cluster_custom_object(GROUP, self.version(), self.name_plural(), str(self.id), + crd_api.delete_cluster_custom_object(GROUP, self.version(), self.name_plural(), self.name, V1DeleteOptions()) + cache_client.delete(self.name_plural() + "_" + self.name) except ApiException as e: if e.status != 404: raise @@ -255,59 +276,69 @@ def __init__(self, raw=None): self._raw['metadata']['namespace'] = None @property - def project_id(self): - return uuid.UUID(self._raw['metadata']['namespace']) + def project_name(self): + project_name = self._raw['metadata']['namespace'] + if project_name is None: + return None + + return self._raw['metadata']['namespace'].replace("sandwich-", "") @property def project(self): - return Project.get(self._raw['metadata']['namespace']) + if self.project_name is None: + return None + return Project.get(self.project_name) @project.setter def project(self, value): - self._raw['metadata']['namespace'] = str(value.id) + self._raw['metadata']['namespace'] = "sandwich-" + value.name def create(self): - if self.project is None: + if self.project_name is None: raise ValueError("Project must be set to create %s".format(self.__class__.__name__)) crd_api = client.CustomObjectsApi() - self._raw = crd_api.create_namespaced_custom_object(GROUP, self.version(), str(self.project.id), + self._raw = crd_api.create_namespaced_custom_object(GROUP, self.version(), "sandwich-" + self.project_name, self.name_plural(), self._raw) + cache_client.set(self.name_plural() + "_" + self.project_name + "_" + self.name, self._raw) @classmethod - def get(cls, project, id, safe=True): - crd_api = client.CustomObjectsApi() - try: - resp = crd_api.get_namespaced_custom_object(GROUP, cls.version(), str(project.id), cls.name_plural(), - str(id)) - if resp is None: - return None - except ApiException as e: - if e.status == 404 and safe: - return None - raise - return cls(resp) + def get(cls, project, name, safe=True, from_cache=True): + resp = None + if from_cache: + resp = cache_client.get(cls.name_plural() + "_" + project.name + "_" + name) + if resp is None: + crd_api = client.CustomObjectsApi() + try: + resp = crd_api.get_namespaced_custom_object(GROUP, cls.version(), "sandwich-" + project.name, + cls.name_plural(), name) + if resp is None: + return None + o = cls(resp) + cache_client.set(cls.name_plural() + "_" + o.project_name + "_" + o.name, o._raw) + except ApiException as e: + if e.status == 404: + cache_client.delete(cls.name_plural() + "_" + project.name + "_" + name) + if safe: + return None + raise + else: + o = cls(resp) - @classmethod - def get_by_name(cls, project, name, safe=True): - try: - objs = cls.list(project, label_selector=NAME_LABEL + "=" + name) - if len(objs) == 0: - return None - return objs[0] - except ApiException as e: - if e.status == 404 and safe: - return None - raise + return o def save(self, ignore=False): crd_api = client.CustomObjectsApi() try: - self._raw = crd_api.replace_namespaced_custom_object(GROUP, self.version(), str(self.project.id), - self.name_plural(), str(self.id), self._raw) + self.updated_at = arrow.now('UTC') + self._raw = crd_api.replace_namespaced_custom_object(GROUP, self.version(), "sandwich-" + self.project_name, + self.name_plural(), self.name, self._raw) + cache_client.set(self.name_plural() + "_" + self.project_name + "_" + self.name, self._raw) except ApiException as e: - if e.status == 409 and ignore: - return + cache_client.delete(self.name_plural() + "_" + self.project.name + "_" + self.name) + if e.status == 409: + if ignore: + return raise e @classmethod @@ -316,24 +347,38 @@ def list_sig(cls): @classmethod def list_all(cls, **kwargs): + items = [] + crd_api = client.CustomObjectsApi() args, sig_kwargs = cls.list_sig() # We use this to query all namespaces raw_list = crd_api.list_cluster_custom_object(*args, **{**sig_kwargs, **kwargs}) - items = [] + pipe = cache_client.pipeline() for item in raw_list['items']: - items.append(cls(item)) + o = cls(item) + items.append(o) + pipe.set(cls.name_plural() + "_" + o.project_name + "_" + o.name, json.dumps(item), + ex=cache_client.default_cache_time) + pipe.execute() + return items @classmethod def list(cls, project, **kwargs): + items = [] + crd_api = client.CustomObjectsApi() args, sig_kwargs = cls.list_sig() - args.append(str(project.id)) + args.append("sandwich-" + project.name) raw_list = crd_api.list_namespaced_custom_object(*args, **{**sig_kwargs, **kwargs}) - items = [] + pipe = cache_client.pipeline() for item in raw_list['items']: - items.append(cls(item)) + o = cls(item) + items.append(o) + pipe.set(cls.name_plural() + "_" + project.name + "_" + o.name, json.dumps(item), + ex=cache_client.default_cache_time) + pipe.execute() + return items def delete(self, force=False): @@ -341,11 +386,14 @@ def delete(self, force=False): if 'delete.sandwichcloud.com' in self._raw['metadata']['finalizers']: self._raw['metadata']['finalizers'].remove('delete.sandwichcloud.com') self.save() + cache_client.delete(self.name_plural() + "_" + self.project.name + "_" + self.name) crd_api = client.CustomObjectsApi() try: - crd_api.delete_namespaced_custom_object(GROUP, self.version(), str(self.project.id), - self.name_plural(), str(self.id), V1DeleteOptions()) + crd_api.delete_namespaced_custom_object(GROUP, self.version(), "sandwich-" + self.project.name, + self.name_plural(), self.name, V1DeleteOptions()) + cache_client.delete(self.name_plural() + "_" + self.project.name + "_" + self.name) except ApiException as e: + cache_client.delete(self.name_plural() + "_" + self.project.name + "_" + self.name) if e.status != 404: raise else: diff --git a/deli/kubernetes/resources/project.py b/deli/kubernetes/resources/project.py index 07e8c99..b1814a3 100644 --- a/deli/kubernetes/resources/project.py +++ b/deli/kubernetes/resources/project.py @@ -1,11 +1,12 @@ -import uuid +import json import arrow from kubernetes import client from kubernetes.client import V1DeleteOptions, V1Namespace from kubernetes.client.rest import ApiException -from deli.kubernetes.resources.const import NAME_LABEL, MEMBER_LABEL, PROJECT_LABEL +from deli.cache import cache_client +from deli.kubernetes.resources.const import PROJECT_LABEL class Project(object): @@ -16,9 +17,8 @@ def __init__(self, raw=None): "apiVersion": "v1", "kind": "Namespace", "metadata": { - "name": str(uuid.uuid4()), + "name": '', "labels": { - NAME_LABEL: None, PROJECT_LABEL: "true" } } @@ -28,48 +28,56 @@ def __init__(self, raw=None): else: self._raw = raw - @property - def id(self): - return uuid.UUID(self._raw['metadata']['name']) - @property def name(self): - return self._raw['metadata']['labels'][NAME_LABEL] + return self._raw['metadata']['name'].replace("sandwich-", "") @name.setter def name(self, value): - self._raw['metadata']['labels'][NAME_LABEL] = value + self._raw['metadata']['name'] = "sandwich-" + value def create(self): core_api = client.CoreV1Api() self._raw = core_api.create_namespace(self._raw).to_dict() + cache_client.set('project_' + self.name, self._raw) - @classmethod - def get(cls, id, safe=True): - core_api = client.CoreV1Api() - try: - resp = core_api.read_namespace(str(id)).to_dict() - if resp is None: - return None - if resp['metadata']['labels'] is None: - return None - if PROJECT_LABEL not in resp['metadata']['labels']: - return None - except ApiException as e: - if e.status == 404 and safe: - return None - raise - return cls(resp) + @property + def state(self): + return self._raw['status']["phase"] @classmethod - def get_by_name(cls, name): - objs = cls.list(label_selector=NAME_LABEL + "=" + name) - if len(objs) == 0: - return None - return objs[0] + def get(cls, name, safe=True, from_cache=True): + resp = None + if from_cache: + resp = cache_client.get("project_" + name) + if resp is None: + core_api = client.CoreV1Api() + try: + resp = core_api.read_namespace("sandwich-" + name).to_dict() + if resp is None: + return None + if resp['metadata']['labels'] is None: + return None + if PROJECT_LABEL not in resp['metadata']['labels']: + return None + o = cls(resp) + if o.state != 'Terminating': # Only cache if not terminating + cache_client.set('project_' + o.name, o._raw) + except ApiException as e: + if e.status == 404: + cache_client.delete('project_' + name) + if safe: + return None + raise + else: + o = cls(resp) + + return o @classmethod def list(cls, **kwargs): + items = [] + if 'label_selector' in kwargs: label_selector = kwargs['label_selector'] if len(label_selector) > 0: @@ -78,11 +86,16 @@ def list(cls, **kwargs): kwargs['label_selector'] += PROJECT_LABEL else: kwargs['label_selector'] = PROJECT_LABEL + core_api = client.CoreV1Api() raw_list = core_api.list_namespace(**kwargs) - items = [] + pipe = cache_client.pipeline() for item in raw_list.items: - items.append(cls(item)) + o = cls(item) + items.append(o) + if o.state != 'Terminating': # Only cache if not terminating + pipe.set("project_" + o.name, json.dumps(o._raw), ex=cache_client.default_cache_time) + pipe.execute() return items @property @@ -95,23 +108,15 @@ def resource_version(self): def delete(self): core_api = client.CoreV1Api() - core_api.delete_namespace(str(self.id), V1DeleteOptions()) + core_api.delete_namespace(self._raw['metadata']['name'], V1DeleteOptions()) + # TODO: Does this cause any side effects since the k8s object isn't deleted right away + cache_client.delete('project_' + self.name) def save(self): core_api = client.CoreV1Api() - self._raw = core_api.replace_namespace(str(self.id), self._raw) - - def is_member(self, username, driver): - label = driver + "." + MEMBER_LABEL + "/" + username - return label in self._raw['metadata']['labels'] - - def get_member_id(self, username, driver): - return self._raw['metadata']['labels'][driver + "." + MEMBER_LABEL + "/" + username] - - def add_member(self, project_member): - label = project_member.driver + "." + MEMBER_LABEL + "/" + project_member.username - self._raw['metadata']['labels'][label] = str(project_member.id) - - def remove_member(self, project_member): - label = project_member.driver + "." + MEMBER_LABEL + "/" + project_member.username - del self._raw['metadata']['labels'][label] + try: + self._raw = core_api.replace_namespace(self._raw['metadata']['name'], self._raw) + cache_client.set('project_' + self.name, self._raw) + except ApiException: + cache_client.delete('project_' + self.name) + raise diff --git a/deli/kubernetes/resources/v1alpha1/flavor/model.py b/deli/kubernetes/resources/v1alpha1/flavor/model.py index 87be610..15e93be 100644 --- a/deli/kubernetes/resources/v1alpha1/flavor/model.py +++ b/deli/kubernetes/resources/v1alpha1/flavor/model.py @@ -1,7 +1,7 @@ -from deli.kubernetes.resources.model import GlobalResourceModel +from deli.kubernetes.resources.model import SystemResourceModel -class Flavor(GlobalResourceModel): +class Flavor(SystemResourceModel): def __init__(self, raw=None): super().__init__(raw) diff --git a/deli/kubernetes/resources/v1alpha1/iam_group/__init__.py b/deli/kubernetes/resources/v1alpha1/iam_group/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/deli/kubernetes/resources/v1alpha1/iam_group/controller.py b/deli/kubernetes/resources/v1alpha1/iam_group/controller.py new file mode 100644 index 0000000..e69de29 diff --git a/deli/kubernetes/resources/v1alpha1/iam_group/model.py b/deli/kubernetes/resources/v1alpha1/iam_group/model.py new file mode 100644 index 0000000..27ddc50 --- /dev/null +++ b/deli/kubernetes/resources/v1alpha1/iam_group/model.py @@ -0,0 +1,21 @@ +from deli.kubernetes.resources.model import SystemResourceModel + + +class IAMSystemGroup(SystemResourceModel): + + def __init__(self, raw=None): + super().__init__(raw) + if raw is None: + self._raw['spec'] = {} + + @property + def email(self): + return "%s@group.system.sandwich.local" % self.name + + @property + def oauth_link(self): + return self._raw['spec']['oauth_link'] + + @oauth_link.setter + def oauth_link(self, value): + self._raw['spec']['oauth_link'] = value diff --git a/deli/kubernetes/resources/v1alpha1/iam_policy/__init__.py b/deli/kubernetes/resources/v1alpha1/iam_policy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/deli/kubernetes/resources/v1alpha1/iam_policy/controller.py b/deli/kubernetes/resources/v1alpha1/iam_policy/controller.py new file mode 100644 index 0000000..49b5b99 --- /dev/null +++ b/deli/kubernetes/resources/v1alpha1/iam_policy/controller.py @@ -0,0 +1,67 @@ +from deli.kubernetes.controller import ModelController +from deli.kubernetes.resources.model import ResourceState +from deli.kubernetes.resources.project import Project +from deli.kubernetes.resources.v1alpha1.iam_policy.model import IAMPolicy +from deli.kubernetes.resources.v1alpha1.iam_role.model import IAMProjectRole, IAMSystemRole + + +class IAMPolicyController(ModelController): + def __init__(self, worker_count, resync_seconds): + super().__init__(worker_count, resync_seconds, IAMPolicy, None) + + def sync_model_handler(self, model): + state_funcs = { + ResourceState.ToCreate: self.to_create, + ResourceState.Creating: self.creating, + ResourceState.Created: self.created, + ResourceState.ToDelete: self.to_delete, + ResourceState.Deleting: self.deleting, + ResourceState.Deleted: self.deleted + } + + if model.state not in state_funcs: + return + + state_funcs[model.state](model) + + def to_create(self, model): + model.state = ResourceState.Creating + model.save() + + def creating(self, model): + model.state = ResourceState.Created + model.save() + + def created(self, model: IAMPolicy): + if model.name == "system": + return + + project = Project.get(model.name) + if project is None: + model.delete() + return + + needs_save = False + for binding in list(model.bindings): + role_name = binding['role'] + if project is not None: + role = IAMProjectRole.get(project, role_name) + else: + role = IAMSystemRole.get(role_name) + if role is None: + needs_save = True + model.bindings.remove(binding) + + if needs_save: + model.save() + + def to_delete(self, model): + model.state = ResourceState.Deleting + model.save() + + def deleting(self, model): + model.state = ResourceState.Deleted + model.save() + + def deleted(self, model): + model.delete(force=True) diff --git a/deli/kubernetes/resources/v1alpha1/iam_policy/model.py b/deli/kubernetes/resources/v1alpha1/iam_policy/model.py new file mode 100644 index 0000000..497dd45 --- /dev/null +++ b/deli/kubernetes/resources/v1alpha1/iam_policy/model.py @@ -0,0 +1,52 @@ +from deli.kubernetes.resources.model import SystemResourceModel + + +class IAMPolicy(SystemResourceModel): + + def __init__(self, raw=None): + super().__init__(raw) + if raw is None: + self._raw['spec'] = { + "bindings": [] + } + + @property + def bindings(self): + return self._raw['spec']['bindings'] + + @bindings.setter + def bindings(self, value): + self._raw['spec']['bindings'] = value + + @classmethod + def create_system_policy(cls): + policy = cls() + policy.name = "system" + policy.bindings = [ + { + "role": "admin", + "members": [ + "serviceAccount:admin@service-account.system.sandwich.local" + ] + } + ] + if policy.get("system") is None: + policy.create() + + @classmethod + def create_project_policy(cls, project, token): + policy = cls() + policy.name = project.name + + if token.service_account is not None: + member_email = "serviceAccount:" + token.email + else: + member_email = "user:" + token.email + + policy.bindings = [ + { + "role": "owner", + "members": [member_email] + } + ] + policy.create() diff --git a/deli/kubernetes/resources/v1alpha1/iam_role/__init__.py b/deli/kubernetes/resources/v1alpha1/iam_role/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/deli/kubernetes/resources/v1alpha1/role/controller.py b/deli/kubernetes/resources/v1alpha1/iam_role/controller.py similarity index 68% rename from deli/kubernetes/resources/v1alpha1/role/controller.py rename to deli/kubernetes/resources/v1alpha1/iam_role/controller.py index 3270d35..c490623 100644 --- a/deli/kubernetes/resources/v1alpha1/role/controller.py +++ b/deli/kubernetes/resources/v1alpha1/iam_role/controller.py @@ -1,12 +1,12 @@ -from deli.counter.auth.policy import POLICIES +from deli.counter.auth.policy import SYSTEM_POLICIES, PROJECT_POLICIES from deli.kubernetes.controller import ModelController from deli.kubernetes.resources.model import ResourceState -from deli.kubernetes.resources.v1alpha1.role.model import ProjectRole, GlobalRole +from deli.kubernetes.resources.v1alpha1.iam_role.model import IAMProjectRole, IAMSystemRole -class GlobalRoleController(ModelController): +class IAMSystemRoleController(ModelController): def __init__(self, worker_count, resync_seconds): - super().__init__(worker_count, resync_seconds, GlobalRole, None) + super().__init__(worker_count, resync_seconds, IAMSystemRole, None) def sync_model_handler(self, model): state_funcs = { @@ -31,11 +31,11 @@ def creating(self, model): model.state = ResourceState.Created model.save() - def created(self, model: GlobalRole): + def created(self, model: IAMSystemRole): if model.name != 'admin': return - policy_names = [p['name'] for p in POLICIES] + policy_names = [p['name'] for p in SYSTEM_POLICIES] if model.policies == policy_names: return @@ -55,9 +55,9 @@ def deleted(self, model): model.delete(force=True) -class ProjectRoleController(ModelController): +class IAMProjectRoleController(ModelController): def __init__(self, worker_count, resync_seconds): - super().__init__(worker_count, resync_seconds, ProjectRole, None) + super().__init__(worker_count, resync_seconds, IAMProjectRole, None) def sync_model_handler(self, model): state_funcs = { @@ -82,25 +82,17 @@ def creating(self, model): model.state = ResourceState.Created model.save() - def created(self, model: ProjectRole): - if model.name not in ['default-member', 'default-service-account']: - return - - member_policies = [] - service_account_policies = [] - - for p in POLICIES: - tags = p.get('tags', []) - if 'default_project_member' in tags: - member_policies.append(p['name']) - if 'default_service_account' in tags: - service_account_policies.append(p['name']) + def created(self, model: IAMProjectRole): + policies = { + 'viewer': [policy['name'] for policy in PROJECT_POLICIES if 'viewer' in policy.get('tag', [])], + 'editor': [policy['name'] for policy in PROJECT_POLICIES if 'editor' in policy.get('tag', [])], + 'owner': [policy['name'] for policy in PROJECT_POLICIES] + } - if model.name == 'default-member': - policies = member_policies - else: - policies = service_account_policies + if model.name not in policies.keys(): + return + policies = policies[model.name] if model.policies == policies: return @@ -115,5 +107,5 @@ def deleting(self, model): model.state = ResourceState.Deleted model.save() - def deleted(self, model: ProjectRole): + def deleted(self, model: IAMProjectRole): model.delete(force=True) diff --git a/deli/kubernetes/resources/v1alpha1/iam_role/model.py b/deli/kubernetes/resources/v1alpha1/iam_role/model.py new file mode 100644 index 0000000..e87453e --- /dev/null +++ b/deli/kubernetes/resources/v1alpha1/iam_role/model.py @@ -0,0 +1,67 @@ +from deli.counter.auth.policy import SYSTEM_POLICIES, PROJECT_POLICIES +from deli.kubernetes.resources.model import SystemResourceModel, ProjectResourceModel +from deli.kubernetes.resources.project import Project + + +class IAMSystemRole(SystemResourceModel): + + def __init__(self, raw=None): + super().__init__(raw) + if raw is None: + self._raw['spec'] = { + 'policies': [] + } + + @property + def policies(self): + return self._raw['spec']['policies'] + + @policies.setter + def policies(self, value): + self._raw['spec']['policies'] = value + + @classmethod + def create_default_roles(cls): + admin_role = cls() + admin_role.name = "admin" + admin_role.policies = [policy['name'] for policy in SYSTEM_POLICIES] + if cls.get(admin_role.name) is None: + admin_role.create() + + +class IAMProjectRole(ProjectResourceModel): + + def __init__(self, raw=None): + super().__init__(raw) + if raw is None: + self._raw['spec'] = { + 'policies': [] + } + + @property + def policies(self): + return self._raw['spec']['policies'] + + @policies.setter + def policies(self, value): + self._raw['spec']['policies'] = value + + @classmethod + def create_default_roles(cls, project: Project): + viewer_role = cls() + viewer_role.name = "viewer" + viewer_role.project = project + viewer_role.policies = [policy['name'] for policy in PROJECT_POLICIES if 'viewer' in policy.get('tag', [])] + viewer_role.create() + + editor_role = cls() + editor_role.name = "editor" + editor_role.project = project + editor_role.policies = [policy['name'] for policy in PROJECT_POLICIES if 'editor' in policy.get('tag', [])] + editor_role.create() + + owner_role = cls() + owner_role.name = "owner" + owner_role.project = project + owner_role.policies = [policy['name'] for policy in PROJECT_POLICIES] + owner_role.create() diff --git a/deli/kubernetes/resources/v1alpha1/iam_service_account/__init__.py b/deli/kubernetes/resources/v1alpha1/iam_service_account/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/deli/kubernetes/resources/v1alpha1/service_account/controller.py b/deli/kubernetes/resources/v1alpha1/iam_service_account/controller.py similarity index 64% rename from deli/kubernetes/resources/v1alpha1/service_account/controller.py rename to deli/kubernetes/resources/v1alpha1/iam_service_account/controller.py index 604f685..0dec336 100644 --- a/deli/kubernetes/resources/v1alpha1/service_account/controller.py +++ b/deli/kubernetes/resources/v1alpha1/iam_service_account/controller.py @@ -1,12 +1,13 @@ +import arrow + from deli.kubernetes.controller import ModelController from deli.kubernetes.resources.model import ResourceState -from deli.kubernetes.resources.v1alpha1.role.model import ProjectRole, GlobalRole -from deli.kubernetes.resources.v1alpha1.service_account.model import ProjectServiceAccount, GlobalServiceAccount +from deli.kubernetes.resources.v1alpha1.iam_service_account.model import ProjectServiceAccount, SystemServiceAccount -class GlobalServiceAccountController(ModelController): +class SystemServiceAccountController(ModelController): def __init__(self, worker_count, resync_seconds): - super().__init__(worker_count, resync_seconds, GlobalServiceAccount, None) + super().__init__(worker_count, resync_seconds, SystemServiceAccount, None) def sync_model_handler(self, model): state_funcs = { @@ -31,19 +32,16 @@ def creating(self, model): model.state = ResourceState.Created model.save() - def created(self, model: GlobalServiceAccount): - roles = [] - for role_id in list(model.role_ids): - role = GlobalRole.get(role_id) - if role is not None: - roles.append(role) - - if len(roles) == len(model.role_ids): - return + def created(self, model): + keys = model.keys + for key, date in dict(keys).items(): + if date.shift(months=+1) < arrow.now(): + # Delete key if it's 1 month after it's expiration + del keys[key] - # Some roles no longer exist so we need to fix that - model.roles = roles - model.save(ignore=True) + if len(keys) < len(model.keys): + model.keys = keys + model.save() def to_delete(self, model): model.state = ResourceState.Deleting @@ -53,7 +51,7 @@ def deleting(self, model): model.state = ResourceState.Deleted model.save() - def deleted(self, model: GlobalServiceAccount): + def deleted(self, model: SystemServiceAccount): model.delete(force=True) @@ -84,22 +82,16 @@ def creating(self, model): model.state = ResourceState.Created model.save() - def created(self, model: ProjectServiceAccount): - if model.name == "default": - return - - roles = [] - for role_id in list(model.role_ids): - role = ProjectRole.get(model.project, role_id) - if role is not None: - roles.append(role) - - if len(roles) == len(model.role_ids): - return + def created(self, model): + keys = model.keys + for key, date in dict(keys).items(): + if date.shift(months=+1) < arrow.now(): + # Delete key if it's 1 month after it's expiration + del keys[key] - # Some roles no longer exist so we need to fix that - model.roles = roles - model.save(ignore=True) + if len(keys) < len(model.keys): + model.keys = keys + model.save() def to_delete(self, model): model.state = ResourceState.Deleting diff --git a/deli/kubernetes/resources/v1alpha1/iam_service_account/model.py b/deli/kubernetes/resources/v1alpha1/iam_service_account/model.py new file mode 100644 index 0000000..b9b3f42 --- /dev/null +++ b/deli/kubernetes/resources/v1alpha1/iam_service_account/model.py @@ -0,0 +1,78 @@ +import arrow + +from deli.kubernetes.resources.model import ProjectResourceModel, SystemResourceModel +from deli.kubernetes.resources.project import Project + + +class SystemServiceAccount(SystemResourceModel): + + def __init__(self, raw=None): + super().__init__(raw) + if raw is None: + self._raw['spec'] = { + 'keys': {} + } + + @property + def email(self): + return "%s@service-account.system.sandwich.local" % self.name + + @property + def keys(self): + keys = {} + for key, date in self._raw['spec']['keys'].items(): + keys[key] = arrow.get(date) + + return keys + + @keys.setter + def keys(self, value): + keys = {} + for key, date in value.items(): + keys[key] = date.isoformat() + + self._raw['spec']['keys'] = keys + + @classmethod + def create_admin_sa(cls): + if SystemServiceAccount.get("admin") is None: + admin_sa = cls() + admin_sa.name = "admin" + admin_sa.create() + + +class ProjectServiceAccount(ProjectResourceModel): + + def __init__(self, raw=None): + super().__init__(raw) + if raw is None: + self._raw['spec'] = { + 'keys': {} + } + + @property + def email(self): + return "%s@service-account.%s.sandwich.local" % (self.name, self.project_name) + + @property + def keys(self): + keys = {} + for key, date in self._raw['spec']['keys'].items(): + keys[key] = arrow.get(date) + + return keys + + @keys.setter + def keys(self, value): + keys = {} + for key, date in value.items(): + keys[key] = date.isoformat() + + self._raw['spec']['keys'] = keys + + @classmethod + def create_default_service_account(cls, project: Project): + service_account = cls() + service_account.name = "default" + service_account.project = project + service_account.create() diff --git a/deli/kubernetes/resources/v1alpha1/image/controller.py b/deli/kubernetes/resources/v1alpha1/image/controller.py index 517f364..fde6c13 100644 --- a/deli/kubernetes/resources/v1alpha1/image/controller.py +++ b/deli/kubernetes/resources/v1alpha1/image/controller.py @@ -2,8 +2,8 @@ from deli.kubernetes.controller import ModelController from deli.kubernetes.resources.model import ResourceState -from deli.kubernetes.resources.project import Project -from deli.kubernetes.resources.v1alpha1.image.model import Image +from deli.kubernetes.resources.v1alpha1.image.model import Image, ImageTask +from deli.kubernetes.resources.v1alpha1.instance.model import Instance class ImageController(ModelController): @@ -36,9 +36,21 @@ def creating(self, model: Image): model.delete() return - if model.file_name is None: - # Image was created via instance so lets wait until the file is ready - return + if model.task == ImageTask.IMAGING_INSTANCE: + if model.file_name is None: + # Image was created via instance so lets wait until the file is ready + from_instance = Instance.get(model.project, model.task_kwargs['instance_name']) + # If the from_instance is gone the image should be deleted + if from_instance is None: + model.delete() + return + # If the instance errored creating the image we should be deleted + if from_instance.state == ResourceState.Error: + model.delete() + return + return + else: + model.task = None defer(model.save) @@ -61,11 +73,6 @@ def created(self, model: Image): model.delete() return - for member_id in model.member_ids(): - if Project.get(member_id) is None: - model.remove_member(member_id) - model.save(ignore=True) - def to_delete(self, model): model.state = ResourceState.Deleting model.save() @@ -84,7 +91,7 @@ def deleting(self, model): self.vmware.delete_image(vmware_client, vmware_image) else: self.logger.warning( - "Tried to delete image %s but couldn't find its backing file" % str(model.id)) + "Tried to delete image %s but couldn't find its backing file" % str(model.name)) model.state = ResourceState.Deleted diff --git a/deli/kubernetes/resources/v1alpha1/image/model.py b/deli/kubernetes/resources/v1alpha1/image/model.py index 7125492..6887359 100644 --- a/deli/kubernetes/resources/v1alpha1/image/model.py +++ b/deli/kubernetes/resources/v1alpha1/image/model.py @@ -1,10 +1,7 @@ import enum -import uuid -from deli.kubernetes.resources.const import REGION_LABEL, IMAGE_VISIBILITY_LABEL, PROJECT_LABEL, MEMBER_LABEL, \ - NAME_LABEL -from deli.kubernetes.resources.model import GlobalResourceModel -from deli.kubernetes.resources.project import Project +from deli.kubernetes.resources.const import REGION_LABEL +from deli.kubernetes.resources.model import ProjectResourceModel from deli.kubernetes.resources.v1alpha1.region.model import Region @@ -13,64 +10,41 @@ class ImageVisibility(enum.Enum): PRIVATE = 'PRIVATE' -class Image(GlobalResourceModel): +class ImageTask(enum.Enum): + IMAGING_INSTANCE = 'IMAGING_INSTANCE' + + +class Image(ProjectResourceModel): def __init__(self, raw=None): super().__init__(raw) if raw is None: - self._raw['metadata']['labels'][PROJECT_LABEL] = '' self._raw['metadata']['labels'][REGION_LABEL] = '' - self._raw['metadata']['labels'][IMAGE_VISIBILITY_LABEL] = ImageVisibility.PRIVATE.value self._raw['spec'] = { 'fileName': None } - - @classmethod - def get_by_name(cls, name, project=None): - label_selector = [NAME_LABEL + "=" + name] - if project is not None: - label_selector.append(PROJECT_LABEL + "=" + str(project.id)) - objs = cls.list(label_selector=",".join(label_selector)) - if len(objs) == 0: - return None - return objs[0] - - @property - def project_id(self): - project_id = self._raw['metadata']['labels'][PROJECT_LABEL] - if project_id == "": - return None - return uuid.UUID(project_id) - - @property - def project(self): - project_id = self.project_id - if project_id is None: - return None - return Project.get(project_id) - - @project.setter - def project(self, value): - self._raw['metadata']['labels'][PROJECT_LABEL] = str(value.id) - self.add_member(value.id) + self._raw['status']['task'] = { + 'name': None, + 'kwargs': {} + } @property - def region_id(self): - region_id = self._raw['metadata']['labels'][REGION_LABEL] - if region_id == "": + def region_name(self): + region_name = self._raw['metadata']['labels'][REGION_LABEL] + if region_name == "": return None - return uuid.UUID(region_id) + return region_name @property def region(self): - region_id = self.region_id - if region_id is None: + region_name = self.region_name + if region_name is None: return None - return Region.get(region_id) + return Region.get(region_name) @region.setter def region(self, value): - self._raw['metadata']['labels'][REGION_LABEL] = str(value.id) + self._raw['metadata']['labels'][REGION_LABEL] = value.name @property def file_name(self): @@ -81,42 +55,23 @@ def file_name(self, value): self._raw['spec']['fileName'] = value @property - def visibility(self): - return ImageVisibility(self._raw['metadata']['labels'][IMAGE_VISIBILITY_LABEL]) - - @visibility.setter - def visibility(self, value): - - if value == ImageVisibility.PUBLIC: - # We are now public so clear members - for label in list(self._raw['metadata']['labels']): - if label.startswith(MEMBER_LABEL) is False: - continue - del self._raw['metadata']['labels'][label] - elif value == ImageVisibility.PRIVATE: - # We are now private so add our project back as a member - self.add_member(self.project_id) - - self._raw['metadata']['labels'][IMAGE_VISIBILITY_LABEL] = value.value - - def add_member(self, project_id): - self._raw['metadata']['labels'][MEMBER_LABEL + "/" + str(project_id)] = "1" - - def remove_member(self, project_id): - self._raw['metadata']['labels'].pop(MEMBER_LABEL + "/" + str(project_id), None) - - def member_ids(self): - member_ids = [] + def task(self): + if self._raw['status']['task']['name'] is None: + return None + return ImageTask(self._raw['status']['task']['name']) - for label in self._raw['metadata']['labels']: - if label.startswith(MEMBER_LABEL) is False: - continue - member_id = label.split("/")[1] - if member_id == self.project_id: - continue - member_ids.append(member_id) + @task.setter + def task(self, value): + if value is None: + self._raw['status']['task']['name'] = None + self.task_kwargs = {} + else: + self._raw['status']['task']['name'] = value.value - return member_ids + @property + def task_kwargs(self): + return self._raw['status']['task']['kwargs'] - def is_member(self, project_id): - return MEMBER_LABEL + "/" + str(project_id) in self._raw['metadata']['labels'] + @task_kwargs.setter + def task_kwargs(self, value): + self._raw['status']['task']['kwargs'] = value diff --git a/deli/kubernetes/resources/v1alpha1/instance/controller.py b/deli/kubernetes/resources/v1alpha1/instance/controller.py index a44858b..ab7f867 100644 --- a/deli/kubernetes/resources/v1alpha1/instance/controller.py +++ b/deli/kubernetes/resources/v1alpha1/instance/controller.py @@ -1,4 +1,5 @@ import math +import uuid import arrow from go_defer import with_defer, defer @@ -89,15 +90,15 @@ def creating(self, model: Instance): if image_size > model.disk: model.error_message = "Requested image requires a disk size of at least %s GB" % image_size return - - old_vm = self.vmware.get_vm(vmware_client, str(model.id), datacenter) - if old_vm is not None: - self.logger.info( - "A backing for the vm %s / %s already exists so it is going to be deleted".format( - model.project.id, - model.id)) - self.vmware.power_off_vm(vmware_client, old_vm, hard=True) - self.vmware.delete_vm(vmware_client, old_vm) + # + # old_vm = self.vmware.get_vm(vmware_client, str(model.vm_id), datacenter) + # if old_vm is not None: + # self.logger.info( + # "A backing for the vm %s / %s already exists so it is going to be deleted".format( + # model.project.name, + # model.name)) + # self.vmware.power_off_vm(vmware_client, old_vm, hard=True) + # self.vmware.delete_vm(vmware_client, old_vm) port_group = self.vmware.get_port_group(vmware_client, network_port.network.port_group, datacenter) cluster = self.vmware.get_cluster(vmware_client, zone.vm_cluster, datacenter) @@ -106,7 +107,7 @@ def creating(self, model: Instance): if zone.vm_folder is not None: folder = self.vmware.get_folder(vmware_client, zone.vm_folder, datacenter) - create_vm_task = self.vmware.create_vm_from_image(vm_name=str(model.id), + create_vm_task = self.vmware.create_vm_from_image(vm_name="sandwich-" + str(uuid.uuid4()), image=vmware_image, datacenter=datacenter, cluster=cluster, @@ -125,61 +126,62 @@ def creating(self, model: Instance): if error is not None: model.error_message = error return + model.vm_id = task.info.result.config.instanceUuid datacenter = self.vmware.get_datacenter(vmware_client, model.region.datacenter) - vmware_vm = self.vmware.get_vm(vmware_client, str(model.id), datacenter) + vmware_vm = self.vmware.get_vm(vmware_client, str(model.vm_id), datacenter) self.vmware.resize_root_disk(vmware_client, model.disk, vmware_vm) self.vmware.setup_serial_connection(vmware_client, self.vspc_url, vmware_vm) if len(model.initial_volumes) > 0: if len(model.initial_volumes_status) == 0: - initial_volume_ids = [] + initial_volume_names = [] for idx, volume_data in enumerate(model.initial_volumes): volume = Volume() volume.project = model.project - volume.name = str(model.id) + "-" + str(idx) + volume.name = model.name + "-" + str(idx) volume.zone = model.zone volume.size = volume_data['size'] volume.create() - initial_volume_ids.append(str(volume.id)) - model.initial_volumes_status = initial_volume_ids + initial_volume_names.append(volume.name) + model.initial_volumes_status = initial_volume_names return else: attached_vols = 0 - for volume_id in model.initial_volumes_status: - volume: Volume = Volume.get(model.project, volume_id) + for volume_name in model.initial_volumes_status: + volume: Volume = Volume.get(model.project, volume_name) if volume is None: - model.error_message = "Volume " + str( - volume.id) + " has disappeared while trying to attach" + model.error_message = "Volume " + volume.name + \ + " has disappeared while trying to attach" return if volume.state in [ResourceState.ToCreate, ResourceState.Creating]: continue if volume.state != ResourceState.Created: model.error_message = "Cannot attach volume " + str( - volume.id) + " while it is in the following state: " + volume.state.value + volume.name) + " while it is in the following state: " + volume.state.value return - if volume.attached_to_id == model.id: + if volume.attached_to_name == model.name: attached_vols += 1 continue - if volume.attached_to_id is not None and volume.attached_to_id != model.id: + if volume.attached_to_name is not None and volume.attached_to_name != model.name: model.error_message = "Volume " + str( - volume.id) + " has been attached to another instance." + volume.name) + " has been attached to another instance." return if volume.task is None: volume.attach(model) volume.save() else: model.error_message = "Cannot attach volume" + str( - volume.id) + " while a task is running on it." + volume.name) + " while a task is running on it." return if attached_vols != len(model.initial_volumes_status): return self.vmware.power_on_vm(vmware_client, vmware_vm) model.task = None + model.power_state = VMPowerState.POWERED_ON model.state = ResourceState.Created - @with_defer def created(self, model: Instance): region = model.region # Check our region, if it is not created we should be deleted @@ -201,31 +203,36 @@ def created(self, model: Instance): model.delete() return - defer(model.save, ignore=True) - # If the image doesn't exist we should clear it - if model.image_id is not None: + if model.image_name is not None: if model.image is None: model.image = None + model.save() + return with self.vmware.client_session() as vmware_client: datacenter = self.vmware.get_datacenter(vmware_client, region.datacenter) - vmware_vm = self.vmware.get_vm(vmware_client, str(model.id), datacenter) + vmware_vm = self.vmware.get_vm(vmware_client, str(model.vm_id), datacenter) if vmware_vm is None: model.error_message = "Backing VM disappeared" + model.save() return if model.task == VMTask.STARTING: self.vmware.power_on_vm(vmware_client, vmware_vm) model.power_state = VMPowerState.POWERED_ON model.task = None + model.save() + return elif model.task == VMTask.STOPPING or model.task == VMTask.RESTARTING: is_shutdown = self.shutdown_vm(vmware_client, vmware_vm, model) if is_shutdown: if model.task == VMTask.RESTARTING: self.vmware.power_on_vm(vmware_client, vmware_vm) model.task = None + model.save() + return elif model.task == VMTask.IMAGING: if 'task_key' not in model.task_kwargs: datastore = self.vmware.get_datastore(vmware_client, region.image_datastore, datacenter) @@ -235,6 +242,8 @@ def created(self, model: Instance): clone_vm_task, image_file_name = self.vmware.clone_and_template_vm(vmware_vm, datastore, folder) model.task_kwargs["image_file_name"] = image_file_name model.task_kwargs["task_key"] = clone_vm_task.info.key + model.save() + return else: task = self.vmware.get_task(vmware_client, model.task_kwargs['task_key']) done, error = self.vmware.is_task_done(task) @@ -242,16 +251,23 @@ def created(self, model: Instance): if error is not None: model.error_message = error return - image: Image = Image.get(model.project, model.task_kwargs['image_id']) + image: Image = Image.get(model.project, model.task_kwargs['image_name']) image.file_name = model.task_kwargs["image_file_name"] image.save() model.task = None + model.save() + return power_state = str(vmware_vm.runtime.powerState) - if power_state == "poweredOn": + if power_state == "poweredOn" and model.power_state != VMPowerState.POWERED_ON: model.power_state = VMPowerState.POWERED_ON - else: + model.save() + return + + if power_state == "poweredOff" and model.power_state != VMPowerState.POWERED_OFF: model.power_state = VMPowerState.POWERED_OFF + model.save() + return def shutdown_vm(self, vmware_client, vmware_vm, model): if 'timeout_at' not in model.task_kwargs: @@ -260,13 +276,13 @@ def shutdown_vm(self, vmware_client, vmware_vm, model): self.vmware.power_off_vm(vmware_client, vmware_vm, hard=True) return True self.vmware.power_off_vm(vmware_client, vmware_vm) - model.task_kwargs["timeout_at"] = arrow.now().shift(seconds=+model.task_kwargs['timeout']).isoformat() + model.task_kwargs["timeout_at"] = arrow.now('UTC').shift(seconds=+model.task_kwargs['timeout']).isoformat() else: power_state = str(vmware_vm.runtime.powerState) if power_state != 'poweredOn': return True timeout_at = arrow.get(model.task_kwargs['timeout_at']) - if timeout_at <= arrow.now(): + if timeout_at <= arrow.now('UTC'): self.vmware.power_off_vm(vmware_client, vmware_vm, hard=True) return True @@ -288,11 +304,11 @@ def deleting(self, model: Instance): if region is not None: with self.vmware.client_session() as vmware_client: datacenter = self.vmware.get_datacenter(vmware_client, region.datacenter) - vmware_vm = self.vmware.get_vm(vmware_client, str(model.id), datacenter) + vmware_vm = self.vmware.get_vm(vmware_client, str(model.vm_id), datacenter) if vmware_vm is None: self.logger.warning( - "Could not find backing vm for instance %s/%s when trying to delete" % (model.project.id, - model.id)) + "Could not find backing vm for instance %s/%s when trying to delete" % (model.project.name, + model.name)) else: power_state = str(vmware_vm.runtime.powerState) if power_state == 'poweredOn': @@ -300,19 +316,19 @@ def deleting(self, model: Instance): if is_shutdown is False: return - for idx, volume_id in enumerate(model.initial_volumes_status): + for idx, volume_name in enumerate(model.initial_volumes_status): delete = model.initial_volumes[idx]['auto_delete'] if delete: - volume: Volume = Volume.get(model.project, volume_id) + volume: Volume = Volume.get(model.project, volume_name) if volume is None: continue if volume.state in [ResourceState.ToDelete, ResourceState.Deleting, ResourceState.Deleted]: continue - if volume.attached_to_id == model.id: + if volume.attached_to_name == model.name: volume.delete() attached_volumes = Volume.list(model.project, - label_selector=ATTACHED_TO_LABEL + "=" + str(model.id)) + label_selector=ATTACHED_TO_LABEL + "=" + str(model.name)) if len(attached_volumes) > 0: for volume in attached_volumes: if volume.state in [ResourceState.ToDelete, ResourceState.Deleting, ResourceState.Deleted]: @@ -340,7 +356,7 @@ def find_best_zone(self, vmware_client, datacenter, region, model): Currently this just finds a zone that can host the instances and returns that. We may want a better way to balance instances across zones in the future """ - zones = Zone.list(label_selector=REGION_LABEL + '=' + str(region.id)) + zones = Zone.list(label_selector=REGION_LABEL + '=' + str(region.name)) for zone in zones: if zone.schedulable is False: @@ -352,13 +368,13 @@ def find_best_zone(self, vmware_client, datacenter, region, model): def can_zone_host(self, vmware_client, datacenter, zone, model): cluster = self.vmware.get_cluster(vmware_client, zone.vm_cluster, datacenter) - instances = Instance.list_all(label_selector=ZONE_LABEL + '=' + str(zone.id)) + instances = Instance.list_all(label_selector=ZONE_LABEL + '=' + str(zone.name)) used_resources = {} for instance in instances: - if instance.id == model.id: + if instance.name == model.name: # Skip own instance continue - vm = self.vmware.get_vm(vmware_client, str(instance.id), datacenter) + vm = self.vmware.get_vm(vmware_client, str(instance.vm_id), datacenter) if vm is None: continue # If the VM doesn't have a host we should reserve it on all hosts diff --git a/deli/kubernetes/resources/v1alpha1/instance/model.py b/deli/kubernetes/resources/v1alpha1/instance/model.py index 64446e4..14a5454 100644 --- a/deli/kubernetes/resources/v1alpha1/instance/model.py +++ b/deli/kubernetes/resources/v1alpha1/instance/model.py @@ -2,14 +2,14 @@ import uuid from deli.kubernetes.resources.const import TAG_LABEL, REGION_LABEL, ZONE_LABEL, IMAGE_LABEL, NETWORK_LABEL, \ - NETWORK_PORT_LABEL, SERVICE_ACCOUNT_LABEL + NETWORK_PORT_LABEL, SERVICE_ACCOUNT_LABEL, VM_ID_LABEL from deli.kubernetes.resources.model import ProjectResourceModel from deli.kubernetes.resources.v1alpha1.flavor.model import Flavor -from deli.kubernetes.resources.v1alpha1.image.model import Image +from deli.kubernetes.resources.v1alpha1.iam_service_account.model import ProjectServiceAccount +from deli.kubernetes.resources.v1alpha1.image.model import Image, ImageTask from deli.kubernetes.resources.v1alpha1.keypair.keypair import Keypair from deli.kubernetes.resources.v1alpha1.network.model import NetworkPort from deli.kubernetes.resources.v1alpha1.region.model import Region -from deli.kubernetes.resources.v1alpha1.service_account.model import ProjectServiceAccount from deli.kubernetes.resources.v1alpha1.zone.model import Zone @@ -37,9 +37,10 @@ def __init__(self, raw=None): self._raw['metadata']['labels'][NETWORK_LABEL] = '' self._raw['metadata']['labels'][NETWORK_PORT_LABEL] = '' self._raw['metadata']['labels'][SERVICE_ACCOUNT_LABEL] = '' + self._raw['metadata']['labels'][VM_ID_LABEL] = '' self._raw['spec'] = { 'flavor': { - 'id': None, + 'name': None, 'vcpus': 1, 'ram': 1024, 'disk': 20, @@ -54,7 +55,7 @@ def __init__(self, raw=None): ], } self._raw['status']['initialVolumes'] = [ - # UUID of volumes with index matched to spec initialVolumes + # name of volumes with index matched to spec initialVolumes ] self._raw['status']['task'] = { 'name': None, @@ -65,95 +66,105 @@ def __init__(self, raw=None): } @property - def region_id(self): - region_id = self._raw['metadata']['labels'][REGION_LABEL] - if region_id == "": + def vm_id(self): + if len(self._raw['metadata']['labels'][VM_ID_LABEL]) > 0: + return uuid.UUID(self._raw['metadata']['labels'][VM_ID_LABEL]) + return None + + @vm_id.setter + def vm_id(self, value): + self._raw['metadata']['labels'][VM_ID_LABEL] = str(value) + + @property + def region_name(self): + region_name = self._raw['metadata']['labels'][REGION_LABEL] + if region_name == "": return None - return uuid.UUID(region_id) + return region_name @property def region(self): - region_id = self.region_id - if region_id is None: + region_name = self.region_name + if region_name is None: return None - return Region.get(region_id) + return Region.get(region_name) @region.setter def region(self, value): - self._raw['metadata']['labels'][REGION_LABEL] = str(value.id) + self._raw['metadata']['labels'][REGION_LABEL] = str(value.name) @property - def zone_id(self): - zone_id = self._raw['metadata']['labels'][ZONE_LABEL] - if zone_id == "": + def zone_name(self): + zone_name = self._raw['metadata']['labels'][ZONE_LABEL] + if zone_name == "": return None - return uuid.UUID(zone_id) + return zone_name @property def zone(self): - zone_id = self.zone_id - if zone_id is None: + zone_name = self.zone_name + if zone_name is None: return None - return Zone.get(zone_id) + return Zone.get(zone_name) @zone.setter def zone(self, value): - self._raw['metadata']['labels'][ZONE_LABEL] = str(value.id) + self._raw['metadata']['labels'][ZONE_LABEL] = str(value.name) @property - def image_id(self): - image_id = self._raw['metadata']['labels'][IMAGE_LABEL] - if image_id == "": + def image_name(self): + image_name = self._raw['metadata']['labels'][IMAGE_LABEL] + if image_name == "": return None - return uuid.UUID(image_id) + return image_name @property def image(self): - image_id = self.image_id - if image_id is None: + image_name = self.image_name + if image_name is None: return None - return Image.get(image_id) + return Image.get(self.project, image_name) @image.setter def image(self, value): - self._raw['metadata']['labels'][IMAGE_LABEL] = str(value.id) + self._raw['metadata']['labels'][IMAGE_LABEL] = str(value.name) @property def network_port_id(self): - network_port_id = self._raw['metadata']['labels'][NETWORK_PORT_LABEL] - if network_port_id == "": + network_port_name = self._raw['metadata']['labels'][NETWORK_PORT_LABEL] + if network_port_name == "": return None - return uuid.UUID(network_port_id) + return network_port_name @property def network_port(self): - network_port_id = self.network_port_id - if network_port_id is None: + network_port_name = self.network_port_id + if network_port_name is None: return None - return NetworkPort.get(self.project, network_port_id) + return NetworkPort.get(self.project, network_port_name) @network_port.setter def network_port(self, value): - self._raw['metadata']['labels'][NETWORK_LABEL] = str(value.network.id) - self._raw['metadata']['labels'][NETWORK_PORT_LABEL] = str(value.id) + self._raw['metadata']['labels'][NETWORK_LABEL] = str(value.network.name) + self._raw['metadata']['labels'][NETWORK_PORT_LABEL] = str(value.name) @property - def service_account_id(self): - service_account_id = self._raw['metadata']['labels'][SERVICE_ACCOUNT_LABEL] - if service_account_id == "": + def service_account_name(self): + service_account_name = self._raw['metadata']['labels'][SERVICE_ACCOUNT_LABEL] + if service_account_name == "": return None - return uuid.UUID(service_account_id) + return service_account_name @property def service_account(self): - service_account_id = self.service_account_id - if service_account_id is None: + service_account_name = self.service_account_name + if service_account_name is None: return None - return ProjectServiceAccount.get(self.project, service_account_id) + return ProjectServiceAccount.get(self.project, service_account_name) @service_account.setter def service_account(self, value): - self._raw['metadata']['labels'][SERVICE_ACCOUNT_LABEL] = str(value.id) + self._raw['metadata']['labels'][SERVICE_ACCOUNT_LABEL] = str(value.name) @property def user_data(self): @@ -209,20 +220,20 @@ def remove_tag(self, tag): del self._raw['metadata']['labels'][TAG_LABEL + '/' + tag] @property - def flavor_id(self): - if self._raw['spec']['flavor']['id'] is None: + def flavor_name(self): + if self._raw['spec']['flavor']['name'] is None: return None - return uuid.UUID(self._raw['spec']['flavor']['id']) + return self._raw['spec']['flavor']['name'] @property def flavor(self): - if self._raw['spec']['flavor']['id'] is None: + if self._raw['spec']['flavor']['name'] is None: return None - return Flavor.get(self._raw['spec']['flavor']['id']) + return Flavor.get(self._raw['spec']['flavor']['name']) @flavor.setter def flavor(self, value): - self._raw['spec']['flavor']['id'] = str(value.id) + self._raw['spec']['flavor']['name'] = value.name self.vcpus = value.vcpus self.ram = value.ram self.disk = value.disk @@ -252,17 +263,17 @@ def disk(self, value): self._raw['spec']['flavor']['disk'] = value @property - def keypair_ids(self): - keypair_ids = [] - for keypair_id in self._raw['spec']['keypairs']: - keypair_ids.append(uuid.UUID(keypair_id)) - return keypair_ids + def keypair_names(self): + keypair_names = [] + for keypair_name in self._raw['spec']['keypairs']: + keypair_names.append(keypair_name) + return keypair_names @property def keypairs(self): keypairs = [] - for keypair_id in self._raw['spec']['keypairs']: - keypair = Keypair.get(self.project, keypair_id) + for keypair_name in self._raw['spec']['keypairs']: + keypair = Keypair.get(self.project, keypair_name) if keypair is not None: keypairs.append(keypair) return keypairs @@ -270,7 +281,7 @@ def keypairs(self): @keypairs.setter def keypairs(self, value): for keypair in value: - self._raw['spec']['keypairs'].append(str(keypair.id)) + self._raw['spec']['keypairs'].append(str(keypair.name)) def action_start(self): self.task = VMTask.STARTING @@ -298,11 +309,15 @@ def action_image(self, image_name): image.region = self.region image.name = image_name image.file_name = None + image.task = ImageTask.IMAGING_INSTANCE + image.task_kwargs = { + 'instance_name': self.name + } image.create() self.task = VMTask.IMAGING self.task_kwargs = { - 'image_id': str(image.id) + 'image_name': image.name } self.save() diff --git a/deli/kubernetes/resources/v1alpha1/network/controller.py b/deli/kubernetes/resources/v1alpha1/network/controller.py index 8051f49..4f55db4 100644 --- a/deli/kubernetes/resources/v1alpha1/network/controller.py +++ b/deli/kubernetes/resources/v1alpha1/network/controller.py @@ -44,7 +44,7 @@ def creating(self, model: Network): with self.vmware.client_session() as vmware_client: datacenter = self.vmware.get_datacenter(vmware_client, region.datacenter) if datacenter is None: - model.error_message = "Could not find VMWare Datacenter for region %s " % str(region.id) + model.error_message = "Could not find VMWare Datacenter for region %s " % str(region.name) return port_group = self.vmware.get_port_group(vmware_client, model.port_group, datacenter) @@ -127,7 +127,7 @@ def creating(self, model: NetworkPort): self.lock.acquire() defer(self.lock.release) - network_ports = NetworkPort.list_all(label_selector=NETWORK_LABEL + "=" + str(network.id)) + network_ports = NetworkPort.list_all(label_selector=NETWORK_LABEL + "=" + str(network.name)) for network_port in network_ports: if network_port.ip_address is not None: usable_addresses.remove(network_port.ip_address) @@ -153,7 +153,7 @@ def to_delete(self, model): model.save() def deleting(self, model): - instances = Instance.list_all(label_selector=NETWORK_PORT_LABEL + "=" + str(model.id)) + instances = Instance.list_all(label_selector=NETWORK_PORT_LABEL + "=" + str(model.name)) if len(instances) > 0: # There is still an instance with the network port so lets wait return diff --git a/deli/kubernetes/resources/v1alpha1/network/model.py b/deli/kubernetes/resources/v1alpha1/network/model.py index 3724d70..2932a14 100644 --- a/deli/kubernetes/resources/v1alpha1/network/model.py +++ b/deli/kubernetes/resources/v1alpha1/network/model.py @@ -1,12 +1,11 @@ import ipaddress -import uuid from deli.kubernetes.resources.const import REGION_LABEL, NETWORK_LABEL -from deli.kubernetes.resources.model import GlobalResourceModel, ProjectResourceModel +from deli.kubernetes.resources.model import SystemResourceModel, ProjectResourceModel from deli.kubernetes.resources.v1alpha1.region.model import Region -class Network(GlobalResourceModel): +class Network(SystemResourceModel): def __init__(self, raw=None): super().__init__(raw) @@ -24,22 +23,22 @@ def __init__(self, raw=None): } @property - def region_id(self): - region_id = self._raw['metadata']['labels'][REGION_LABEL] - if region_id == "": + def region_name(self): + region_name = self._raw['metadata']['labels'][REGION_LABEL] + if region_name == "": return None - return uuid.UUID(region_id) + return region_name @property def region(self): - region_id = self.region_id - if region_id is None: + region_name = self.region_name + if region_name is None: return None - return Region.get(region_id) + return Region.get(region_name) @region.setter def region(self, value): - self._raw['metadata']['labels'][REGION_LABEL] = str(value.id) + self._raw['metadata']['labels'][REGION_LABEL] = value.name @property def port_group(self): @@ -105,8 +104,8 @@ def __init__(self, raw=None): } @property - def network_id(self): - return uuid.UUID(self._raw['metadata']['labels'][NETWORK_LABEL]) + def network_name(self): + return self._raw['metadata']['labels'][NETWORK_LABEL] @property def network(self): @@ -114,7 +113,7 @@ def network(self): @network.setter def network(self, value): - self._raw['metadata']['labels'][NETWORK_LABEL] = str(value.id) + self._raw['metadata']['labels'][NETWORK_LABEL] = str(value.name) @property def ip_address(self): diff --git a/deli/kubernetes/resources/v1alpha1/project_member/controller.py b/deli/kubernetes/resources/v1alpha1/project_member/controller.py deleted file mode 100644 index f509c35..0000000 --- a/deli/kubernetes/resources/v1alpha1/project_member/controller.py +++ /dev/null @@ -1,31 +0,0 @@ -from deli.kubernetes.controller import ModelController -from deli.kubernetes.resources.model import ResourceState -from deli.kubernetes.resources.v1alpha1.project_member.model import ProjectMember - - -class ProjectMemberController(ModelController): - def __init__(self, worker_count, resync_seconds): - super().__init__(worker_count, resync_seconds, ProjectMember, None) - - def sync_model_handler(self, model): - state_funcs = { - ResourceState.ToDelete: self.to_delete, - ResourceState.Deleting: self.deleting, - ResourceState.Deleted: self.deleted - } - - if model.state not in state_funcs: - return - - state_funcs[model.state](model) - - def to_delete(self, model): - model.state = ResourceState.Deleting - model.save() - - def deleting(self, model): - model.state = ResourceState.Deleted - model.save() - - def deleted(self, model: ProjectMember): - model.delete(force=True) diff --git a/deli/kubernetes/resources/v1alpha1/project_member/model.py b/deli/kubernetes/resources/v1alpha1/project_member/model.py deleted file mode 100644 index 67c8082..0000000 --- a/deli/kubernetes/resources/v1alpha1/project_member/model.py +++ /dev/null @@ -1,45 +0,0 @@ -import uuid - -from deli.kubernetes.resources.model import ProjectResourceModel - - -class ProjectMember(ProjectResourceModel): - - def __init__(self, raw=None): - super().__init__(raw) - if raw is None: - self._raw['spec'] = { - 'username': None, - 'driver': None, - 'roles': [], - } - - @property - def username(self): - return self._raw['spec']['username'] - - @username.setter - def username(self, value): - self._raw['spec']['username'] = value - - @property - def driver(self): - return self._raw['spec']['driver'] - - @driver.setter - def driver(self, value): - self._raw['spec']['driver'] = value - - @property - def roles(self): - roles = [] - for role_id in self._raw['spec']['roles']: - roles.append(uuid.UUID(role_id)) - return roles - - @roles.setter - def roles(self, value): - role_ids = [] - for role in value: - role_ids.append(str(role.id)) - self._raw['spec']['roles'] = role_ids diff --git a/deli/kubernetes/resources/v1alpha1/project_quota/controller.py b/deli/kubernetes/resources/v1alpha1/project_quota/controller.py index 0224ace..32df9ad 100644 --- a/deli/kubernetes/resources/v1alpha1/project_quota/controller.py +++ b/deli/kubernetes/resources/v1alpha1/project_quota/controller.py @@ -1,5 +1,6 @@ from deli.kubernetes.controller import ModelController from deli.kubernetes.resources.model import ResourceState +from deli.kubernetes.resources.project import Project from deli.kubernetes.resources.v1alpha1.instance.model import Instance from deli.kubernetes.resources.v1alpha1.project_quota.model import ProjectQuota from deli.kubernetes.resources.v1alpha1.volume.model import Volume @@ -33,21 +34,30 @@ def creating(self, model): model.save() def created(self, model: ProjectQuota): - model.used_vcpu = 0 - model.used_ram = 0 - model.used_disk = 0 + used_vcpu = 0 + used_ram = 0 + used_disk = 0 - instances = Instance.list(model.project) + project = Project.get(model.name) + if project is None: + model.delete() + return + + instances = Instance.list(project) for instance in instances: - model.used_vcpu += instance.vcpus - model.used_ram += instance.ram - model.used_disk += instance.disk + used_vcpu += instance.vcpus + used_ram += instance.ram + used_disk += instance.disk - volumes = Volume.list(model.project) + volumes = Volume.list(project) for volume in volumes: - model.used_disk += volume.size + used_disk += volume.size - model.save(ignore=True) + if used_vcpu != model.used_vcpu or used_ram != model.used_ram or used_disk != model.used_disk: + model.used_vcpu = used_vcpu + model.used_ram = used_ram + model.used_disk = used_disk + model.save(ignore=True) def to_delete(self, model): model.state = ResourceState.Deleting diff --git a/deli/kubernetes/resources/v1alpha1/project_quota/model.py b/deli/kubernetes/resources/v1alpha1/project_quota/model.py index 66dc239..8e2b9df 100644 --- a/deli/kubernetes/resources/v1alpha1/project_quota/model.py +++ b/deli/kubernetes/resources/v1alpha1/project_quota/model.py @@ -1,7 +1,7 @@ -from deli.kubernetes.resources.model import ProjectResourceModel +from deli.kubernetes.resources.model import SystemResourceModel -class ProjectQuota(ProjectResourceModel): +class ProjectQuota(SystemResourceModel): def __init__(self, raw=None): super().__init__(raw) diff --git a/deli/kubernetes/resources/v1alpha1/region/controller.py b/deli/kubernetes/resources/v1alpha1/region/controller.py index f1026e0..95df1fb 100644 --- a/deli/kubernetes/resources/v1alpha1/region/controller.py +++ b/deli/kubernetes/resources/v1alpha1/region/controller.py @@ -69,15 +69,15 @@ def deleted(self, model): def can_delete(self, model): # These resources need the region to exist to successfully delete - zones = Zone.list(label_selector=REGION_LABEL + "=" + str(model.id)) + zones = Zone.list(label_selector=REGION_LABEL + "=" + model.name) if len(zones) > 0: return False - images = Image.list(label_selector=REGION_LABEL + "=" + str(model.id)) + images = Image.list_all(label_selector=REGION_LABEL + "=" + model.name) if len(images) > 0: return False - instances = Instance.list_all(label_selector=REGION_LABEL + "=" + str(model.id)) + instances = Instance.list_all(label_selector=REGION_LABEL + "=" + model.name) if len(instances) > 0: return False diff --git a/deli/kubernetes/resources/v1alpha1/region/model.py b/deli/kubernetes/resources/v1alpha1/region/model.py index 8363679..334dbcd 100644 --- a/deli/kubernetes/resources/v1alpha1/region/model.py +++ b/deli/kubernetes/resources/v1alpha1/region/model.py @@ -1,7 +1,7 @@ -from deli.kubernetes.resources.model import GlobalResourceModel +from deli.kubernetes.resources.model import SystemResourceModel -class Region(GlobalResourceModel): +class Region(SystemResourceModel): def __init__(self, raw=None): super().__init__(raw) diff --git a/deli/kubernetes/resources/v1alpha1/role/model.py b/deli/kubernetes/resources/v1alpha1/role/model.py deleted file mode 100644 index a4db117..0000000 --- a/deli/kubernetes/resources/v1alpha1/role/model.py +++ /dev/null @@ -1,84 +0,0 @@ -from deli.counter.auth.policy import POLICIES -from deli.kubernetes.resources.model import GlobalResourceModel, ProjectResourceModel -from deli.kubernetes.resources.project import Project - - -class GlobalRole(GlobalResourceModel): - - def __init__(self, raw=None): - super().__init__(raw) - if raw is None: - self._raw['spec'] = { - 'policies': [] - } - - @classmethod - def kind(cls): - return "SandwichGlobalRole" - - @property - def policies(self): - return self._raw['spec']['policies'] - - @policies.setter - def policies(self, value): - self._raw['spec']['policies'] = value - - @classmethod - def create_default_roles(cls): - admin_policies = [] - - for policy in POLICIES: - admin_policies.append(policy['name']) - - admin_role = cls() - admin_role.name = "admin" - admin_role.policies = admin_policies - if cls.get_by_name(admin_role.name) is None: - admin_role.create() - - -class ProjectRole(ProjectResourceModel): - - def __init__(self, raw=None): - super().__init__(raw) - if raw is None: - self._raw['spec'] = { - 'policies': [] - } - - @classmethod - def kind(cls): - return "SandwichProjectRole" - - @property - def policies(self): - return self._raw['spec']['policies'] - - @policies.setter - def policies(self, value): - self._raw['spec']['policies'] = value - - @classmethod - def create_default_roles(cls, project: Project): - member_policies = [] - service_account_policies = [] - - for policy in POLICIES: - tags = policy.get('tags', []) - if 'default_project_member' in tags: - member_policies.append(policy['name']) - if 'default_service_account' in tags: - service_account_policies.append(policy['name']) - - member_role = cls() - member_role.name = "default-member" - member_role.project = project - member_role.policies = member_policies - member_role.create() - - sa_role = cls() - sa_role.name = "default-service-account" - sa_role.project = project - sa_role.policies = service_account_policies - sa_role.create() diff --git a/deli/kubernetes/resources/v1alpha1/service_account/model.py b/deli/kubernetes/resources/v1alpha1/service_account/model.py deleted file mode 100644 index f895d41..0000000 --- a/deli/kubernetes/resources/v1alpha1/service_account/model.py +++ /dev/null @@ -1,107 +0,0 @@ -import uuid - -from deli.kubernetes.resources.model import ProjectResourceModel, GlobalResourceModel -from deli.kubernetes.resources.project import Project -from deli.kubernetes.resources.v1alpha1.role.model import ProjectRole, GlobalRole - - -class GlobalServiceAccount(GlobalResourceModel): - - def __init__(self, raw=None): - super().__init__(raw) - if raw is None: - self._raw['spec'] = { - 'roles': [], - 'keys': [] - } - - @property - def role_ids(self): - roles_ids = [] - for role_id in self._raw['spec']['roles']: - roles_ids.append(uuid.UUID(role_id)) - return roles_ids - - @property - def roles(self): - roles = [] - for role_id in self._raw['spec']['roles']: - role = GlobalRole.get(role_id) - if role is not None: - roles.append(role) - return roles - - @roles.setter - def roles(self, value): - role_ids = [] - for role in value: - role_ids.append(str(role.id)) - self._raw['spec']['roles'] = role_ids - - @property - def keys(self): - return self._raw['spec']['keys'] - - @keys.setter - def keys(self, value): - self._raw['spec']['keys'] = value - - @classmethod - def create_admin_sa(cls): - if GlobalServiceAccount.get_by_name("admin") is None: - admin_role = GlobalRole.get_by_name("admin") - - admin_sa = cls() - admin_sa.name = "admin" - admin_sa.roles = [admin_role] - admin_sa.create() - - -class ProjectServiceAccount(ProjectResourceModel): - - def __init__(self, raw=None): - super().__init__(raw) - if raw is None: - self._raw['spec'] = { - 'roles': [], - 'keys': [] - } - - @property - def role_ids(self): - roles_ids = [] - for role_id in self._raw['spec']['roles']: - roles_ids.append(uuid.UUID(role_id)) - return roles_ids - - @property - def roles(self): - roles = [] - for role_id in self._raw['spec']['roles']: - role = ProjectRole.get(self.project, role_id) - if role is not None: - roles.append(role) - return roles - - @roles.setter - def roles(self, value): - role_ids = [] - for role in value: - role_ids.append(str(role.id)) - self._raw['spec']['roles'] = role_ids - - @property - def keys(self): - return self._raw['spec']['keys'] - - @keys.setter - def keys(self, value): - self._raw['spec']['keys'] = value - - @classmethod - def create_default_service_account(cls, project: Project): - service_account = cls() - service_account.name = "default" - service_account.project = project - service_account.roles = [ProjectRole.get_by_name(project, 'default-service-account')] - service_account.create() diff --git a/deli/kubernetes/resources/v1alpha1/volume/controller.py b/deli/kubernetes/resources/v1alpha1/volume/controller.py index a2387ab..3cbc954 100644 --- a/deli/kubernetes/resources/v1alpha1/volume/controller.py +++ b/deli/kubernetes/resources/v1alpha1/volume/controller.py @@ -50,16 +50,17 @@ def creating(self, model: Volume): datastore = self.vmware.get_datastore(vmware_client, zone.vm_datastore, datacenter) cloned_from: Volume = model.cloned_from if cloned_from is None: - if model.cloned_from_id: + if model.cloned_from_name: model.error_message = "Could not clone volume, parent diapered." return - model.backing_id = self.vmware.create_disk(vmware_client, str(model.id), model.size, datastore) + model.backing_id = self.vmware.create_disk(vmware_client, model.name, model.size, datastore) model.task = None model.state = ResourceState.Created return else: if 'task_key' not in model.task_kwargs: - task = self.vmware.clone_disk(vmware_client, str(model.id), str(cloned_from.backing_id), datastore) + task = self.vmware.clone_disk(vmware_client, model.name, str(cloned_from.backing_id), + datastore) model.task_kwargs = {"task_key": task.info.key} else: task = self.vmware.get_task(vmware_client, model.task_kwargs['task_key']) @@ -69,65 +70,77 @@ def creating(self, model: Volume): model.error_message = error return - model.backing_id = task.info.result.config.id.id + model.backing_id = task.info.result.config.name.name model.task = None model.state = ResourceState.Created @with_defer def created(self, model: Volume): - defer(model.save, ignore=True) zone = model.zone - if zone.state == ResourceState.Deleting: - model.state = ResourceState.ToDelete + needs_save = False - with self.vmware.client_session() as vmware_client: - datacenter = self.vmware.get_datacenter(vmware_client, model.region.datacenter) - datastore = self.vmware.get_datastore(vmware_client, zone.vm_datastore, datacenter) - if model.task == VolumeTask.ATTACHING: - instance = Instance.get(model.project, model.task_kwargs['to']) - if instance is None: - # Attaching to instance doesn't exist - model.task = None - return - if instance.state in [ResourceState.ToDelete, ResourceState.Deleting, ResourceState.Deleted, - ResourceState.Error]: - # Attaching to instance is deleting or errored. + def save(): + if needs_save: + model.save(ignore=True) + + defer(save) + + if model.task is not None: + needs_save = True + with self.vmware.client_session() as vmware_client: + datacenter = self.vmware.get_datacenter(vmware_client, model.region.datacenter) + datastore = self.vmware.get_datastore(vmware_client, zone.vm_datastore, datacenter) + if model.task == VolumeTask.ATTACHING: + instance = Instance.get(model.project, model.task_kwargs['to']) + if instance is None: + # Attaching to instance doesn't exist + model.task = None + return + if instance.state in [ResourceState.ToDelete, ResourceState.Deleting, ResourceState.Deleted, + ResourceState.Error]: + # Attaching to instance is deleting or errored. + model.task = None + return + vm = self.vmware.get_vm(vmware_client, str(instance.name), datacenter) + if vm is None: + # VM doesn't exist + model.task = None + return + self.vmware.attach_disk(vmware_client, model.backing_id, datastore, vm) + model.attached_to = instance model.task = None - return - vm = self.vmware.get_vm(vmware_client, str(instance.id), datacenter) - if vm is None: - # VM doesn't exist + elif model.task == VolumeTask.DETACHING: + self.detach_disk(vmware_client, datacenter, model) + model.attached_to = None model.task = None - return - self.vmware.attach_disk(vmware_client, model.backing_id, datastore, vm) - model.attached_to = instance - model.task = None - elif model.task == VolumeTask.DETACHING: - self.detach_disk(vmware_client, datacenter, model) - model.attached_to = None - model.task = None - elif model.task == VolumeTask.GROWING: - self.vmware.grow_disk(vmware_client, model.backing_id, model.task_kwargs['size'], datastore) - model.size = model.task_kwargs['size'] - model.task = None - elif model.task == VolumeTask.CLONING: - # Check new volume - # If it's none, created or errored then we are done cloning - new_volume = Volume.get(model.project, model.task_kwargs['volume_id']) - if new_volume is None or new_volume.state in [ResourceState.Created, ResourceState.Error]: + elif model.task == VolumeTask.GROWING: + self.vmware.grow_disk(vmware_client, model.backing_id, model.task_kwargs['size'], datastore) + model.size = model.task_kwargs['size'] model.task = None + elif model.task == VolumeTask.CLONING: + # Check new volume + # If it's none, created or errored then we are done cloning + new_volume = Volume.get(model.project, model.task_kwargs['volume_name']) + if new_volume is None or new_volume.state in [ResourceState.Created, ResourceState.Error]: + model.task = None - if model.attached_to_id is not None: - if model.attached_to is None: - model.attached_to = None + if zone.state == ResourceState.Deleting: + model.state = ResourceState.ToDelete + needs_save = True + + if model.attached_to_name is not None: + if model.attached_to is None: + model.attached_to = None + needs_save = True - if model.cloned_from_id is not None: - if model.cloned_from is None: - model.cloned_from = None + if model.cloned_from_name is not None: + if model.cloned_from is None: + model.cloned_from = None + needs_save = True def detach_disk(self, vmware_client, datacenter, model): - vm = self.vmware.get_vm(vmware_client, str(model.attached_to_id), datacenter) + vm = self.vmware.get_vm(vmware_client, str(model.attached_to_name), datacenter) if vm is not None: self.vmware.detach_disk(vmware_client, model.backing_id, vm) model.attached_to = None diff --git a/deli/kubernetes/resources/v1alpha1/volume/model.py b/deli/kubernetes/resources/v1alpha1/volume/model.py index a902ce4..2a23085 100644 --- a/deli/kubernetes/resources/v1alpha1/volume/model.py +++ b/deli/kubernetes/resources/v1alpha1/volume/model.py @@ -1,5 +1,4 @@ import enum -import uuid from deli.kubernetes.resources.const import REGION_LABEL, ZONE_LABEL, ATTACHED_TO_LABEL from deli.kubernetes.resources.model import ProjectResourceModel @@ -34,37 +33,37 @@ def __init__(self, raw=None): } @property - def region_id(self): - region_id = self._raw['metadata']['labels'][REGION_LABEL] - if region_id == "": + def region_name(self): + region_name = self._raw['metadata']['labels'][REGION_LABEL] + if region_name == "": return None - return uuid.UUID(region_id) + return region_name @property def region(self): - region_id = self.region_id - if region_id is None: + region_name = self.region_name + if region_name is None: return None - return Region.get(region_id) + return Region.get(region_name) @property - def zone_id(self): - zone_id = self._raw['metadata']['labels'][ZONE_LABEL] - if zone_id == "": + def zone_name(self): + zone_name = self._raw['metadata']['labels'][ZONE_LABEL] + if zone_name == "": return None - return uuid.UUID(zone_id) + return zone_name @property def zone(self): - zone_id = self.zone_id - if zone_id is None: + zone_name = self.zone_name + if zone_name is None: return None - return Zone.get(zone_id) + return Zone.get(zone_name) @zone.setter def zone(self, value): - self._raw['metadata']['labels'][REGION_LABEL] = str(value.region.id) - self._raw['metadata']['labels'][ZONE_LABEL] = str(value.id) + self._raw['metadata']['labels'][REGION_LABEL] = str(value.region.name) + self._raw['metadata']['labels'][ZONE_LABEL] = str(value.name) @property def size(self): @@ -83,15 +82,15 @@ def backing_id(self, value): self._raw['spec']['backingId'] = value @property - def cloned_from_id(self): + def cloned_from_name(self): if self._raw['spec']['clonedFrom'] is None: return None - return uuid.UUID(self._raw['spec']['clonedFrom']) + return self._raw['spec']['clonedFrom'] @property def cloned_from(self): - if self.cloned_from_id is not None: - return Volume.get(self.project, str(self.cloned_from_id)) + if self.cloned_from_name is not None: + return Volume.get(self.project, str(self.cloned_from_name)) return None @cloned_from.setter @@ -99,28 +98,28 @@ def cloned_from(self, value): if value is None: self._raw['spec']['clonedFrom'] = None else: - self._raw['spec']['clonedFrom'] = str(value.id) + self._raw['spec']['clonedFrom'] = str(value.name) @property - def attached_to_id(self): - instance_id = self._raw['metadata']['labels'][ATTACHED_TO_LABEL] - if instance_id == "": + def attached_to_name(self): + instance_name = self._raw['metadata']['labels'][ATTACHED_TO_LABEL] + if instance_name == "": return None - return uuid.UUID(instance_id) + return instance_name @property def attached_to(self): - instance_id = self.attached_to_id - if instance_id is None: + instance_name = self.attached_to_name + if instance_name is None: return None - return Instance.get(self.project, instance_id) + return Instance.get(self.project, instance_name) @attached_to.setter def attached_to(self, value): if value is None: self._raw['metadata']['labels'][ATTACHED_TO_LABEL] = "" else: - self._raw['metadata']['labels'][ATTACHED_TO_LABEL] = str(value.id) + self._raw['metadata']['labels'][ATTACHED_TO_LABEL] = str(value.name) @property def task(self): @@ -147,5 +146,5 @@ def task_kwargs(self, value): def attach(self, instance: Instance): self.task = VolumeTask.ATTACHING self.task_kwargs = { - 'to': str(instance.id) + 'to': str(instance.name) } diff --git a/deli/kubernetes/resources/v1alpha1/zone/controller.py b/deli/kubernetes/resources/v1alpha1/zone/controller.py index 301fd4e..c2b2ed2 100644 --- a/deli/kubernetes/resources/v1alpha1/zone/controller.py +++ b/deli/kubernetes/resources/v1alpha1/zone/controller.py @@ -76,11 +76,11 @@ def deleted(self, model): def can_delete(self, model): # These resources need the zone to exist to successfully delete - instances = Instance.list_all(label_selector=ZONE_LABEL + "=" + str(model.id)) + instances = Instance.list_all(label_selector=ZONE_LABEL + "=" + str(model.name)) if len(instances) > 0: return False - volumes = Volume.list_all(label_selector=ZONE_LABEL + "=" + str(model.id)) + volumes = Volume.list_all(label_selector=ZONE_LABEL + "=" + str(model.name)) if len(volumes) > 0: return False diff --git a/deli/kubernetes/resources/v1alpha1/zone/model.py b/deli/kubernetes/resources/v1alpha1/zone/model.py index de17844..5b12b36 100644 --- a/deli/kubernetes/resources/v1alpha1/zone/model.py +++ b/deli/kubernetes/resources/v1alpha1/zone/model.py @@ -1,11 +1,9 @@ -import uuid - from deli.kubernetes.resources.const import REGION_LABEL -from deli.kubernetes.resources.model import GlobalResourceModel +from deli.kubernetes.resources.model import SystemResourceModel from deli.kubernetes.resources.v1alpha1.region.model import Region -class Zone(GlobalResourceModel): +class Zone(SystemResourceModel): def __init__(self, raw=None): super().__init__(raw) @@ -22,22 +20,22 @@ def __init__(self, raw=None): } @property - def region_id(self): - region_id = self._raw['metadata']['labels'][REGION_LABEL] - if region_id == "": + def region_name(self): + region_name = self._raw['metadata']['labels'][REGION_LABEL] + if region_name == "": return None - return uuid.UUID(region_id) + return region_name @property def region(self): - region_id = self.region_id - if region_id is None: + region_name = self.region_name + if region_name is None: return None - return Region.get(region_id) + return Region.get(region_name) @region.setter def region(self, value): - self._raw['metadata']['labels'][REGION_LABEL] = str(value.id) + self._raw['metadata']['labels'][REGION_LABEL] = value.name @property def vm_cluster(self): diff --git a/deli/manager/cli/commands/run.py b/deli/manager/cli/commands/run.py index fbbb132..3b5b683 100644 --- a/deli/manager/cli/commands/run.py +++ b/deli/manager/cli/commands/run.py @@ -1,4 +1,5 @@ import argparse +import datetime import enum import ipaddress import json @@ -16,8 +17,16 @@ from kubernetes import config, client from kubernetes.client import Configuration +from deli.cache import cache_client from deli.kubernetes.resources.v1alpha1.flavor.controller import FlavorController from deli.kubernetes.resources.v1alpha1.flavor.model import Flavor +from deli.kubernetes.resources.v1alpha1.iam_policy.controller import IAMPolicyController +from deli.kubernetes.resources.v1alpha1.iam_policy.model import IAMPolicy +from deli.kubernetes.resources.v1alpha1.iam_role.controller import IAMSystemRoleController, IAMProjectRoleController +from deli.kubernetes.resources.v1alpha1.iam_role.model import IAMSystemRole, IAMProjectRole +from deli.kubernetes.resources.v1alpha1.iam_service_account.controller import SystemServiceAccountController, \ + ProjectServiceAccountController +from deli.kubernetes.resources.v1alpha1.iam_service_account.model import SystemServiceAccount, ProjectServiceAccount from deli.kubernetes.resources.v1alpha1.image.controller import ImageController from deli.kubernetes.resources.v1alpha1.image.model import Image from deli.kubernetes.resources.v1alpha1.instance.controller import InstanceController @@ -26,17 +35,10 @@ from deli.kubernetes.resources.v1alpha1.keypair.keypair import Keypair from deli.kubernetes.resources.v1alpha1.network.controller import NetworkController, NetworkPortController from deli.kubernetes.resources.v1alpha1.network.model import Network, NetworkPort -from deli.kubernetes.resources.v1alpha1.project_member.controller import ProjectMemberController -from deli.kubernetes.resources.v1alpha1.project_member.model import ProjectMember from deli.kubernetes.resources.v1alpha1.project_quota.controller import ProjectQuotaController from deli.kubernetes.resources.v1alpha1.project_quota.model import ProjectQuota from deli.kubernetes.resources.v1alpha1.region.controller import RegionController from deli.kubernetes.resources.v1alpha1.region.model import Region -from deli.kubernetes.resources.v1alpha1.role.controller import GlobalRoleController, ProjectRoleController -from deli.kubernetes.resources.v1alpha1.role.model import GlobalRole, ProjectRole -from deli.kubernetes.resources.v1alpha1.service_account.controller import GlobalServiceAccountController, \ - ProjectServiceAccountController -from deli.kubernetes.resources.v1alpha1.service_account.model import GlobalServiceAccount, ProjectServiceAccount from deli.kubernetes.resources.v1alpha1.volume.controller import VolumeController from deli.kubernetes.resources.v1alpha1.volume.model import Volume from deli.kubernetes.resources.v1alpha1.zone.controller import ZoneController @@ -92,7 +94,12 @@ def setup_arguments(self, parser): required_group.add_argument("--menu-url", action=EnvDefault, envvar="MENU_URL", required=True, help="Telnet URL to the menu server") + required_group.add_argument("--redis-url", action=EnvDefault, envvar="REDIS_URL", required=True, + help="URL to the redis server for caching") + def run(self, args) -> int: + cache_client.connect(url=args.redis_url) + if args.kube_config != "" or args.kube_master != "": self.logger.info("Using kube-config configuration") Configuration.set_default(Configuration()) @@ -126,28 +133,34 @@ def json_encoder(self, o): # pragma: no cover return str(o) if isinstance(o, enum.Enum): return o.value + if isinstance(o, datetime.datetime): + return str(o.isoformat()) return old_json_encoder(self, o) json.JSONEncoder.default = json_encoder self.logger.info("Creating CRDs") - GlobalRole.create_crd() - GlobalRole.wait_for_crd() - GlobalRole.create_default_roles() - ProjectRole.create_crd() - ProjectRole.wait_for_crd() - ProjectMember.create_crd() - ProjectMember.wait_for_crd() - ProjectQuota.create_crd() - ProjectQuota.wait_for_crd() + IAMSystemRole.create_crd() + IAMSystemRole.wait_for_crd() + IAMProjectRole.create_crd() + IAMProjectRole.wait_for_crd() - GlobalServiceAccount.create_crd() - GlobalServiceAccount.wait_for_crd() - GlobalServiceAccount.create_admin_sa() + IAMPolicy.create_crd() + IAMPolicy.wait_for_crd() + IAMPolicy.create_system_policy() + + SystemServiceAccount.create_crd() + SystemServiceAccount.wait_for_crd() ProjectServiceAccount.create_crd() ProjectServiceAccount.wait_for_crd() + IAMSystemRole.create_default_roles() + SystemServiceAccount.create_admin_sa() + + ProjectQuota.create_crd() + ProjectQuota.wait_for_crd() + Region.create_crd() Region.wait_for_crd() Zone.create_crd() @@ -190,14 +203,14 @@ def on_started_leading(self): self.logger.info("Started leading... starting controllers") self.launch_controller(RegionController(1, 30, self.vmware)) self.launch_controller(ZoneController(1, 30, self.vmware)) - self.launch_controller(GlobalRoleController(1, 30)) - self.launch_controller(ProjectRoleController(1, 30)) - self.launch_controller(ProjectMemberController(1, 30)) + self.launch_controller(IAMSystemRoleController(1, 30)) + self.launch_controller(IAMProjectRoleController(1, 30)) + self.launch_controller(IAMPolicyController(1, 30)) self.launch_controller(ProjectQuotaController(1, 30)) self.launch_controller(NetworkController(1, 30, self.vmware)) self.launch_controller(NetworkPortController(1, 30)) self.launch_controller(ImageController(4, 30, self.vmware)) - self.launch_controller(GlobalServiceAccountController(1, 30)) + self.launch_controller(SystemServiceAccountController(1, 30)) self.launch_controller(ProjectServiceAccountController(1, 30)) self.launch_controller(FlavorController(1, 30)) self.launch_controller(VolumeController(4, 30, self.vmware)) diff --git a/deli/manager/vmware.py b/deli/manager/vmware.py index 87c211f..987175d 100644 --- a/deli/manager/vmware.py +++ b/deli/manager/vmware.py @@ -30,8 +30,8 @@ def get_folder(self, vmware_client, folder_name, datacenter): def get_image(self, vmware_client, image_name, datacenter): return self.get_obj(vmware_client, vim.VirtualMachine, image_name, folder=datacenter.vmFolder) - def get_vm(self, vmware_client, vm_name, datacenter): - return self.get_obj(vmware_client, vim.VirtualMachine, vm_name, folder=datacenter.vmFolder) + def get_vm(self, vmware_client, vm_uuid, datacenter): + return vmware_client.content.searchIndex.FindByUuid(datacenter, vm_uuid, vmSearch=True, instanceUuid=True) def get_cluster(self, vmware_client, cluster_name, datacenter): return self.get_obj(vmware_client, vim.ClusterComputeResource, cluster_name, folder=datacenter.hostFolder) @@ -61,7 +61,7 @@ def create_disk(self, vmware_client, disk_name, size, datastore): self.wait_for_tasks(vmware_client, [task]) vStorageObject = task.info.result - return vStorageObject.config.id.id + return vStorageObject.config.name.name def clone_disk(self, vmware_client, disk_name, disk_id, datastore): vStorageManager = vmware_client.RetrieveContent().vStorageObjectManager @@ -261,7 +261,7 @@ def clone_and_template_vm(self, vm, datastore, folder): clonespec.powerOn = False clonespec.template = True - file_name = str(uuid.uuid4()) + file_name = "sandwich-" + str(uuid.uuid4()) task = vm.Clone(folder=folder, name=file_name, spec=clonespec) return task, file_name diff --git a/deli/menu/cli/commands/run.py b/deli/menu/cli/commands/run.py index d5d3497..73de59f 100644 --- a/deli/menu/cli/commands/run.py +++ b/deli/menu/cli/commands/run.py @@ -9,12 +9,14 @@ import arrow import urllib3 -from clify.daemon import Daemon from dotenv import load_dotenv from kubernetes import config, client from kubernetes.client import Configuration +from vmw_cloudinit_metadata.cli.commands.run import RunMetadata +from vmw_cloudinit_metadata.vspc.server import VSPCServer -from deli.menu.vspc.server import VSPCServer +from deli.cache import cache_client +from deli.menu.metadata.driver import SandwichDriver class EnvDefault(argparse.Action): @@ -35,10 +37,7 @@ def __call__(self, parser, namespace, values, option_string=None): # pragma: no setattr(namespace, self.dest, values) -class RunMetadata(Daemon): - def __init__(self): - super().__init__('run', 'Run the Sandwich Cloud Metadata Server') - self.vspc_server = None +class RunMetadataMenu(RunMetadata): def setup_arguments(self, parser): load_dotenv(os.path.join(os.getcwd(), '.env')) @@ -46,41 +45,35 @@ def setup_arguments(self, parser): help="Path to a kubeconfig. Only required if out-of-cluster.") parser.add_argument('--kube-master', action=EnvDefault, envvar="KUBEMASTER", required=False, default="", help="The address of the Kubernetes API server (overrides any value in kubeconfig)") - parser.add_argument('--out-of-band', action=EnvDefault, envvar="OUT_OF_BAND", required=False, default=None, - help="Path to .yaml (.yml) files to use for metadata") parser.add_argument("--fernet-key", action=EnvDefault, envvar="FERNET_KEY", required=True, help="The fernet key to use to hand out tokens.") + parser.add_argument("--redis-url", action=EnvDefault, envvar="REDIS_URL", required=True, + help="URL to the redis server for caching") def run(self, args) -> int: + cache_client.connect(url=args.redis_url) + + self.logger.info("Using Kubernetes configuration for metadata") + if args.kube_config != "" or args.kube_master != "": + self.logger.info("Using kube-config configuration") + Configuration.set_default(Configuration()) + if args.kube_config != "": + config.load_kube_config(config_file=args.kube_config) + if args.kube_master != "": + Configuration._default.host = args.kube_master - if hasattr(args, "out_of_band"): - self.logger.info("Using out of band configuration for metadata") - if os.path.isdir(args.out_of_band) is False: - self.logger.error("Could not find directory for out of band configuration " + args.out_of_band) - return 1 - os.environ['OUT_OF_BAND'] = args.out_of_band else: - self.logger.info("Using Kubernetes configuration for metadata") - if args.kube_config != "" or args.kube_master != "": - self.logger.info("Using kube-config configuration") - Configuration.set_default(Configuration()) - if args.kube_config != "": - config.load_kube_config(config_file=args.kube_config) - if args.kube_master != "": - Configuration._default.host = args.kube_master - - else: - self.logger.info("Using in-cluster configuration") - config.load_incluster_config() - - while True: - try: - client.CoreV1Api().list_namespace() - break - except urllib3.exceptions.HTTPError as e: - self.logger.error( - "Error connecting to the Kubernetes API. Trying again in 5 seconds. Error: " + str(e)) - time.sleep(5) + self.logger.info("Using in-cluster configuration") + config.load_incluster_config() + + while True: + try: + client.CoreV1Api().list_namespace() + break + except urllib3.exceptions.HTTPError as e: + self.logger.error( + "Error connecting to the Kubernetes API. Trying again in 5 seconds. Error: " + str(e)) + time.sleep(5) os.environ['FERNET_KEY'] = args.fernet_key @@ -102,12 +95,6 @@ def json_encoder(self, o): # pragma: no cover json.JSONEncoder.default = json_encoder - self.vspc_server = VSPCServer('sandwich') + self.vspc_server = VSPCServer("sandwich", SandwichDriver({})) self.vspc_server.start() - return 0 - - def on_shutdown(self, signum=None, frame=None): - self.logger.info("Shutting down the Metadata Server") - if self.vspc_server is not None: - self.vspc_server.stop() diff --git a/deli/menu/cli/main.py b/deli/menu/cli/main.py index 92ec562..1b7f067 100644 --- a/deli/menu/cli/main.py +++ b/deli/menu/cli/main.py @@ -1,8 +1,8 @@ from deli.menu.cli.app import MetadataApplication -from deli.menu.cli.commands.run import RunMetadata +from deli.menu.cli.commands.run import RunMetadataMenu def main(): app = MetadataApplication() - RunMetadata().register(app) + RunMetadataMenu().register(app) app.run() diff --git a/deli/menu/metadata/__init__.py b/deli/menu/metadata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/deli/menu/metadata/driver.py b/deli/menu/metadata/driver.py new file mode 100644 index 0000000..8d7034b --- /dev/null +++ b/deli/menu/metadata/driver.py @@ -0,0 +1,64 @@ +from typing import Optional + +from deli.kubernetes.resources.const import VM_ID_LABEL +from deli.kubernetes.resources.v1alpha1.instance.model import Instance +from deli.kubernetes.resources.v1alpha1.network.model import Network +from deli.menu.metadata.vm_client import SandwichVMClient +from vmw_cloudinit_metadata.drivers.driver import Driver +from vmw_cloudinit_metadata.vspc.vm_client import InstanceData, InstanceMetadata, InstanceNetworkData + + +class SandwichDriver(Driver): + + def new_client(self, vm_name, writer): + return SandwichVMClient(vm_name, writer, self) + + def parse_options(self, opts): + pass + + def get_sandwich_instance(self, vm_client) -> Optional[Instance]: + vm_uuid = vm_client.vm_vc_uuid + instances = Instance.list_all(label_selector=VM_ID_LABEL + "=" + str(vm_uuid)) + if len(instances) == 0: + self.logger.warning("Could not find any instances with the uuid of '%s'" % vm_uuid) + return None + if len(instances) > 1: + self.logger.warning("Found multiple instances with the uuid of '%s'" % vm_uuid) + return None + return instances[0] + + def get_instance(self, vm_name) -> Optional[InstanceData]: + instance = self.get_sandwich_instance(vm_name) + if instance is None: + return None + network_port = instance.network_port + network: Network = network_port.network + + keypairs = [] + for keypair in instance.keypairs: + keypairs.append(keypair.public_key) + + instance_data = InstanceData() + + metadata = InstanceMetadata() + metadata.ami_id = instance.image_name + metadata.instance_id = instance.name + metadata.region = instance.region.name + metadata.availability_zone = instance.zone.name + metadata.tags = instance.tags + metadata.public_keys = keypairs + metadata.hostname = 'ip-' + str(network_port.ip_address).replace(".", "-") + instance_data.metadata = metadata + + network_data = InstanceNetworkData() + network_data.address = str(network_port.ip_address) + network_data.netmask = network.cidr.with_netmask.split("/")[1] + network_data.gateway = str(network.gateway) + network_data.search = ["sandwich.local"] + network_data.nameservers = [str(ns) for ns in network.dns_servers] + instance_data.network = network_data + + instance_data.userdata = instance.user_data + + instance_data.validate() + return instance_data diff --git a/deli/menu/metadata/vm_client.py b/deli/menu/metadata/vm_client.py new file mode 100644 index 0000000..8ca73a6 --- /dev/null +++ b/deli/menu/metadata/vm_client.py @@ -0,0 +1,60 @@ +import enum +import os + +import arrow +from cryptography.fernet import Fernet +from vmw_cloudinit_metadata.vspc.vm_client import VMClient + +from deli.counter.auth.token import Token +from deli.kubernetes.resources.v1alpha1.instance.model import Instance + + +class PacketCode(enum.Enum): + # Incoming packets + REQUEST_METADATA = 'REQUEST_METADATA' + REQUEST_NETWORKDATA = 'REQUEST_NETWORKDATA' + REQUEST_USERDATA = 'REQUEST_USERDATA' + REQUEST_SECURITYDATA = 'REQUEST_SECURITYDATA' + # Outgoing packets + RESPONSE_METADATA = 'RESPONSE_METADATA' + RESPONSE_NETWORKDATA = 'RESPONSE_NETWORKDATA' + RESPONSE_USERDATA = 'RESPONSE_USERDATA' + RESPONSE_SECURITYDATA = 'RESPONSE_SECURITYDATA' + + +class SandwichVMClient(VMClient): + + def __init__(self, vm_name, writer, driver): + self.fernet = Fernet(os.environ['FERNET_KEY']) + super().__init__(vm_name, writer, driver) + + async def write_security_data(self): + + instance: Instance = self.driver.get_sandwich_instance(self) + if instance is None: + return + + token = Token() + token.email = instance.service_account.email + token.metadata['instance'] = instance.name + token.expires_at = arrow.now('UTC').shift(minutes=+30) + + await self.write(PacketCode.RESPONSE_SECURITYDATA, token.marshal(self.fernet).decode()) + + async def process_packets(self, packet_code, data): + try: + packet_code = PacketCode(packet_code) + except ValueError: + self.logger.error("Received unknown packet code '%s' from vm '%s'" % (packet_code, self.vm_name)) + return + + self.logger.debug("Received packet code '%s' from vm '%s'" % (packet_code, self.vm_name)) + + if packet_code == PacketCode.REQUEST_METADATA: + await self.write_metadata() + elif packet_code == PacketCode.REQUEST_USERDATA: + await self.write_userdata() + elif packet_code == PacketCode.REQUEST_NETWORKDATA: + await self.write_networkdata() + elif packet_code == PacketCode.REQUEST_SECURITYDATA: + await self.write_security_data() diff --git a/deli/menu/models/out_of_band.py b/deli/menu/models/out_of_band.py deleted file mode 100644 index a0a0d4b..0000000 --- a/deli/menu/models/out_of_band.py +++ /dev/null @@ -1,30 +0,0 @@ -import ipaddress - -from ingredients_http.schematics.types import IPv4NetworkType, IPv4AddressType -from schematics import Model -from schematics.exceptions import ValidationError -from schematics.types import StringType, ModelType, DictType, ListType - - -class OutOfBandNetwork(Model): - cidr = IPv4NetworkType(required=True) - gateway = IPv4AddressType(required=True) - ip_address = IPv4AddressType(required=True) - dns_servers = ListType(IPv4AddressType, required=True, min_size=1) - - def validate_gateway(self, data, value): - cidr: ipaddress.IPv4Network = data['cidr'] - - if value not in cidr: - raise ValidationError('is not an address within ' + str(cidr)) - - return value - - -class OutOfBandInstance(Model): - region_name = StringType(required=True) - zone_name = StringType(required=True) - network = ModelType(OutOfBandNetwork, required=True) - keypairs = ListType(StringType, default=list) - tags = DictType(StringType, default=dict) - user_data = StringType(default="#cloud-config\n{}") diff --git a/deli/menu/vspc/async_telnet.py b/deli/menu/vspc/async_telnet.py deleted file mode 100644 index 5b03e0d..0000000 --- a/deli/menu/vspc/async_telnet.py +++ /dev/null @@ -1,185 +0,0 @@ -# This file has been taken and modified from -# https://github.com/openstack/vmware-vspc/blob/a4c2934d509cea638208e31658a3eb8f2fdc7f08/vspc/async_telnet.py -# It is therefore using a different license on copyright as shown bellow - -# Copyright (c) 2001-2016 Python Software Foundation -# Copyright (c) 2017 VMware Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Telnet client class using asyncio. -Based on the standard telnetlib module from Python3. -""" -import asyncio - -# Telnet protocol characters (don't change) -IAC = bytes([255]) # "Interpret As Command" -DONT = bytes([254]) -DO = bytes([253]) -WONT = bytes([252]) -WILL = bytes([251]) -theNULL = bytes([0]) -SE = bytes([240]) # Subnegotiation End -NOP = bytes([241]) # No Operation -SB = bytes([250]) # Subnegotiation Begin -CR = bytes([13]) # Carriage return -NOOPT = bytes([0]) - - -class AsyncTelnet: - def __init__(self, reader, opt_handler): - self._reader = reader - self._opt_handler = opt_handler - self.rawq = b'' - self.irawq = 0 - self.cookedq = b'' - self.eof = 0 - self.iacseq = b'' # Buffer for IAC sequence. - self.sb = 0 # flag for SB and SE sequence. - self.sbdataq = b'' - - @asyncio.coroutine - def process_rawq(self): - """Transfer from raw queue to cooked queue. - - Set self.eof when connection is closed. - """ - buf = [b'', b''] - try: - while self.rawq: - c = yield from self.rawq_getchar() - if not self.iacseq: - if self.sb == 0 and c == theNULL: - continue - if self.sb == 0 and c == b"\021": - continue - if c != IAC: - buf[self.sb] = buf[self.sb] + c - continue - else: - self.iacseq += c - elif len(self.iacseq) == 1: - # 'IAC: IAC CMD [OPTION only for WILL/WONT/DO/DONT]' - if c in (DO, DONT, WILL, WONT): - self.iacseq += c - continue - - self.iacseq = b'' - if c == IAC: - buf[self.sb] = buf[self.sb] + c - else: - if c == SB: # SB ... SE start. - self.sb = 1 - self.sbdataq = b'' - elif c == SE: - self.sb = 0 - self.sbdataq = self.sbdataq + buf[1] - buf[1] = b'' - yield from self._opt_handler(c, NOOPT, - data=self.sbdataq) - elif len(self.iacseq) == 2: - cmd = self.iacseq[1:2] - self.iacseq = b'' - opt = c - if cmd in (DO, DONT): - yield from self._opt_handler(cmd, opt) - elif cmd in (WILL, WONT): - yield from self._opt_handler(cmd, opt) - except EOFError: # raised by self.rawq_getchar() - self.iacseq = b'' # Reset on EOF - self.sb = 0 - pass - self.cookedq = self.cookedq + buf[0] - self.sbdataq = self.sbdataq + buf[1] - - @asyncio.coroutine - def rawq_getchar(self): - """Get next char from raw queue. - - Raise EOFError when connection is closed. - """ - if not self.rawq: - yield from self.fill_rawq() - if self.eof: - raise EOFError - c = self.rawq[self.irawq:self.irawq + 1] - self.irawq = self.irawq + 1 - if self.irawq >= len(self.rawq): - self.rawq = b'' - self.irawq = 0 - return c - - @asyncio.coroutine - def fill_rawq(self): - """Fill raw queue from exactly one recv() system call. - - Set self.eof when connection is closed. - """ - if self.irawq >= len(self.rawq): - self.rawq = b'' - self.irawq = 0 - # The buffer size should be fairly small so as to avoid quadratic - # behavior in process_rawq() above - buf = yield from self._reader.read(50) - self.eof = (not buf) - self.rawq = self.rawq + buf - - @asyncio.coroutine - def read_some(self): - """Read at least one byte of cooked data unless EOF is hit. - - Return b'' if EOF is hit. - """ - yield from self.process_rawq() - while not self.cookedq and not self.eof: - yield from self.fill_rawq() - yield from self.process_rawq() - buf = self.cookedq - self.cookedq = b'' - return buf - - @asyncio.coroutine - def read_byte(self): - """Read one byte of cooked data - """ - buf = b'' - if len(self.cookedq) > 0: - buf = bytes([self.cookedq[0]]) - self.cookedq = self.cookedq[1:] - else: - yield from self.process_rawq() - if not self.eof: - yield from self.fill_rawq() - yield from self.process_rawq() - # There now should be data so lets read again - buf = yield from self.read_byte() - - return buf - - @asyncio.coroutine - def read_line(self): - """Read data until \n is found - """ - buf = b'' - while not self.eof and buf.endswith(b'\n') is False: - buf += yield from self.read_byte() - - if self.eof: - buf = b'' - - # Remove \n character - buf = buf.replace(b'\n', b'') - - return buf diff --git a/deli/menu/vspc/server.py b/deli/menu/vspc/server.py deleted file mode 100644 index d725917..0000000 --- a/deli/menu/vspc/server.py +++ /dev/null @@ -1,231 +0,0 @@ -# This file has been taken and modified from -# https://github.com/openstack/vmware-vspc/blob/a4c2934d509cea638208e31658a3eb8f2fdc7f08/vspc/server.py -# It is therefore using a different license on copyright as shown bellow - -# Copyright (c) 2017 VMware Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import asyncio -import base64 -import functools -import logging -import os - -from deli.menu.vspc import async_telnet -from deli.menu.vspc.async_telnet import IAC, SB, SE, WILL, WONT, DONT, DO -from deli.menu.vspc.vm_client import VMClient - -BINARY = bytes([0]) # 8-bit data path -SGA = bytes([3]) # suppress go ahead -VMWARE_EXT = bytes([232]) - -KNOWN_SUBOPTIONS_1 = bytes([0]) -KNOWN_SUBOPTIONS_2 = bytes([1]) -VMOTION_BEGIN = bytes([40]) -VMOTION_GOAHEAD = bytes([41]) -VMOTION_NOTNOW = bytes([43]) -VMOTION_PEER = bytes([44]) -VMOTION_PEER_OK = bytes([45]) -VMOTION_COMPLETE = bytes([46]) -VMOTION_ABORT = bytes([48]) -VM_VC_UUID = bytes([80]) -GET_VM_VC_UUID = bytes([81]) -VM_NAME = bytes([82]) -GET_VM_NAME = bytes([83]) -DO_PROXY = bytes([70]) -WILL_PROXY = bytes([71]) -WONT_PROXY = bytes([73]) - -SUPPORTED_OPTS = (KNOWN_SUBOPTIONS_1 + KNOWN_SUBOPTIONS_2 + VMOTION_BEGIN + - VMOTION_GOAHEAD + VMOTION_NOTNOW + VMOTION_PEER + - VMOTION_PEER_OK + VMOTION_COMPLETE + VMOTION_ABORT + - VM_VC_UUID + GET_VM_VC_UUID + VM_NAME + GET_VM_NAME + - DO_PROXY + WILL_PROXY + WONT_PROXY) - - -class VSPCServer(object): - def __init__(self, uri): - self.logger = logging.getLogger("%s.%s" % (self.__module__, self.__class__.__name__)) - self.uri = uri - self.sock_to_client = {} - - self.server = None - self.loop = None - - async def handle_known_suboptions(self, writer, data): - socket = writer.get_extra_info('socket') - peer = socket.getpeername() - self.logger.debug("<< %s KNOWN-SUBOPTIONS-1 %s", peer, data) - self.logger.debug(">> %s KNOWN-SUBOPTIONS-2 %s", peer, SUPPORTED_OPTS) - writer.write(IAC + SB + VMWARE_EXT + KNOWN_SUBOPTIONS_2 + - SUPPORTED_OPTS + IAC + SE) - self.logger.debug(">> %s GET-VM-NAME", peer) - writer.write(IAC + SB + VMWARE_EXT + GET_VM_NAME + IAC + SE) - await writer.drain() - - async def handle_do_proxy(self, writer, data): - socket = writer.get_extra_info('socket') - peer = socket.getpeername() - dir, uri = data[0], data[1:].decode('ascii') - self.logger.debug("<< %s DO-PROXY %c %s", peer, dir, uri) - if chr(dir) != 'C' or uri != self.uri: - self.logger.debug(">> %s WONT-PROXY", peer) - writer.write(IAC + SB + VMWARE_EXT + WONT_PROXY + IAC + SE) - await writer.drain() - writer.close() - else: - self.logger.debug(">> %s WILL-PROXY", peer) - writer.write(IAC + SB + VMWARE_EXT + WILL_PROXY + IAC + SE) - await writer.drain() - - def handle_vm_name(self, socket, writer, data): - peer = socket.getpeername() - vm_name = data.decode('ascii') - self.logger.debug("<< %s VM-NAME %s", peer, vm_name) - self.sock_to_client[socket] = VMClient(vm_name, writer) - - async def handle_vmotion_begin(self, writer, data): - socket = writer.get_extra_info('socket') - peer = socket.getpeername() - self.logger.debug("<< %s VMOTION-BEGIN %s", peer, data) - secret = os.urandom(4) - self.logger.debug(">> %s VMOTION-GOAHEAD %s %s", peer, data, secret) - writer.write(IAC + SB + VMWARE_EXT + VMOTION_GOAHEAD + - data + secret + IAC + SE) - await writer.drain() - - async def handle_vmotion_peer(self, writer, data): - socket = writer.get_extra_info('socket') - peer = socket.getpeername() - self.logger.debug("<< %s VMOTION-PEER %s", peer, data) - self.logger.debug("<< %s VMOTION-PEER-OK %s", peer, data) - writer.write(IAC + SB + VMWARE_EXT + VMOTION_PEER_OK + data + IAC + SE) - await writer.drain() - - def handle_vmotion_complete(self, socket, data): - peer = socket.getpeername() - self.logger.debug("<< %s VMOTION-COMPLETE %s", peer, data) - - async def handle_do(self, writer, opt): - socket = writer.get_extra_info('socket') - peer = socket.getpeername() - self.logger.debug("<< %s DO %s", peer, opt) - if opt in (BINARY, SGA): - self.logger.debug(">> %s WILL", peer) - writer.write(IAC + WILL + opt) - await writer.drain() - else: - self.logger.debug(">> %s WONT", peer) - writer.write(IAC + WONT + opt) - await writer.drain() - - async def handle_will(self, writer, opt): - socket = writer.get_extra_info('socket') - peer = socket.getpeername() - self.logger.debug("<< %s WILL %s", peer, opt) - if opt in (BINARY, SGA, VMWARE_EXT): - self.logger.debug(">> %s DO", peer) - writer.write(IAC + DO + opt) - await writer.drain() - else: - self.logger.debug(">> %s DONT", peer) - writer.write(IAC + DONT + opt) - await writer.drain() - - async def option_handler(self, cmd, opt, writer, data=None): - socket = writer.get_extra_info('socket') - if cmd == SE and data[0:1] == VMWARE_EXT: - vmw_cmd = data[1:2] - if vmw_cmd == KNOWN_SUBOPTIONS_1: - await self.handle_known_suboptions(writer, data[2:]) - elif vmw_cmd == DO_PROXY: - await self.handle_do_proxy(writer, data[2:]) - elif vmw_cmd == VM_NAME: - self.handle_vm_name(socket, writer, data[2:]) - elif vmw_cmd == VMOTION_BEGIN: - await self.handle_vmotion_begin(writer, data[2:]) - elif vmw_cmd == VMOTION_PEER: - await self.handle_vmotion_peer(writer, data[2:]) - elif vmw_cmd == VMOTION_COMPLETE: - self.handle_vmotion_complete(socket, data[2:]) - else: - self.logger.error("Unknown VMware cmd: %s %s", vmw_cmd, data[2:]) - writer.close() - elif cmd == DO: - await self.handle_do(writer, opt) - elif cmd == WILL: - await self.handle_will(writer, opt) - - async def process_packet(self, vm_client: VMClient, data): - data = data.decode('ascii') - - if data.startswith('!!') is False: # Make sure the data is a packet - self.logger.warning("Received a bad packet from " + vm_client.vm_name) - return - else: - data = data[2:] - - packet_code, data = data.split('#') # Packet code and data is split by # - - if len(data) > 0: - data = base64.b64decode(data).decode('ascii') # Packet data is base64 encoded - - await vm_client.process_packets(packet_code, data) - - async def handle_telnet(self, reader, writer): - opt_handler = functools.partial(self.option_handler, writer=writer) - telnet = async_telnet.AsyncTelnet(reader, opt_handler) - socket = writer.get_extra_info('socket') - peer = socket.getpeername() - self.logger.info("%s connected", peer) - data = await telnet.read_line() - vm_client = self.sock_to_client.get(socket) - if vm_client is None: - self.logger.error("%s didn't present UUID", peer) - writer.close() - return - try: - while data: - await self.process_packet(vm_client, data) - data = await telnet.read_line() - except Exception: - self.logger.exception("Raised exception while processing packet") - finally: - self.sock_to_client.pop(socket, None) - self.logger.info("%s disconnected", peer) - writer.close() - - def start(self): - self.loop = asyncio.get_event_loop() - ssl_context = None - # if CONF.cert: - # ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - # ssl_context.load_cert_chain(certfile=CONF.cert, keyfile=CONF.key) - coro = asyncio.start_server(self.handle_telnet, - '0.0.0.0', - 13370, - ssl=ssl_context, - loop=self.loop) - self.server = self.loop.run_until_complete(coro) - - # Serve requests until Ctrl+C is pressed - self.logger.info("Serving on %s", self.server.sockets[0].getsockname()) - self.loop.run_forever() - - def stop(self): - # Close the server - self.server.close() - self.loop.run_until_complete(self.server.wait_closed()) - self.loop.close() diff --git a/deli/menu/vspc/vm_client.py b/deli/menu/vspc/vm_client.py deleted file mode 100644 index 43f1d06..0000000 --- a/deli/menu/vspc/vm_client.py +++ /dev/null @@ -1,298 +0,0 @@ -import base64 -import enum -import json -import logging -import os -import os.path -import uuid - -import arrow -import yaml -from cryptography.fernet import Fernet - -from deli.counter.auth.token import Token -from deli.kubernetes.resources.const import ID_LABEL -from deli.kubernetes.resources.v1alpha1.image.model import Image -from deli.kubernetes.resources.v1alpha1.instance.model import Instance -from deli.kubernetes.resources.v1alpha1.keypair.keypair import Keypair -from deli.kubernetes.resources.v1alpha1.network.model import NetworkPort, Network -from deli.kubernetes.resources.v1alpha1.region.model import Region -from deli.kubernetes.resources.v1alpha1.service_account.model import ProjectServiceAccount -from deli.kubernetes.resources.v1alpha1.zone.model import Zone -from deli.menu.models.out_of_band import OutOfBandInstance -from deli.menu.vspc.async_telnet import CR - - -class PacketCode(enum.Enum): - # Incoming packets - REQUEST_METADATA = 'REQUEST_METADATA' - REQUEST_NETWORKDATA = 'REQUEST_NETWORKDATA' - REQUEST_USERDATA = 'REQUEST_USERDATA' - REQUEST_SECURITYDATA = 'REQUEST_SECURITYDATA' - # Outgoing packets - RESPONSE_METADATA = 'RESPONSE_METADATA' - RESPONSE_NETWORKDATA = 'RESPONSE_NETWORKDATA' - RESPONSE_USERDATA = 'RESPONSE_USERDATA' - RESPONSE_SECURITYDATA = 'RESPONSE_SECURITYDATA' - - -class VMClient(object): - - def __init__(self, vm_name, writer): - self.logger = logging.getLogger("%s.%s" % (self.__module__, self.__class__.__name__)) - self.vm_name = vm_name - self.writer = writer - - self.fernet = Fernet(os.environ['FERNET_KEY']) - self.out_of_band = os.environ.get('OUT_OF_BAND') - - def get_instance(self): - if self.out_of_band is not None: - - instance_file = None - if os.path.isfile(os.path.join(self.out_of_band, self.vm_name + ".yaml")): - instance_file = os.path.join(self.out_of_band, self.vm_name + ".yaml") - elif os.path.isfile(os.path.join(self.out_of_band, self.vm_name + ".yml")): - instance_file = os.path.join(self.out_of_band, self.vm_name + ".yml") - - if instance_file is None: - self.logger.error("Could not find metadata file for instance " + self.vm_name) - return None - - with open(instance_file) as f: - instance_yaml = yaml.load(f) - try: - instance_model = OutOfBandInstance(instance_yaml) - instance_model.validate() - except Exception: - self.logger.exception("Error loading metadata for instance " + self.vm_name) - return None - - # Override instance properties - @property - def image_id(self): - return self._image.id - - @property - def project_id(self): - return uuid.uuid4() - - @property - def image(self): - return self._image - - @image.setter - def image(self, value): - self._image = value - - @property - def region(self): - return self._region - - @region.setter - def region(self, value): - self._region = value - - @property - def zone(self): - return self._zone - - @zone.setter - def zone(self, value): - self._zone = value - - @property - def keypairs(self): - return self._keypairs - - @keypairs.setter - def keypairs(self, value): - self._keypairs = value - - @property - def network_port(self): - return self._network_port - - @network_port.setter - def network_port(self, value): - self._network_port = value - - @property - def service_account(self): - return self._service_account - - @service_account.setter - def service_account(self, value): - self._service_account = value - - Instance.image_id = image_id - Instance.project_id = project_id - Instance.image = image - Instance.region = region - Instance.zone = zone - Instance.keypairs = keypairs - Instance.network_port = network_port - Instance.service_account = service_account - - # Override network port properties - @property - def network(self): - return self._network - - @network.setter - def network(self, value): - self._network = value - - NetworkPort.network = network - - network = Network() - network.cidr = instance_model.network.cidr - network.gateway = instance_model.network.gateway - network.dns_servers = instance_model.network.dns_servers - - network_port = NetworkPort() - network_port.ip_address = instance_model.network.ip_address - network_port.network = network - - region = Region() - region.name = instance_model.region_name - - zone = Zone() - zone.name = instance_model.zone_name - - service_account = ProjectServiceAccount() - service_account.name = "None" - - instance = Instance() - instance.image = Image() - instance.region = region - instance.zone = zone - instance.service_account = service_account - - for k, v in instance_model.tags.items(): - instance.add_tag(k, v) - - keypairs = [] - for public_key in instance_model.keypairs: - keypair = Keypair() - keypair.public_key = public_key - keypairs.append(keypair) - - instance.keypairs = keypairs - instance.network_port = network_port - instance.user_data = instance_model.user_data - - return instance - else: - instances = Instance.list_all(label_selector=ID_LABEL + "=" + self.vm_name) - if len(instances) == 0: - self.logger.warning("Could not find any instances with the id of '%s'" % self.vm_name) - return None - if len(instances) > 1: - self.logger.warning("Found multiple instances with the id of '%s'" % self.vm_name) - return None - return instances[0] - - async def write_metadata(self): - - instance: Instance = self.get_instance() - if instance is None: - return - - keypairs = [] - for keypair in instance.keypairs: - keypairs.append(keypair.public_key) - - network_port = instance.network_port - - metadata = { - 'ami-id': instance.image_id, - 'instance-id': instance.id, - 'region': instance.region.name, - 'availability-zone': instance.zone.name, - 'tags': instance.tags, - 'public-keys': keypairs, - 'hostname': 'ip-' + str(network_port.ip_address).replace(".", "-"), - 'local-hostname': 'ip-' + str(network_port.ip_address).replace(".", "-"), - } - - await self.write(PacketCode.RESPONSE_METADATA, json.dumps(metadata)) - - async def write_networkdata(self): - - instance: Instance = self.get_instance() - if instance is None: - return - - network_port = instance.network_port - network = network_port.network - - networkdata = { - 'version': 1, - 'config': [ - { - "type": "physical", - "name": "eth0", - "subnets": [ - { - "type": "static", - "address": str(network_port.ip_address), - "netmask": network.cidr.with_netmask.split("/")[1], - "gateway": str(network.gateway), - "dns_search": ["sandwich.local"], - "dns_nameservers": [str(ns) for ns in network.dns_servers] - } - ] - } - ] - } - - await self.write(PacketCode.RESPONSE_NETWORKDATA, yaml.safe_dump(networkdata, default_flow_style=False)) - - async def write_userdata(self): - instance: Instance = self.get_instance() - if instance is None: - return - - await self.write(PacketCode.RESPONSE_USERDATA, instance.user_data) - - async def write_security_data(self): - - instance: Instance = self.get_instance() - if instance is None: - return - - service_account = instance.service_account - - token = Token() - token.expires_at = arrow.now().shift(minutes=+30) - token.driver_name = 'metadata' - token.project_id = instance.project_id - token.service_account_id = service_account.id - token.project_role_ids = service_account.role_ids - - await self.write(PacketCode.RESPONSE_SECURITYDATA, token.marshal(self.fernet).decode()) - - async def write(self, packet_code, data): - b64data = base64.b64encode(data.encode()).decode('ascii') - packet_data = "!!" + packet_code.value + "#" + b64data + '\n' - self.writer.write(packet_data.encode() + CR) - await self.writer.drain() - - async def process_packets(self, packet_code, data): - try: - packet_code = PacketCode(packet_code) - except ValueError: - self.logger.error("Received unknown packet code '%s' from vm '%s'" % (packet_code, self.vm_name)) - return - - self.logger.debug("Received packet code '%s' from vm '%s'" % (packet_code, self.vm_name)) - - if packet_code == PacketCode.REQUEST_METADATA: - await self.write_metadata() - elif packet_code == PacketCode.REQUEST_USERDATA: - await self.write_userdata() - elif packet_code == PacketCode.REQUEST_SECURITYDATA: - await self.write_security_data() - elif packet_code == PacketCode.REQUEST_NETWORKDATA: - await self.write_networkdata() diff --git a/docker-compose.yaml b/docker-compose.yaml index 3501c40..759ed08 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -35,6 +35,11 @@ services: - --master=http://kube-api:8080 networks: - sandwich + # Redis to do cache stuffs + redis: + image: redis:alpine + ports: + - "6379:6379" networks: sandwich: diff --git a/docker/counter/.env-sample b/docker/counter/.env-sample index 1826fce..936bc66 100644 --- a/docker/counter/.env-sample +++ b/docker/counter/.env-sample @@ -19,28 +19,31 @@ KUBEMASTER= # key in the list is the primary key. # To rotate keys simply generate a new key and put it in the front of the list # then after a while remove the old key from the list +# These keys are used to generate service account tokens AUTH_FERNET_KEYS= -#################### -# GITHUB AUTH # -#################### +## +# OAuth +# Only RSA RS256 signed tokens are supported +# Required scopes: openid profile email +## +# URL of the OpenID Provider +OPENID_ISSUER_URL= -# Only populate these values if using the Github Auth Driver +# Client crendentials to auth with the OpenID Prover +OPENID_CLIENT_ID= +OPENID_CLIENT_SECRET= -# Github Application Creds -GITHUB_CLIENT_ID= -GITHUB_CLIENT_SECRET= +# JWT Claim to use as the user's email +OPENID_EMAIL_CLAIM=email -# Github org users must be a part of and used for team searching -GITHUB_ORG= +# JWT claim to use as the user's groups +OPENID_GROUPS_CLAIM=groups -# A static mapping of sandwich cloud roles to github teams -# ${role_name}:${team_name} -# These static mappings will override GITHUB_TEAM_ROLES_PREFIX if a role is found -GITHUB_TEAM_ROLES="admin:sandwich-admin,role1:sandwich-role1" +#################### +# REDIS # +#################### -# Prefix to use when searching for sandwich cloud roles. -# If no static mapping for a role is given this prefix will be used. -# i.e For the role named "role1" with a prefix of "sandwich-" a team -# of "sandwich-role1" will be searched for in the github org -GITHUB_TEAM_ROLES_PREFIX="sandwich-" \ No newline at end of file +# Redis URL to connect to +# Used to cache various things +REDIS_URL= diff --git a/docker/counter/Dockerfile b/docker/counter/Dockerfile index 5576bc0..2259884 100644 --- a/docker/counter/Dockerfile +++ b/docker/counter/Dockerfile @@ -7,9 +7,7 @@ RUN apk --no-cache add --virtual build-deps \ build-base bash linux-headers openssl-dev pcre-dev libffi-dev && \ pip install uwsgi dumb-init -# COPY tar.gz from build container # Install it - COPY dist/. . RUN bash -c "pip install *" diff --git a/docker/counter/wsgi.ini b/docker/counter/wsgi.ini index d6372fc..a9690ba 100644 --- a/docker/counter/wsgi.ini +++ b/docker/counter/wsgi.ini @@ -1,4 +1,5 @@ [uwsgi] +plugins = python3 protocol = http socket = 0.0.0.0:8080 module = deli.counter.http.wsgi:application diff --git a/docker/manager/Dockerfile b/docker/manager/Dockerfile index db700b1..b2ad75a 100644 --- a/docker/manager/Dockerfile +++ b/docker/manager/Dockerfile @@ -7,9 +7,7 @@ RUN apk --no-cache add --virtual build-deps \ build-base bash linux-headers openssl-dev pcre-dev libffi-dev && \ pip install dumb-init -# COPY tar.gz from build container # Install it - COPY dist/. . RUN bash -c "pip install *" diff --git a/docker/menu/.env-sample b/docker/menu/.env-sample index 12d93f4..f045cf6 100644 --- a/docker/menu/.env-sample +++ b/docker/menu/.env-sample @@ -18,13 +18,10 @@ KUBEMASTER= # This must match a fernet key that the api server (counter) uses FERNET_KEY= - #################### -# OUT OF BAND # +# REDIS # #################### -# Path to .yaml (.yml) files to use for metadata -# This is useful if you just want metadata without using -# the rest of Sandwich Cloud. -# Setting this will ignore any kubernetes settings -OUT_OF_BAND= \ No newline at end of file +# Redis URL to connect to +# Used to cache various things +REDIS_URL= \ No newline at end of file diff --git a/docker/menu/Dockerfile b/docker/menu/Dockerfile index 5c60a50..ac52b2f 100644 --- a/docker/menu/Dockerfile +++ b/docker/menu/Dockerfile @@ -7,9 +7,7 @@ RUN apk --no-cache add --virtual build-deps \ build-base bash linux-headers openssl-dev pcre-dev libffi-dev && \ pip install dumb-init -# COPY tar.gz from build container # Install it - COPY dist/. . RUN bash -c "pip install *" diff --git a/requirements.txt b/requirements.txt index c418faa..d9d2cc3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ clify==0.0.3 # MIT -pbr==3.1.1 # Apache 2.0 +pbr==4.0.3 # Apache 2.0 python-dotenv==0.7.1 # BSD 3-clause go-defer==1.0.1 # BSD 3-clause kubernetes==4.0.0 # Apache 2.0 @@ -10,11 +10,13 @@ simple-settings==0.12.0 # MIT arrow==0.10.0 # Apache 2.0 pyvmomi==6.5.0.2017.5-1 # Apache 2.0 cryptography==2.1.4 # Apache 2.0 or BSD -sqlalchemy==1.1.14 # MIT -sqlalchemy-utils==0.32.16 # BSD-3-Clause -alembic==0.9.5 # MIT passlib==1.7.1 # BSD-3-Clause bcrypt==3.1.4 # Apache 2.0 pygithub==1.35 # LGPL +redis==2.10.6 # MIT +requests==2.19.1 # Apache 2.0 +python-jose==3.0.0 # MIT +apispec==0.38.0 # MIT pyk8s-controller==0.0.7 # MIT -ingredients.http==0.0.14 # MIT \ No newline at end of file +ingredients.http==0.0.15 # MIT +vmw-cloudinit-metadata==0.0.8 # MIT \ No newline at end of file diff --git a/setup.py b/setup.py index 8935042..aa8980d 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup setup( - setup_requires=['pbr>=3.0.1', 'setuptools>=17.1'], + setup_requires=['pbr>=4.0.3', 'setuptools>=17.1'], pbr=True, )