From 8a381f120b1703d909e4184091338d09a180fdd6 Mon Sep 17 00:00:00 2001 From: Joe Stubbs Date: Mon, 7 Oct 2019 10:45:45 -0500 Subject: [PATCH 01/12] fix reference to Abaco.SYNC_HINT causing exception that was preventing sync actors worker pool from being reduced to 1. --- actors/controllers.py | 2 +- actors/metrics_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/actors/controllers.py b/actors/controllers.py index 34790c2e..904021e8 100644 --- a/actors/controllers.py +++ b/actors/controllers.py @@ -138,7 +138,7 @@ def check_metrics(self, actor_ids, inbox_lengths, cmd_length): except: hints = [] for hint in hints: - if hint == actor.SYNC_HINT: + if hint == Actor.SYNC_HINT: is_sync_actor = True break metrics_utils.scale_down(actor_id, is_sync_actor) diff --git a/actors/metrics_utils.py b/actors/metrics_utils.py index d22a28e2..e45accd6 100644 --- a/actors/metrics_utils.py +++ b/actors/metrics_utils.py @@ -179,7 +179,7 @@ def scale_down(actor_id, is_sync_actor=False): if len(workers) == 1 and is_sync_actor: logger.debug("only one worker, on sync actor. checking worker idle time..") try: - sync_max_idle_time = int(Config.get('worker', 'sync_max_idle_time')) + sync_max_idle_time = int(Config.get('workers', 'sync_max_idle_time')) except Exception as e: logger.error(f"Got exception trying to read sync_max_idle_time from config; e:{e}") sync_max_idle_time = DEFAULT_SYNC_MAX_IDLE_TIME From 2b7e9f3a5be0f3b74f6eb00367a26cef14abba6a Mon Sep 17 00:00:00 2001 From: Joe Stubbs Date: Tue, 8 Oct 2019 17:23:22 -0500 Subject: [PATCH 02/12] add PUT /aliases/{alias} endpoint; update alias endpoints to check for access to underlying actor_id. --- actors/auth.py | 26 +++++++++++++-- actors/codes.py | 2 ++ actors/controllers.py | 77 ++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 98 insertions(+), 7 deletions(-) diff --git a/actors/auth.py b/actors/auth.py index 2fac4f80..40c9d793 100644 --- a/actors/auth.py +++ b/actors/auth.py @@ -84,6 +84,19 @@ def check_nonce(): any privileged action cannot be taken via a nonce. """ logger.debug("top of check_nonce") + # first check whether the request is even valid - + if hasattr(request, 'url_rule'): + logger.debug("request.url_rule: {}".format(request.url_rule)) + if hasattr(request.url_rule, 'rule'): + logger.debug("url_rule.rule: {}".format(request.url_rule.rule)) + else: + logger.info("url_rule has no rule.") + raise ResourceError( + "Invalid request: the API endpoint does not exist or the provided HTTP method is not allowed.", 405) + else: + logger.info("Request has no url_rule") + raise ResourceError( + "Invalid request: the API endpoint does not exist or the provided HTTP method is not allowed.", 405) try: nonce_id = request.args['x-nonce'] except KeyError: @@ -312,12 +325,16 @@ def check_privileged(): logger.debug("not trying to use privileged options.") return True -def check_permissions(user, identifier, level): +def check_permissions(user, identifier, level, roles=None): """Check the permissions store for user and level. Here, `identifier` is a unique id in the permissions_store; e.g., actor db_id or alias_id. """ logger.debug("Checking user: {} permissions for identifier: {}".format(user, identifier)) - # get all permissions for this actor + # first, if roles were passed, check for admin role - + if roles: + if codes.ADMIN_ROLE in roles: + return True + # get all permissions for this actor - permissions = get_permissions(identifier) for p_user, p_name in permissions.items(): # if the actor has been shared with the WORLD_USER anyone can use it @@ -354,7 +371,10 @@ def get_db_id(): logger.error("Unrecognized request -- could not find the actor id. path_split: {}".format(path_split)) raise PermissionsException("Not authorized.") logger.debug("path_split: {}".format(path_split)) - actor_identifier = path_split[idx] + try: + actor_identifier = path_split[idx] + except IndexError: + raise ResourceError("Unable to parse actor identifier: is it missing from the URL?", 404) logger.debug("actor_identifier: {}; tenant: {}".format(actor_identifier, g.tenant)) try: actor_id = Actor.get_actor_id(g.tenant, actor_identifier) diff --git a/actors/codes.py b/actors/codes.py index d6dc3179..5e7870ac 100644 --- a/actors/codes.py +++ b/actors/codes.py @@ -67,6 +67,8 @@ def __repr__(self): PERMISSION_LEVELS = (NONE.name, READ.name, EXECUTE.name, UPDATE.name) +ALIAS_NONCE_PERMISSION_LEVELS = (NONE.name, READ.name, EXECUTE.name) + # role set by agaveflask in case the access_control_type is none ALL_ROLE = 'ALL' diff --git a/actors/controllers.py b/actors/controllers.py index 904021e8..8bf5e973 100644 --- a/actors/controllers.py +++ b/actors/controllers.py @@ -15,7 +15,7 @@ from auth import check_permissions, get_tas_data, tenant_can_use_tas, get_uid_gid_homedir, get_token_default from channels import ActorMsgChannel, CommandChannel, ExecutionResultsChannel, WorkerChannel -from codes import SUBMITTED, COMPLETE, SHUTTING_DOWN, PERMISSION_LEVELS, READ, UPDATE, EXECUTE, PERMISSION_LEVELS, PermissionLevel +from codes import SUBMITTED, COMPLETE, SHUTTING_DOWN, PERMISSION_LEVELS, ALIAS_NONCE_PERMISSION_LEVELS, READ, UPDATE, EXECUTE, PERMISSION_LEVELS, PermissionLevel from config import Config from errors import DAOError, ResourceError, PermissionsException, WorkerException from models import dict_to_camel, display_time, is_hashid, Actor, Alias, Execution, ExecutionsSummary, Nonce, Worker, get_permissions, \ @@ -333,6 +333,10 @@ def post(self): logger.debug("did not find actor: {}.".format(dbid)) raise ResourceError( "No actor found with id: {}.".format(actor_id), 404) + # update 10/2019: check that use has UPDATE permission on the actor - + if not check_permissions(user=g.user, identifier=dbid, level=codes.UPDATE): + raise PermissionsException(f"Not authorized -- you do not have access to {actor_id}.") + # supply "provided" fields: args['tenant'] = g.tenant args['db_id'] = dbid @@ -358,9 +362,66 @@ def get(self, alias): logger.debug("did not find alias with id: {}".format(alias)) raise ResourceError( "No alias found: {}.".format(alias), 404) - logger.debug("found actor {}".format(alias)) + logger.debug("found alias {}".format(alias)) return ok(result=alias.display(), msg="Alias retrieved successfully.") + def validate_put(self): + logger.debug("top of validate_put") + try: + data = request.get_json() + except: + data = None + if data and 'alias' in data or 'alias' in request.form: + logger.debug("found alias in the PUT.") + raise DAOError("Invalid alias update description. The alias itself cannot be updated in a PUT request.") + parser = Alias.request_parser() + logger.debug("got the alias parser") + # remove since alias is only required for POST, not PUT + parser.remove_argument('alias') + try: + args = parser.parse_args() + except BadRequest as e: + msg = 'Unable to process the JSON description.' + if hasattr(e, 'data'): + msg = e.data.get('message') + raise DAOError("Invalid alias description. Missing required field: {}".format(msg)) + return args + + def put(self, alias): + logger.debug("top of PUT /actors/aliases/{}".format(alias)) + alias_id = Alias.generate_alias_id(g.tenant, alias) + try: + alias_obj = Alias.from_db(alias_store[alias_id]) + except KeyError: + logger.debug("did not find alias with id: {}".format(alias)) + raise ResourceError("No alias found: {}.".format(alias), 404) + logger.debug("found alias {}".format(alias_obj)) + args = self.validate_put() + actor_id = args.get('actor_id') + if Config.get('web', 'case') == 'camel': + actor_id = args.get('actorId') + dbid = Actor.get_dbid(g.tenant, actor_id) + # update 10/2019: check that use has UPDATE permission on the actor - + if not check_permissions(user=g.user, identifier=dbid, level=codes.UPDATE, roles=g.roles): + raise PermissionsException(f"Not authorized -- you do not have UPDATE " + f"access to the actor you want to associate with this alias.") + logger.debug(f"dbid: {dbid}") + # supply "provided" fields: + args['tenant'] = alias_obj.tenant + args['db_id'] = dbid + args['owner'] = alias_obj.owner + args['alias'] = alias_obj.alias + args['alias_id'] = alias_obj.alias_id + args['api_server'] = alias_obj.api_server + logger.debug("Instantiating alias object. args: {}".format(args)) + new_alias_obj = Alias(**args) + logger.debug("Alias object instantiated; updating alias in alias_store. " + "alias: {}".format(new_alias_obj)) + alias_store[alias_id] = new_alias_obj + logger.info("alias updated for actor: {}.".format(dbid)) + set_permission(g.user, new_alias_obj.alias_id, UPDATE) + return ok(result=new_alias_obj.display(), msg="Actor alias updated successfully.") + def delete(self, alias): logger.debug("top of DELETE /actors/aliases/{}".format(alias)) alias_id = Alias.generate_alias_id(g.tenant, alias) @@ -370,6 +431,13 @@ def delete(self, alias): logger.debug("did not find alias with id: {}".format(alias)) raise ResourceError( "No alias found: {}.".format(alias), 404) + + # update 10/2019: check that use has UPDATE permission on the actor - + # TODO - check: do we want to require UPDATE on the actor to delete the alias? Seems like UPDATE + # on the alias iteself should be sufficient... + # if not check_permissions(user=g.user, identifier=alias.db_id, level=codes.UPDATE): + # raise PermissionsException(f"Not authorized -- you do not have UPDATE " + # f"access to the actor associated with this alias.") try: del alias_store[alias_id] # also remove all permissions - there should be at least one permissions associated @@ -435,10 +503,11 @@ def validate_post(self): msg = e.data.get('message') raise DAOError("Invalid nonce description: {}".format(msg)) # additional checks + if 'level' in args: - if not args['level'] in PERMISSION_LEVELS: + if not args['level'] in ALIAS_NONCE_PERMISSION_LEVELS: raise DAOError("Invalid nonce description. " - "The level attribute must be one of: {}".format(PERMISSION_LEVELS)) + "The level attribute must be one of: {}".format(ALIAS_NONCE_PERMISSION_LEVELS)) if Config.get('web', 'case') == 'snake': if 'max_uses' in args: self.validate_max_uses(args['max_uses']) From d49b544d0028a6a4c1166017256df489c5952672 Mon Sep 17 00:00:00 2001 From: Joe Stubbs Date: Tue, 8 Oct 2019 17:24:02 -0500 Subject: [PATCH 03/12] new tests for alias changes and initial 1.5.0 CHANGELOG placeholder. --- CHANGELOG.md | 16 +++++ tests/test_abaco_core.py | 124 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 133 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 040dd01d..d2da97b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,22 @@ # Change Log All notable changes to this project will be documented in this file. +## 1.5.0 - 2019-11-10 (target) +### Added +- Added an endpoint `PUT /actors/aliases/{alias}` for updating the +definition of an alias. Requires `UPDATE` permission for the alias as well as for the actor to which the alias should be defined. + +### Changed +- Fixed issue where autoscaler did not properly scale down worker pools for actors with the `sync` hint. They are now scaled down to 1. +- The permission check on all on all `/aliases/{alias}` endpoints has been updated to require UPDATE on the associated `actor_id`. +- Fixed error messaging when using a nonce and the API endpoint+HTTP verb combination do not exist. +- The admin role is now recognized when checking access to certain objects in some edge cases, including when a nonce is used. + + +### Removed +- It is no longer possible to create an alias nonce for permission levels UPDATE. + + ## 1.4.0 - 2019-09-16 ### Added - Added `hints` attribute to the actor data model, a list of strings representing metadata about an actor. "Official" diff --git a/tests/test_abaco_core.py b/tests/test_abaco_core.py index e542ae43..2fc2fd3b 100644 --- a/tests/test_abaco_core.py +++ b/tests/test_abaco_core.py @@ -10,8 +10,44 @@ # Then, also from the root directory, execute: # docker run -e base_url=http://172.17.0.1:8000 -e case=camel -v $(pwd)/local-dev.conf:/etc/service.conf -it --rm abaco/testsuite$TAG # Change the -e case=camel to -e case=snake depending on the functionality you want to test. - # +# +# Test Suite Outline +# I_ non-http tests +# II) actor registrations +# III) invlaid http endpoint tests +# IV) check actors are ready +# V) execution tests +# VI) update actor tests +# VII) custom queue tests (WIP) +# VIII) alias tests +# IX) nonce tests +# X) workers tests +# XI) roles and authorization tests +# XII) tenancy tests +# XIII) events tests + +# Design Notes +# 1. The *headers() functions in the util.py module provides the JWTs used for the tests. The plain headers() returns a +# JWT for a user with the Abaco admin role (username='testuser'), but there is also limited_headers() and +# priv_headers() +# for getting JWTs for other users. +# 2. The get_actor_id() function by default returns the abaco_test_suite actor_id but takes an optional name parameter +# for getting the id of a different actor. +# +# 3. Actors registered and owned by testuser: +# abaco_test_suite +# abaco_test_suite_alias -- add "jane" and "doe" aliases to this actor; same user. +# abaco_test_suite_statelesss +# abaco_test_suite_hints +# abaco_test_suite_default_env +# abaco_test_suite_func +# abaco_test_suite_sleep_loop +# +# 4. Actors registered and owned by testotheruser user (limited): +# abaco_test_suite_limited_user +# + # # --- Original notes for running natively ------ # Start the local development abaco stack (docker-compose-local.yml) and run these tests with py.test from the cwd. # $ py.test test_abaco_core.py @@ -185,6 +221,19 @@ def test_register_stateless_actor(headers): assert result['name'] == 'abaco_test_suite_statelesss' assert result['id'] is not None +@pytest.mark.regapi +def test_register_hints_actor(headers): + url = '{}/{}'.format(base_url, '/actors') + data = {'image': 'abacosamples/wc', 'name': 'abaco_test_suite_hints', 'hints': ['sync', 'test', 'hint_1']} + rsp = requests.post(url, data=data, headers=headers) + result = basic_response_checks(rsp) + assert 'description' in result + assert 'owner' in result + assert result['owner'] == 'testuser' + assert result['image'] == 'abacosamples/wc' + assert result['name'] == 'abaco_test_suite_hints' + assert result['id'] is not None + @pytest.mark.regapi def test_register_actor_default_env(headers): @@ -222,6 +271,20 @@ def test_register_actor_func(headers): assert result['name'] == 'abaco_test_suite_func' assert result['id'] is not None +@pytest.mark.regapi +def test_register_actor_limited_user(headers): + url = '{}/{}'.format(base_url, '/actors') + data = {'image': 'abacosamples/test', 'name': 'abaco_test_suite_limited_user'} + rsp = requests.post(url, data=data, headers=limited_headers()) + result = basic_response_checks(rsp) + assert 'description' in result + assert 'owner' in result + assert result['owner'] == 'testotheruser' + assert result['image'] == 'abacosamples/test' + assert result['name'] == 'abaco_test_suite_limited_user' + assert result['id'] is not None + + @pytest.mark.regapi def test_register_actor_sleep_loop(headers): url = '{}/{}'.format(base_url, '/actors') @@ -461,6 +524,10 @@ def test_alias_actor_is_ready(headers): actor_id = get_actor_id(headers, name='abaco_test_suite_alias') check_actor_is_ready(headers, actor_id) +@pytest.mark.regapi +def test_hints_actor_is_ready(headers): + actor_id = get_actor_id(headers, name='abaco_test_suite_hints') + check_actor_is_ready(headers, actor_id) @pytest.mark.regapi def test_stateless_actor_is_ready(headers): @@ -472,7 +539,6 @@ def test_default_env_actor_is_ready(headers): actor_id = get_actor_id(headers, name='abaco_test_suite_default_env') check_actor_is_ready(headers, actor_id) - @pytest.mark.regapi def test_func_actor_is_ready(headers): actor_id = get_actor_id(headers, name='abaco_test_suite_func') @@ -733,6 +799,16 @@ def test_execute_actor_json(headers): data = {'key1': 'value1', 'key2': 'value2'} execute_actor(headers, actor_id=actor_id, json_data=data) +def test_execute_basic_actor_synchronous(headers): + actor_id = get_actor_id(headers) + data = {'message': 'testing execution'} + execute_actor(headers, actor_id, data=data, synchronous=True) + + +# ################## +# updates to actors +# ################## + def test_update_actor(headers): actor_id = get_actor_id(headers) url = '{}/actors/{}'.format(base_url, actor_id) @@ -774,11 +850,6 @@ def test_update_actor_other_user(headers): else: assert not result['lastUpdateTime'] == orig_actor['lastUpdateTime'] -def test_execute_basic_actor_synchronous(headers): - actor_id = get_actor_id(headers) - data = {'message': 'testing execution'} - execute_actor(headers, actor_id, data=data, synchronous=True) - ############### # actor queue @@ -906,6 +977,7 @@ def test_actor_with_default_queue(headers): def test_2_actors_with_different_queues(headers): pass + # ########## # alias API # ########## @@ -1003,6 +1075,28 @@ def test_list_alias_permission(headers): assert result[owner] == 'UPDATE' +@pytest.mark.aliastest +def test_update_alias(headers): + # alias UPDATE permissions alone are not sufficient to change the definition of the alias to an actor - + # the user must have UPDATE access to the underlying actor_id as well. + url = '{}/actors/aliases/{}'.format(base_url, ALIAS_1) + # change the alias to point to the "abaco_test_suite" actor: + actor_id = get_actor_id(headers) + field = 'actor_id' + if case == 'camel': + field = 'actorId' + data = {field: actor_id} + rsp = requests.put(url, data=data, headers=headers) + result = basic_response_checks(rsp) + assert field in result + assert result[field] == actor_id + # now, change the alias back to point to the original "abaco_test_suite_alias" actor: + actor_id = get_actor_id(headers, name='abaco_test_suite_alias') + data = {field: actor_id} + rsp = requests.put(url, data=data, headers=headers) + result = basic_response_checks(rsp) + + @pytest.mark.aliastest def test_other_user_cant_list_alias(headers): url = '{}/actors/aliases/{}'.format(base_url, ALIAS_1) @@ -1040,6 +1134,22 @@ def test_other_user_still_cant_list_actor(headers): data = response_format(rsp) assert 'you do not have access to this actor' in data['message'] +@pytest.mark.aliastest +def test_other_user_still_cant_update_alias_wo_actor(headers): + # alias UPDATE permissions alone are not sufficient to change the definition of the alias to an actor - + # the user must have UPDATE access to the underlying actor_id as well. + url = '{}/actors/aliases/{}'.format(base_url, ALIAS_1) + # priv user does not have access to the abaco_test_suite_alias actor + field = 'actor_id' + if case == 'camel': + field = 'actorId' + data = {field: get_actor_id(headers, name="abaco_test_suite_alias")} + + rsp = requests.put(url, data=data, headers=priv_headers()) + assert rsp.status_code == 400 + data = response_format(rsp) + assert 'ou do not have UPDATE access to the actor you want to associate with this alias' in data['message'] + @pytest.mark.aliastest def test_other_user_still_cant_create_alias_nonce(headers): # alias permissions do not confer access to the actor itself, and alias nonces require BOTH From 5fe56cc652bb5275780dc0c24a15443b22643296 Mon Sep 17 00:00:00 2001 From: Joe Stubbs Date: Thu, 24 Oct 2019 13:26:26 -0500 Subject: [PATCH 04/12] comment --- actors/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actors/auth.py b/actors/auth.py index 40c9d793..fdca2843 100644 --- a/actors/auth.py +++ b/actors/auth.py @@ -286,7 +286,7 @@ def check_privileged(): data = request.get_json() if not data: data = request.form - # various APIs (e.g., the state api) allow an arbitary JSON serializable objects which won't have a get method: + # various APIs (e.g., the state api) allow an arbitrary JSON serializable objects which won't have a get method: if not hasattr(data, 'get'): return True if not codes.PRIVILEGED_ROLE in g.roles: From 082c2f0dd1b45cee3d724304c0b4d6fa28d2a979 Mon Sep 17 00:00:00 2001 From: Joe Stubbs Date: Thu, 24 Oct 2019 13:27:02 -0500 Subject: [PATCH 05/12] add a message to the logs indicating that the log limit was exceeded. --- actors/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actors/models.py b/actors/models.py index ad9da2da..3d3ed0fd 100644 --- a/actors/models.py +++ b/actors/models.py @@ -1019,7 +1019,7 @@ def set_logs(cls, exc_id, logs): max_log_length = DEFAULT_MAX_LOG_LENGTH if len(logs) > DEFAULT_MAX_LOG_LENGTH: logger.info("truncating log for execution: {}".format(exc_id)) - logs = logs[:max_log_length] + logs = logs[:max_log_length] + " LOG LIMIT EXCEEDED; this execution log was TRUNCATED!" start_timer = timeit.default_timer() if log_ex > 0: logger.info("Storing log with expiry. exc_id: {}".format(exc_id)) From ece274ea6657e519a7d6e9fe55d7e4d686ed2f7a Mon Sep 17 00:00:00 2001 From: Joe Stubbs Date: Thu, 24 Oct 2019 13:28:05 -0500 Subject: [PATCH 06/12] logging --- actors/controllers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/actors/controllers.py b/actors/controllers.py index 8bf5e973..849338d9 100644 --- a/actors/controllers.py +++ b/actors/controllers.py @@ -683,9 +683,11 @@ def get(self): return ok(result=actors, msg="Actors retrieved successfully.") def validate_post(self): + logger.debug("top of validate post in /actors") parser = Actor.request_parser() try: args = parser.parse_args() + logger.debug(f"initial actor args from parser: {args}") if args['queue']: queues_list = Config.get('spawner', 'host_queues').replace(' ', '') valid_queues = queues_list.split(',') From aa34b6da30502c57f09219bb015d6c13146f63a7 Mon Sep 17 00:00:00 2001 From: Joe Stubbs Date: Thu, 24 Oct 2019 13:28:58 -0500 Subject: [PATCH 07/12] update the tests to work with the autoscaler turned on. --- local-dev.conf | 4 ++-- tests/test_abaco_core.py | 15 +++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/local-dev.conf b/local-dev.conf index f71911d2..9d5149b9 100644 --- a/local-dev.conf +++ b/local-dev.conf @@ -93,7 +93,7 @@ dd: unix://var/run/docker.sock init_count: 1 # set whether autoscaling is enabled -autoscaling = false +autoscaling = true # max length of time, in seconds, an actor container is allowed to execute before being killed. # set to -1 for indefinite execution time. @@ -134,7 +134,7 @@ auto_remove: true # Whether the workers should have OAuth clients generated for them: generate_clients: False # specifiy client generation is available for a specific tenant - -DEV-STAGING_generate_clients: True +# DEV-STAGING_generate_clients: True # a list of mounts to make for every actor container, separated by comma (,) characters. # Mounts should be of the form :: diff --git a/tests/test_abaco_core.py b/tests/test_abaco_core.py index 2fc2fd3b..c1aae892 100644 --- a/tests/test_abaco_core.py +++ b/tests/test_abaco_core.py @@ -36,7 +36,7 @@ # for getting the id of a different actor. # # 3. Actors registered and owned by testuser: -# abaco_test_suite +# abaco_test_suite -- a stateful actor # abaco_test_suite_alias -- add "jane" and "doe" aliases to this actor; same user. # abaco_test_suite_statelesss # abaco_test_suite_hints @@ -1678,7 +1678,7 @@ def switch_tenant_in_header(headers): jwt = headers.get('X-Jwt-Assertion-DEV-DEVELOP') return {'X-Jwt-Assertion-DEV-STAGING': jwt} - +@pytest.mark.tenant def test_tenant_list_actors(headers): # passing another tenant should result in 0 actors. headers = switch_tenant_in_header(headers) @@ -1687,17 +1687,19 @@ def test_tenant_list_actors(headers): result = basic_response_checks(rsp) assert len(result) == 0 +@pytest.mark.tenant def test_tenant_register_actor(headers): headers = switch_tenant_in_header(headers) url = '{}/{}'.format(base_url, '/actors') - data = {'image': 'jstubbs/abaco_test', 'name': 'abaco_test_suite_other_tenant'} - rsp = requests.post(url, data=data, headers=headers) + data = {'image': 'jstubbs/abaco_test', 'name': 'abaco_test_suite_other_tenant', 'stateless': False} + rsp = requests.post(url, json=data, headers=headers) result = basic_response_checks(rsp) assert 'description' in result assert result['image'] == 'jstubbs/abaco_test' assert result['name'] == 'abaco_test_suite_other_tenant' assert result['id'] is not None +@pytest.mark.tenant def test_tenant_actor_is_ready(headers): headers = switch_tenant_in_header(headers) count = 0 @@ -1712,6 +1714,7 @@ def test_tenant_actor_is_ready(headers): count += 1 assert False +@pytest.mark.tenant def test_tenant_list_registered_actors(headers): # passing another tenant should result in 1 actor. headers = switch_tenant_in_header(headers) @@ -1720,6 +1723,7 @@ def test_tenant_list_registered_actors(headers): result = basic_response_checks(rsp) assert len(result) == 1 +@pytest.mark.tenant def test_tenant_list_actor(headers): headers = switch_tenant_in_header(headers) actor_id = get_actor_id(headers, name='abaco_test_suite_other_tenant') @@ -1731,6 +1735,7 @@ def test_tenant_list_actor(headers): assert result['name'] == 'abaco_test_suite_other_tenant' assert result['id'] is not None +@pytest.mark.tenant def test_tenant_list_executions(headers): headers = switch_tenant_in_header(headers) actor_id = get_actor_id(headers, name='abaco_test_suite_other_tenant') @@ -1739,6 +1744,7 @@ def test_tenant_list_executions(headers): result = basic_response_checks(rsp) assert len(result.get('executions')) == 0 +@pytest.mark.tenant def test_tenant_list_messages(headers): headers = switch_tenant_in_header(headers) actor_id = get_actor_id(headers, name='abaco_test_suite_other_tenant') @@ -1747,6 +1753,7 @@ def test_tenant_list_messages(headers): result = basic_response_checks(rsp) assert result.get('messages') == 0 +@pytest.mark.tenant def test_tenant_list_workers(headers): headers = switch_tenant_in_header(headers) actor_id = get_actor_id(headers, name='abaco_test_suite_other_tenant') From ad041034fe2ad4d697ccbbef6b0a19e10722bc0f Mon Sep 17 00:00:00 2001 From: Joe Stubbs Date: Thu, 24 Oct 2019 15:41:21 -0500 Subject: [PATCH 08/12] new sample to exercise mem limits --- samples/mem_limit/Dockerfile | 6 ++++++ samples/mem_limit/README.rst | 0 samples/mem_limit/actor.py | 39 ++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 samples/mem_limit/Dockerfile create mode 100644 samples/mem_limit/README.rst create mode 100644 samples/mem_limit/actor.py diff --git a/samples/mem_limit/Dockerfile b/samples/mem_limit/Dockerfile new file mode 100644 index 00000000..d331e738 --- /dev/null +++ b/samples/mem_limit/Dockerfile @@ -0,0 +1,6 @@ +# Image: abacosamples/mem_limit +from abacosamples/py3_base + +ADD actor.py /actor.py + +CMD ["python", "/actor.py"] diff --git a/samples/mem_limit/README.rst b/samples/mem_limit/README.rst new file mode 100644 index 00000000..e69de29b diff --git a/samples/mem_limit/actor.py b/samples/mem_limit/actor.py new file mode 100644 index 00000000..c51aa8c0 --- /dev/null +++ b/samples/mem_limit/actor.py @@ -0,0 +1,39 @@ +import sys + +from agavepy.actors import get_context + +def main(): + + # default size, in GB, to test + print("start of mem_limit") + SIZE_DEFAULT = 2 + context = get_context() + try: + msg_dict = context['message_dict'] + except KeyError as e: + print("KeyError getting message_dict".format(e)) + msg_dict = None + + if msg_dict: + try: + size = int(msg_dict.get('size', SIZE_DEFAULT)) + except Exception as e: + print("Got exception {} trying to read size from msg_dict".format(e)) + if size > 10: + print("Invalid size parameter - received a value larger than the max value of 10.") + raise Exception() + print("mem_limit using size: {}".format(size)) + # try to create a python object of a certain size - + ONE_G = 1000000000 + factor = size * ONE_G + try: + s = factor*'a' + except MemoryError as e: + print("got a memory error; e: {}".format(e)) + except Exception as e: + print("Got a non-MemoryError exception; e: {}".format(e)) + print("actual size of var: {}".format(sys.getsizeof(s))) + print("mem_limit completed.") + +if __name__ == '__main__': + main() From 8d1eb0c4e4ee8efea9349e6f939ef894962f3200 Mon Sep 17 00:00:00 2001 From: Joe Stubbs Date: Fri, 25 Oct 2019 10:52:38 -0500 Subject: [PATCH 09/12] add fix of nonce-alias redemption bug. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2da97b5..d7d20493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. definition of an alias. Requires `UPDATE` permission for the alias as well as for the actor to which the alias should be defined. ### Changed +- Fixed a bug where nonces defined for aliases would not be honored when using the alias in the URL (they were only honored when using the actor id assigned to the alias). - Fixed issue where autoscaler did not properly scale down worker pools for actors with the `sync` hint. They are now scaled down to 1. - The permission check on all on all `/aliases/{alias}` endpoints has been updated to require UPDATE on the associated `actor_id`. - Fixed error messaging when using a nonce and the API endpoint+HTTP verb combination do not exist. From e1949fe0aafb3b8b8b5f570fef316d237fed6203 Mon Sep 17 00:00:00 2001 From: Joe Stubbs Date: Mon, 28 Oct 2019 13:06:58 -0500 Subject: [PATCH 10/12] add PUT for alias --- CHANGELOG.md | 1 - docs/specs/openapi_v3.yml | 69 +++++++++++++++++++++++++++++++++++---- 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7d20493..99548137 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,6 @@ definition of an alias. Requires `UPDATE` permission for the alias as well as fo - Fixed error messaging when using a nonce and the API endpoint+HTTP verb combination do not exist. - The admin role is now recognized when checking access to certain objects in some edge cases, including when a nonce is used. - ### Removed - It is no longer possible to create an alias nonce for permission levels UPDATE. diff --git a/docs/specs/openapi_v3.yml b/docs/specs/openapi_v3.yml index 801f4de7..86513bb1 100644 --- a/docs/specs/openapi_v3.yml +++ b/docs/specs/openapi_v3.yml @@ -109,6 +109,36 @@ paths: properties: result: $ref: '#/components/schemas/Actor' + put: + tags: + - Actors + summary: Update an actor + description: Update an actor's definition. + operationId: updateActor + parameters: + - name: actor_id + in: path + description: Unique ID of the actor + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/NewActor' + responses: + 200: + description: OK + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/BasicResponse' + properties: + result: + $ref: '#/components/schemas/Actor' delete: tags: - Actors @@ -745,6 +775,37 @@ paths: properties: result: $ref: '#/components/schemas/Alias' + put: + tags: + - Actors + - Aliases + summary: Update an actor alias + description: Update an alias definition. + operationId: updateActorAlias + parameters: + - name: alias + in: path + description: Unique alias of the actor + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/NewAlias' + responses: + 200: + description: OK + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/BasicResponse' + properties: + result: + $ref: '#/components/schemas/Alias' delete: tags: - Actors @@ -1181,10 +1242,4 @@ components: type: array items: type: string - description: The roles associated with the nonce. - - - - - - \ No newline at end of file + description: The roles associated with the nonce. \ No newline at end of file From 1f12baada3283e6ccb884519954bff33ac3d0d81 Mon Sep 17 00:00:00 2001 From: Joe Stubbs Date: Mon, 28 Oct 2019 15:18:44 -0500 Subject: [PATCH 11/12] fix mounts definition in spec file. --- docs/specs/openapi_v3.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/specs/openapi_v3.yml b/docs/specs/openapi_v3.yml index 86513bb1..2fbe30a0 100644 --- a/docs/specs/openapi_v3.yml +++ b/docs/specs/openapi_v3.yml @@ -104,8 +104,8 @@ paths: content: application/json: schema: - allOf: - - $ref: '#/components/schemas/BasicResponse' +# allOf: +# - $ref: '#/components/schemas/BasicResponse' properties: result: $ref: '#/components/schemas/Actor' @@ -134,8 +134,8 @@ paths: content: application/json: schema: - allOf: - - $ref: '#/components/schemas/BasicResponse' +# allOf: +# - $ref: '#/components/schemas/BasicResponse' properties: result: $ref: '#/components/schemas/Actor' @@ -959,8 +959,8 @@ components: Actor: type: object - allOf: - - $ref: '#/components/schemas/ArrayOfActorMounts' +# allOf: +# - $ref: '#/components/schemas/ArrayOfActorMounts' properties: id: type: string @@ -983,6 +983,8 @@ components: link: type: string description: Actor identifier of actor to link this actor's events too. May be an actor id or an alias. Cycles not permitted. + mounts: + $ref: '#/components/schemas/ArrayOfActorMounts' owner: type: string description: The user who created this actor. @@ -1242,4 +1244,9 @@ components: type: array items: type: string - description: The roles associated with the nonce. \ No newline at end of file + description: The roles associated with the nonce. + + + + + From 6e72b0261337e4a4944ab1cae70039bb0574eace Mon Sep 17 00:00:00 2001 From: Joe Stubbs Date: Mon, 28 Oct 2019 15:52:37 -0500 Subject: [PATCH 12/12] Add 1.5.0 entry. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99548137..782b6891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Change Log All notable changes to this project will be documented in this file. -## 1.5.0 - 2019-11-10 (target) +## 1.5.0 - 2019-10-29 ### Added - Added an endpoint `PUT /actors/aliases/{alias}` for updating the definition of an alias. Requires `UPDATE` permission for the alias as well as for the actor to which the alias should be defined.