From 94b13455d8fed8d544a2aecda48b7c04b0fe2cd5 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sat, 6 Jan 2018 11:10:03 -0800 Subject: [PATCH 01/31] Update setup.py * Add support for custom worker identifiers (activity and deciders). * Add logging on deciders. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a2c3fe3..113502c 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='Garcon', - version='0.3.5', + version='0.4.0', url='https://github.com/xethorn/garcon/', author='Michael Ortali', author_email='hello@xethorn.net', From df028d6ae02176e5c4cdf0affb39bd2112d811b4 Mon Sep 17 00:00:00 2001 From: Bakuutin Date: Mon, 1 Jan 2018 13:43:22 +0300 Subject: [PATCH 02/31] Added tox --- requirements.txt | 3 ++- tox.ini | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tox.ini diff --git a/requirements.txt b/requirements.txt index cdc0b3a..14675f4 100755 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ backoff==1.4.3 boto==2.35.1 pytest-cov==1.8.1 pytest==2.6.4 -python-coveralls==2.4.3 \ No newline at end of file +python-coveralls==2.4.3 +tox==2.9.1 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..8d7ccb0 --- /dev/null +++ b/tox.ini @@ -0,0 +1,12 @@ +[tox] +envlist = py27,py34,py35,py36 + +[testenv] +deps=pytest +commands=py.test + +[testenv:py27] +deps = -rrequirements-py2.txt + +[testenv:py3] +deps = -rrequirements.txt From e5ba08f4c99f88312b667aa8a68d9d6953759fad Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sun, 15 Jul 2018 17:22:07 -0700 Subject: [PATCH 03/31] Migrate from boto2 to boto3. Boto3 has been out for some time now and boto2 will likely be deprecated in months / years to come. Since most AWS users are now on boto3, migrating Garcon to Boto3. This upgrade introduces couple breaking changes: * `Workflow` now needs a `client` (`boto3.client('aws', region_name='us-east-1')`) to work. It enables workflows to run on specific AWS regions. * `ActivityExecution` is now passed to the Runner instead of the `Activity`. Similar to boto2 activities, you can perform operations on the execution such as `complete`, `fail` and `heartbeat`. --- .travis.yml | 23 ++- README.rst | 10 +- example/custom_decider/run.py | 22 ++- example/custom_decider/workflow.py | 22 +-- example/simple/run.py | 19 ++- example/simple/workflow.py | 10 +- garcon/activity.py | 139 +++++++++++++---- garcon/decider.py | 115 ++++++++------ garcon/utils.py | 8 +- requirements.txt | 6 +- setup.py | 3 + tests/conftest.py | 13 ++ tests/fixtures/flows/example.py | 4 +- tests/test_activity.py | 235 +++++++++++++++-------------- tests/test_context.py | 11 -- tests/test_decider.py | 166 ++++++++++---------- tests/test_runner.py | 23 ++- tests/test_utils.py | 22 +-- tox.ini | 2 +- 19 files changed, 492 insertions(+), 361 deletions(-) create mode 100644 tests/conftest.py diff --git a/.travis.yml b/.travis.yml index 2a998ac..897cfb3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,24 @@ language: python -python: - - "2.7" - - "3.4" +matrix: + include: + - python: 2.7 + dist: trusty + sudo: false + - python: 3.3 + dist: trusty + sudo: false + - python: 3.4 + dist: trusty + sudo: false + - python: 3.5 + dist: trusty + sudo: false + - python: 3.6 + dist: trusty + sudo: false + - python: 3.7 + dist: xenial + sudo: true env: - BOTO_CONFIG=/tmp/nowhere install: diff --git a/README.rst b/README.rst index b76fd53..5a26689 100644 --- a/README.rst +++ b/README.rst @@ -10,8 +10,8 @@ Lightweight library for AWS SWF. Requirements ~~~~~~~~~~~~ -- Python 2.7, 3.4 (tested) -- Boto 2.34.0 (tested) +- Python 2.7, 3.4, 3.5, 3.6, 3.7 (tested) +- Boto3 (tested) Goal ~~~~ @@ -35,13 +35,15 @@ be completed. from __future__ import print_function + import boto3 from garcon import activity from garcon import runner + client = boto3.client('swf', region_name='us-east-1') domain = 'dev' name = 'workflow_sample' - create = activity.create(domain, name) + create = activity.create(client, domain, name) test_activity_1 = create( name='activity_1', @@ -97,4 +99,4 @@ Contributors .. |Build Status| image:: https://travis-ci.org/xethorn/garcon.svg :target: https://travis-ci.org/xethorn/garcon .. |Coverage Status| image:: https://coveralls.io/repos/xethorn/garcon/badge.svg?branch=master - :target: https://coveralls.io/r/xethorn/garcon?branch=master \ No newline at end of file + :target: https://coveralls.io/r/xethorn/garcon?branch=master diff --git a/example/custom_decider/run.py b/example/custom_decider/run.py index 7d0f13a..8b4eaf0 100644 --- a/example/custom_decider/run.py +++ b/example/custom_decider/run.py @@ -1,20 +1,28 @@ from garcon import activity from garcon import decider from threading import Thread -import boto.swf.layer2 as swf import time +import boto3 import workflow # Initiate the workflow on the dev domain and custom_decider name. -flow = workflow.Workflow('dev', 'custom_decider') -deciderworker = decider.DeciderWorker(flow) +client = boto3.client('swf', region_name='us-east-1') +workflow = workflow.Workflow(client, 'dev', 'custom_decider') +deciderworker = decider.DeciderWorker(workflow) -# swf.WorkflowType( -# name=flow.name, domain=flow.domain, -# version='1.0', task_list=flow.name).start() +client.start_workflow_execution( + domain=workflow.domain, + workflowId='unique-workflow-identifier', + workflowType=dict( + name=workflow.name, + version='1.0'), + executionStartToCloseTimeout='3600', + taskStartToCloseTimeout='3600', + childPolicy='TERMINATE', + taskList=dict(name=workflow.name)) -Thread(target=activity.ActivityWorker(flow).run).start() +Thread(target=activity.ActivityWorker(workflow).run).start() while(True): deciderworker.run() time.sleep(1) diff --git a/example/custom_decider/workflow.py b/example/custom_decider/workflow.py index f8c701c..6f7eaef 100644 --- a/example/custom_decider/workflow.py +++ b/example/custom_decider/workflow.py @@ -9,21 +9,20 @@ Workflow: 1. Enter in the coffee shop. 2. Order. - 2.1 Chocolate chip cookie. - 2.2 Coffee. + 2.1 Coffee. + 2.2 Chocolate chip cookie. 3. Finalize the order (any of the activites below can be done in any order) 3.1 Pays. 3.2 Get the order. 4. Leave the coffee shop. Result: - -entering coffee shop -ordering: coffee -ordering: chocolate_chip_cookie -pay $6.98 -get order -leaving the coffee shop + entering coffee shop + ordering: coffee + ordering: chocolate_chip_cookie + pay $6.98 + get order + leaving the coffee shop """ from garcon import activity @@ -33,7 +32,7 @@ class Workflow: - def __init__(self, domain, name): + def __init__(self, client, domain, name): """Create the workflow. Args: @@ -42,7 +41,8 @@ def __init__(self, domain, name): """ self.name = name self.domain = domain - self.create_activity = activity.create(domain, name) + self.client = client + self.create_activity = activity.create(client, domain, name) def decider(self, schedule, context=None): """Custom deciders. diff --git a/example/simple/run.py b/example/simple/run.py index ddb475d..97d630d 100644 --- a/example/simple/run.py +++ b/example/simple/run.py @@ -1,18 +1,23 @@ from garcon import activity from garcon import decider from threading import Thread -import boto.swf.layer2 as swf +import boto3 import time -import test_flow +import workflow -deciderworker = decider.DeciderWorker(test_flow) +client = boto3.client('swf', region_name='us-east-1') +deciderworker = decider.DeciderWorker(client, workflow) -swf.WorkflowType( - name=test_flow.name, domain=test_flow.domain, - version='1.0', task_list=test_flow.name).start() +client.start_workflow_execution( + domain=workflow.domain, + workflowId='unique-workflow-identifier', + workflowType=dict( + name=workflow.name, + version='1.0'), + taskList=dict(name=workflow.name)) -Thread(target=activity.ActivityWorker(test_flow).run).start() +Thread(target=activity.ActivityWorker(client, workflow).run).start() while(True): deciderworker.run() time.sleep(1) diff --git a/example/simple/workflow.py b/example/simple/workflow.py index 3bc0ba8..e75f1d1 100644 --- a/example/simple/workflow.py +++ b/example/simple/workflow.py @@ -1,16 +1,15 @@ -import boto.swf.layer2 as swf - from garcon import activity from garcon import runner +import boto3 import logging import random logger = logging.getLogger(__name__) - +client = boto3.client('swf', region_name='us-east-1') domain = 'dev' name = 'workflow_sample' -create = activity.create(domain, name) +create = activity.create(client, domain, name) def activity_failure(context, activity): @@ -18,8 +17,7 @@ def activity_failure(context, activity): if num != 3: logger.warn('activity_3: fails') raise Exception('fails') - - print('activity_3: end') + logger.debug('activity_3: end') test_activity_1 = create( diff --git a/garcon/activity.py b/garcon/activity.py index df03027..1921980 100755 --- a/garcon/activity.py +++ b/garcon/activity.py @@ -13,10 +13,12 @@ Create an activity:: + import boto3 from garcon import activity # First step is to create the workflow on a specific domain. - create = activity.create('domain') + client = boto3.client('swf') + create = activity.create(client, 'domain', 'workflow-name') initial_activity = create( # Name of your activity @@ -37,18 +39,16 @@ """ -import backoff -import boto.exception as boto_exception -import boto.swf.layer2 as swf +from botocore import exceptions import itertools import json import threading +import backoff from garcon import log from garcon import utils from garcon import runner - ACTIVITY_STANDBY = 0 ACTIVITY_SCHEDULED = 1 ACTIVITY_COMPLETED = 2 @@ -243,13 +243,25 @@ def create_execution_input(self): return activity_input -class Activity(swf.ActivityWorker, log.GarconLogger): +class Activity(log.GarconLogger): version = '1.0' task_list = None + def __init__(self, client): + """Instantiates an activity. + + Args: + client: the boto client used for this activity. + """ + + self.client = client + self.name = None + self.domain = None + self.task_list = None + @backoff.on_exception( backoff.expo, - boto_exception.SWFResponseError, + exceptions.ClientError, max_tries=5, giveup=utils.non_throttle_error, on_backoff=utils.throttle_backoff_handler, @@ -266,9 +278,22 @@ def poll_for_activity(self, identity=None): identity (str): Identity of the worker making the request, which is recorded in the ActivityTaskStarted event in the AWS console. This enables diagnostic tracing when problems arise. + Return: + ActivityExecution: activity execution. """ - return self.poll(identity=identity) + additional_params = {} + if identity: + additional_params.update(identity=identity) + + execution_definition = self.client.poll_for_activity_task( + domain=self.domain, taskList=dict(name=self.task_list), + **additional_params) + + return ActivityExecution( + self.client, execution_definition.get('activityId'), + execution_definition.get('taskToken'), + execution_definition.get('input')) def run(self, identity=None): """Activity Runner. @@ -278,15 +303,14 @@ def run(self, identity=None): previous activity is consumed (context). Args: - activity_id (str): Identity of the worker making the request, which + identity (str): Identity of the worker making the request, which is recorded in the ActivityTaskStarted event in the AWS console. This enables diagnostic tracing when problems arise. """ - try: if identity: self.logger.debug('Polling with {}'.format(identity)) - activity_task = self.poll_for_activity(identity) + execution = self.poll_for_activity(identity) except Exception as error: # Catch exceptions raised during poll() to avoid an Activity thread # dying & worker daemon unable to process the affected Activity. @@ -297,27 +321,20 @@ def run(self, identity=None): # when an exception occurs. if self.on_exception: self.on_exception(self, error) - self.logger.error(error, exc_info=True) return True - packed_context = activity_task.get('input') - context = dict() - - if packed_context: - context = json.loads(packed_context) - self.set_log_context(context) - - if 'activityId' in activity_task: + self.set_log_context(execution.context) + if execution.activity_id: try: - context = self.execute_activity(context) - self.complete(result=json.dumps(context)) + context = self.execute_activity(execution) + execution.complete(context) except Exception as error: # If the workflow has been stopped, it is not possible for the # activity to be updated – it throws an exception which stops # the worker immediately. try: - self.fail(reason=str(error)[:255]) + execution.fail(str(error)[:255]) if self.on_exception: self.on_exception(self, error) except: @@ -326,14 +343,17 @@ def run(self, identity=None): self.unset_log_context() return True - def execute_activity(self, context): + def execute_activity(self, activity): """Execute the runner. Args: - context (dict): The flow context. + execution (ActivityExecution): the activity execution. + + Return: + dict: The result of the operation. """ - return self.runner.execute(self, context) + return self.runner.execute(activity, activity.context) def hydrate(self, data): """Hydrate the task with information provided. @@ -428,6 +448,7 @@ def __init__(self, timeout=None, heartbeat=None): will be equal to the timeout. """ + Activity.__init__(self, client=None) self.runner = runner.External(timeout=timeout, heartbeat=heartbeat) def run(self): @@ -440,6 +461,56 @@ def run(self): return False +class ActivityExecution: + + def __init__(self, client, activity_id, task_token, context): + """Create an an activity execution. + + Args: + client (boto3.client): the boto client (for easy access if needed). + activity_id (str): the activity id. + task_token (str): the task token. + context (str): data for the execution. + """ + + self.client = client + self.activity_id = activity_id + self.task_token = task_token + self.context = context and json.loads(context) or dict() + + def heartbeat(self, details=None): + """Create a task heartbeat. + + Args: + details (str): details to add to the heartbeat. + """ + + self.client.record_activity_task_heartbeat(self.task_token, + details=details or '') + + def fail(self, reason=None): + """Mark the activity execution as failed. + + Args: + reason (str): optional reason for the failure. + """ + + self.client.respond_activity_task_failed( + taskToken=self.task_token, + reason=reason or '') + + def complete(self, context=None): + """Mark the activity execution as completed. + + Args: + context (str or dict): the context result of the operation. + """ + + self.client.respond_activity_task_completed( + taskToken=self.task_token, + result=json.dumps(context)) + + class ActivityWorker(): def __init__(self, flow, activities=None): @@ -549,7 +620,7 @@ def wait(self): """Wait until ready. """ - if not self.ready(): + if not self.ready: raise ActivityInstanceNotReadyException() @@ -560,11 +631,11 @@ def worker_runner(worker): worker (object): the Activity worker. """ - while(worker.run()): + while (worker.run()): continue -def create(domain, name, version='1.0', on_exception=None): +def create(client, domain, workflow_name, version='1.0', on_exception=None): """Helper method to create Activities. The helper method simplifies the creation of an activity by setting the @@ -576,8 +647,9 @@ def create(domain, name, version='1.0', on_exception=None): activity. Always make sure your activity name is unique. Args: + client (boto3.client): the boto3 client. domain (str): the domain name. - name (str): name of the activity. + workflow_name (str): workflow name. version (str): activity version. on_exception (callable): the error handler. @@ -586,7 +658,7 @@ def create(domain, name, version='1.0', on_exception=None): """ def wrapper(**options): - activity = Activity() + activity = Activity(client) if options.get('external'): activity = ExternalActivity( @@ -594,7 +666,7 @@ def wrapper(**options): heartbeat=options.get('heartbeat')) activity_name = '{name}_{activity}'.format( - name=name, + name=workflow_name, activity=options.get('name')) activity.hydrate(dict( @@ -610,6 +682,7 @@ def wrapper(**options): schedule_to_start=options.get('schedule_to_start'), on_exception=options.get('on_exception') or on_exception)) return activity + return wrapper @@ -635,7 +708,7 @@ def find_available_activities(flow, history, context): if states.get_last_state() != ACTIVITY_FAILED: continue elif (not instance.retry or - instance.retry < count_activity_failures(states)): + instance.retry < count_activity_failures(states)): raise Exception( 'The activity failures has exceeded its retry limit.') diff --git a/garcon/decider.py b/garcon/decider.py index f45e717..30eda46 100755 --- a/garcon/decider.py +++ b/garcon/decider.py @@ -7,18 +7,15 @@ executed and when based on the flow procided. """ -from boto.swf.exceptions import SWFDomainAlreadyExistsError -from boto.swf.exceptions import SWFTypeAlreadyExistsError -import boto.swf.layer2 as swf import functools import json +import uuid from garcon import activity from garcon import event from garcon import log - -class DeciderWorker(swf.Decider, log.GarconLogger): +class DeciderWorker(log.GarconLogger): def __init__(self, flow, register=True): """Initialize the Decider Worker. @@ -28,18 +25,18 @@ def __init__(self, flow, register=True): register (boolean): If this flow needs to be register on AWS. """ + self.client = flow.client self.flow = flow self.domain = flow.domain self.version = getattr(flow, 'version', '1.0') self.activities = activity.find_workflow_activities(flow) self.task_list = flow.name self.on_exception = getattr(flow, 'on_exception', None) - super(DeciderWorker, self).__init__() if register: self.register() - def get_history(self, poll): + def get_history(self, identity, poll): """Get all the history. The full history needs to be recovered from SWF to make sure that all @@ -47,6 +44,7 @@ def get_history(self, poll): 100 events are provided, this methods retrieves all events. Args: + identity (string): The identity of the decider that pulls the information. poll (object): The poll object (see AWS SWF for details.) Return: list: All the events. @@ -54,7 +52,11 @@ def get_history(self, poll): events = poll['events'] while 'nextPageToken' in poll: - poll = self.poll(next_page_token=poll['nextPageToken']) + poll = self.client.poll_for_decision_task( + domain=self.domain, + identity=identity, + taskList=dict(name=self.task_list), + next_page_token=poll['nextPageToken']) if 'events' in poll: events += poll['events'] @@ -62,7 +64,6 @@ def get_history(self, poll): # Remove all the events that are related to decisions and only. return [e for e in events if not e['eventType'].startswith('Decision')] - def get_activity_states(self, history): """Get the activity states from the history. @@ -86,28 +87,31 @@ def register(self): """ registerables = [] - registerables.append(swf.Domain(name=self.domain)) - registerables.append(swf.WorkflowType( - domain=self.domain, - name=self.task_list, - version=self.version, - task_list=self.task_list)) + registerables.append(( + self.client.register_domain, + dict(name=self.domain, + workflowExecutionRetentionPeriodInDays='90'))) + registerables.append(( + self.client.register_workflow_type, + dict( + domain=self.domain, + name=self.task_list, + version=self.version))) for current_activity in self.activities: - registerables.append( - swf.ActivityType( + registerables.append(( + self.client.register_activity_type, + dict( domain=self.domain, name=current_activity.name, - version=self.version, - task_list=current_activity.task_list)) + version=self.version))) - for swf_entity in registerables: + for callable, data in registerables: try: - swf_entity.register() - except (SWFDomainAlreadyExistsError, SWFTypeAlreadyExistsError): - print( - swf_entity.__class__.__name__, swf_entity.name, - 'already exists') + callable(**data) + except Exception as e: + print(e) + print(data.get('name'), 'already exists') def create_decisions_from_flow(self, decisions, activity_states, context): """Create the decisions from the flow. @@ -117,7 +121,7 @@ def create_decisions_from_flow(self, decisions, activity_states, context): to schedule is thus very straightforward. Args: - decisions (Layer1Decisions): the layer decision for swf. + decisions (list): the layer decision for swf. activity_states (dict): all the state activities. context (dict): the context of the activities. """ @@ -125,7 +129,6 @@ def create_decisions_from_flow(self, decisions, activity_states, context): try: for current in activity.find_available_activities( self.flow, activity_states, context.current): - schedule_activity_task( decisions, current, version=self.version) else: @@ -133,9 +136,13 @@ def create_decisions_from_flow(self, decisions, activity_states, context): activity.find_uncomplete_activities( self.flow, activity_states, context.current)) if not activities: - decisions.complete_workflow_execution() + decisions.append(dict( + decisionType='CompleteWorkflowExecution')) except Exception as e: - decisions.fail_workflow_execution(reason=str(e)) + decisions.append(dict( + decisionType='FailWorkflowExecution', + failWorkflowExecutionDecisionAttributes=dict( + reason=str(e)))) if self.on_exception: self.on_exception(self, e) self.logger.error(e, exc_info=True) @@ -149,7 +156,7 @@ def delegate_decisions(self, decisions, decider, history, context): and if scheduled, returns its result. Args: - decisions (Layer1Decisions): the layer decision for swf. + decisions (list): the layer decision for swf. decider (callable): the decider (it needs to have schedule) history (dict): all the state activities and its history. context (dict): the context of the activities. @@ -172,11 +179,15 @@ def delegate_decisions(self, decisions, decider, history, context): # When no exceptions are raised and the method decider has returned # it means that there i nothing left to do in the current decider. if schedule_context.completed: - decisions.complete_workflow_execution() + decisions.append(dict( + decisionType='CompleteWorkflowExecution')) except activity.ActivityInstanceNotReadyException: pass except Exception as e: - decisions.fail_workflow_execution(reason=str(e)) + decisions.append(dict( + decisionType='FailWorkflowExecution', + failWorkflowExecutionDecisionAttributes=dict( + reason=str(e)))) if self.on_exception: self.on_exception(self, e) self.logger.error(e, exc_info=True) @@ -205,7 +216,10 @@ def run(self, identity=None): """ try: - poll = self.poll(identity=identity) + poll = self.client.poll_for_decision_task( + domain=self.domain, + taskList=dict(name=self.task_list), + identity=identity or '') except Exception as error: # Catch exceptions raised during poll() to avoid a Decider thread # dying & the daemon unable to process subsequent workflows. @@ -224,19 +238,21 @@ def run(self, identity=None): if 'events' not in poll: return True - history = self.get_history(poll) + history = self.get_history(identity, poll) activity_states = self.get_activity_states(history) current_context = event.get_current_context(history) current_context.set_workflow_execution_info(poll, self.domain) - decisions = swf.Layer1Decisions() + decisions = [] if not custom_decider: self.create_decisions_from_flow( decisions, activity_states, current_context) else: self.delegate_decisions( decisions, custom_decider, activity_states, current_context) - self.complete(decisions=decisions) + self.client.respond_decision_task_completed( + taskToken=poll.get('taskToken'), + decisions=decisions) return True @@ -277,16 +293,19 @@ def schedule_activity_task( id (str): optional id of the activity instance. """ - decisions.schedule_activity_task( - id or instance.id, - instance.activity_name, - version, - task_list=instance.activity_worker.task_list, - input=json.dumps(instance.create_execution_input()), - heartbeat_timeout=str(instance.heartbeat_timeout), - start_to_close_timeout=str(instance.timeout), - schedule_to_start_timeout=str(instance.schedule_to_start), - schedule_to_close_timeout=str(instance.schedule_to_close)) + decisions.append(dict( + decisionType='ScheduleActivityTask', + scheduleActivityTaskDecisionAttributes=dict( + activityId=id or instance.id, + activityType=dict( + name=instance.activity_name, + version=version), + taskList=dict(name=instance.activity_worker.task_list), + input=json.dumps(instance.create_execution_input()), + heartbeatTimeout=str(instance.heartbeat_timeout), + startToCloseTimeout=str(instance.timeout), + scheduleToStartTimeout=str(instance.schedule_to_start), + scheduleToCloseTimeout=str(instance.schedule_to_close)))) def schedule( @@ -299,7 +318,7 @@ def schedule( input with the full execution context to send the data to the activity. Args: - decisions (Layer1Decisions): the layer decision for swf. + decisions (list): the layer decision for swf. schedule_context (dict): information about the schedule. history (dict): history of the execution. context (dict): context of the execution. @@ -340,7 +359,7 @@ def schedule( if states.get_last_state() != activity.ACTIVITY_FAILED: continue elif (not current.retry or - current.retry < activity.count_activity_failures(states)): + current.retry < activity.count_activity_failures(states)): raise Exception( 'The activity failures has exceeded its retry limit.') diff --git a/garcon/utils.py b/garcon/utils.py index 3cba14e..0958e2f 100644 --- a/garcon/utils.py +++ b/garcon/utils.py @@ -30,18 +30,18 @@ def create_dictionary_key(dictionary): return hashlib.sha1(key_parts.encode('utf-8')).hexdigest() -def non_throttle_error(swf_response_error): +def non_throttle_error(exception): """Activity Runner. Determine whether SWF Exception was a throttle or a different error. Args: - error: boto.exception.SWFResponseError instance. + exception: botocore.exceptions.Client instance. Return: - bool: True if SWFResponseError was a throttle, False otherwise. + bool: True if exception was a throttle, False otherwise. """ - return swf_response_error.error_code != 'ThrottlingException' + return exception.response.get('Error').get('Code') != 'ThrottlingException' def throttle_backoff_handler(details): """Callback to be used when a throttle backoff is invoked. diff --git a/requirements.txt b/requirements.txt index 14675f4..102ce9c 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ backoff==1.4.3 -boto==2.35.1 -pytest-cov==1.8.1 -pytest==2.6.4 +boto3==1.7.58 +pytest-cov==2.5.1 +pytest==3.6.3 python-coveralls==2.4.3 tox==2.9.1 diff --git a/setup.py b/setup.py index 113502c..9dc1ea5 100755 --- a/setup.py +++ b/setup.py @@ -26,4 +26,7 @@ classifiers=[ 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', ],) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..1205cbd --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,13 @@ +try: + from unittest.mock import MagicMock +except: + from mock import MagicMock + +import pytest +import boto3 + + +@pytest.fixture +def boto_client(monkeypatch): + """Create a fake boto client.""" + return MagicMock(spec=boto3.client('swf', region_name='us-east-1')) diff --git a/tests/fixtures/flows/example.py b/tests/fixtures/flows/example.py index d0b0be7..d3c37d8 100755 --- a/tests/fixtures/flows/example.py +++ b/tests/fixtures/flows/example.py @@ -3,10 +3,12 @@ from garcon import activity from garcon import runner +import boto3 +client = boto3.client('swf', region_name='us-east-1') domain = 'dev' name = 'workflow_name' -create = activity.create(domain, name) +create = activity.create(client, domain, name) activity_1 = create( name='activity_1', diff --git a/tests/test_activity.py b/tests/test_activity.py index ba7d9df..ec9c92a 100755 --- a/tests/test_activity.py +++ b/tests/test_activity.py @@ -2,10 +2,14 @@ from __future__ import print_function try: from unittest.mock import MagicMock + from unittest.mock import ANY except: from mock import MagicMock + from mock import ANY +from botocore import exceptions import json import pytest +import sys from garcon import activity from garcon import event @@ -14,28 +18,26 @@ from garcon import utils from tests.fixtures import decider -import boto.exception as boto_exception - def activity_run( - monkeypatch, poll=None, complete=None, fail=None, execute=None): + monkeypatch, boto_client, poll=None, complete=None, fail=None, + execute=None): """Create an activity. """ - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) - - current_activity = activity.Activity() + current_activity = activity.Activity(boto_client) poll = poll or dict() monkeypatch.setattr( current_activity, 'execute_activity', execute or MagicMock(return_value=dict())) monkeypatch.setattr( - current_activity, 'poll', MagicMock(return_value=poll)) + boto_client, 'poll_for_activity_task', MagicMock(return_value=poll)) monkeypatch.setattr( - current_activity, 'complete', complete or MagicMock()) + boto_client, 'respond_activity_task_completed', complete or MagicMock()) monkeypatch.setattr( - current_activity, 'fail', fail or MagicMock()) + boto_client, 'respond_activity_task_failed', fail or MagicMock()) + return current_activity @@ -61,104 +63,104 @@ def dgenerator(context): @pytest.fixture def poll(): - return dict(activityId='something') + return dict( + activityId='something', + taskToken='taskToken') -def test_poll_for_activity(monkeypatch, poll=poll): +def test_poll_for_activity(monkeypatch, poll, boto_client): """Test that poll_for_activity successfully polls. """ - current_activity = activity_run(monkeypatch, poll) - current_activity.poll.return_value = 'foo' + activity_task = poll + current_activity = activity_run(monkeypatch, boto_client, poll) + boto_client.poll_for_activity_task.return_value = activity_task - activity_results = current_activity.poll_for_activity() - assert current_activity.poll.called - assert activity_results == 'foo' + activity_execution = current_activity.poll_for_activity() + assert boto_client.poll_for_activity_task.called + assert activity_execution.task_token is poll.get('taskToken') -def test_poll_for_activity_throttle_retry(monkeypatch, poll=poll): +def test_poll_for_activity_throttle_retry(monkeypatch, poll, boto_client): """Test that SWF throttles are retried during polling. """ - current_activity = activity_run(monkeypatch, poll) - - response_status = 400 - response_reason = 'Bad Request' - reponse_body = ( - '{"__type": "com.amazon.coral.availability#ThrottlingException",' - '"message": "Rate exceeded"}') - json_body = json.loads(reponse_body) - exception = boto_exception.SWFResponseError( - response_status, response_reason, body=json_body) - current_activity.poll.side_effect = exception + current_activity = activity_run(monkeypatch, boto_client, poll) + boto_client.poll_for_activity_task.side_effect = exceptions.ClientError( + {'Error': {'Code': 'ThrottlingException'}}, + 'operation name') - with pytest.raises(boto_exception.SWFResponseError): + with pytest.raises(exceptions.ClientError): current_activity.poll_for_activity() - assert current_activity.poll.call_count == 5 + assert boto_client.poll_for_activity_task.call_count == 5 -def test_poll_for_activity_error(monkeypatch, poll=poll): +def test_poll_for_activity_error(monkeypatch, poll, boto_client): """Test that non-throttle errors during poll are thrown. """ - current_activity = activity_run(monkeypatch, poll) + current_activity = activity_run(monkeypatch, boto_client, poll) exception = Exception() - current_activity.poll.side_effect = exception + boto_client.poll_for_activity_task.side_effect = exception with pytest.raises(Exception): current_activity.poll_for_activity() -def test_poll_for_activity_identity(monkeypatch, poll=poll): +def test_poll_for_activity_identity(monkeypatch, poll, boto_client): """Test that identity is passed to poll_for_activity. """ - current_activity = activity_run(monkeypatch, poll) + current_activity = activity_run(monkeypatch, boto_client, poll) current_activity.poll_for_activity(identity='foo') - current_activity.poll.assert_called_with(identity='foo') + boto_client.poll_for_activity_task.assert_called_with( + domain=ANY, taskList=ANY, identity='foo') -def test_poll_for_activity_no_identity(monkeypatch, poll=poll): +def test_poll_for_activity_no_identity(monkeypatch, poll, boto_client): """Test poll_for_activity works without identity passed as param. """ - current_activity = activity_run(monkeypatch, poll) + current_activity = activity_run(monkeypatch, boto_client, poll) current_activity.poll_for_activity() - current_activity.poll.assert_called_with(identity=None) + boto_client.poll_for_activity_task.assert_called_with( + domain=ANY, taskList=ANY) -def test_run_activity(monkeypatch, poll): +def test_run_activity(monkeypatch, poll, boto_client): """Run an activity. """ - current_activity = activity_run(monkeypatch, poll=poll) + current_activity = activity_run(monkeypatch, boto_client, poll=poll) current_activity.run() - current_activity.poll.assert_called_with(identity=None) + boto_client.poll_for_activity_task.assert_called_with( + domain=ANY, taskList=ANY) assert current_activity.execute_activity.called - assert current_activity.complete.called + assert boto_client.respond_activity_task_completed.called -def test_run_activity_identity(monkeypatch, poll): +def test_run_activity_identity(monkeypatch, poll, boto_client): """Run an activity with identity as param. """ - current_activity = activity_run(monkeypatch, poll=poll) + current_activity = activity_run(monkeypatch, boto_client, poll=poll) current_activity.run(identity='foo') - current_activity.poll.assert_called_with(identity='foo') + boto_client.poll_for_activity_task.assert_called_with( + domain=ANY, taskList=ANY, identity='foo') assert current_activity.execute_activity.called - assert current_activity.complete.called + assert boto_client.respond_activity_task_completed.called -def test_run_capture_exception(monkeypatch, poll): +def test_run_capture_exception(monkeypatch, poll, boto_client): """Run an activity with an exception raised during activity execution. """ - current_activity = activity_run(monkeypatch, poll=poll) + current_activity = activity_run(monkeypatch, boto_client, poll) current_activity.on_exception = MagicMock() current_activity.execute_activity = MagicMock() error_msg_long = "Error" * 100 @@ -166,28 +168,32 @@ def test_run_capture_exception(monkeypatch, poll): current_activity.execute_activity.side_effect = Exception(error_msg_long) current_activity.run() - assert current_activity.poll.called + assert boto_client.poll_for_activity_task.called assert current_activity.execute_activity.called assert current_activity.on_exception.called - current_activity.fail.assert_called_with(reason=actual_error_msg) - assert not current_activity.complete.called + boto_client.respond_activity_task_failed.assert_called_with( + taskToken=poll.get('taskToken'), + reason=actual_error_msg) + assert not boto_client.respond_activity_task_completed.called -def test_run_capture_poll_exception(monkeypatch, poll): +def test_run_capture_poll_exception(monkeypatch, boto_client, poll): """Run an activity with an exception raised during poll. """ - current_activity = activity_run(monkeypatch, poll=poll) + current_activity = activity_run(monkeypatch, boto_client, poll=poll) + current_activity.on_exception = MagicMock() current_activity.execute_activity = MagicMock() + exception = Exception('poll exception') - current_activity.poll.side_effect = exception + boto_client.poll_for_activity_task.side_effect = exception current_activity.run() - assert current_activity.poll.called + assert boto_client.poll_for_activity_task.called assert current_activity.on_exception.called assert not current_activity.execute_activity.called - assert not current_activity.complete.called + assert not boto_client.respond_activity_task_completed.called current_activity.on_exception = None current_activity.logger.error = MagicMock() @@ -195,97 +201,103 @@ def test_run_capture_poll_exception(monkeypatch, poll): current_activity.logger.error.assert_called_with(exception, exc_info=True) -def test_run_activity_without_id(monkeypatch): +def test_run_activity_without_id(monkeypatch, boto_client): """Run an activity without an activity id. """ - current_activity = activity_run(monkeypatch, dict()) + current_activity = activity_run(monkeypatch, boto_client, poll=dict()) current_activity.run() - assert current_activity.poll.called + assert boto_client.poll_for_activity_task.called assert not current_activity.execute_activity.called - assert not current_activity.complete.called + assert not boto_client.respond_activity_task_completed.called -def test_run_activity_with_context(monkeypatch, poll): +def test_run_activity_with_context(monkeypatch, boto_client, poll): """Run an activity with a context. """ context = dict(foo='bar') poll.update(input=json.dumps(context)) - current_activity = activity_run(monkeypatch, poll=poll) + current_activity = activity_run(monkeypatch, boto_client, poll=poll) current_activity.run() - activity_context = current_activity.execute_activity.call_args[0][0] - assert activity_context == context + activity_execution = current_activity.execute_activity.call_args[0][0] + assert activity_execution.context == context -def test_run_activity_with_result(monkeypatch, poll): +def test_run_activity_with_result(monkeypatch, boto_client, poll): """Run an activity with a result. """ - resp = dict(foo='bar') - mock = MagicMock(return_value=resp) - current_activity = activity_run(monkeypatch, poll=poll, execute=mock) + result = dict(foo='bar') + mock = MagicMock(return_value=result) + current_activity = activity_run(monkeypatch, boto_client, poll=poll, + execute=mock) current_activity.run() - result = current_activity.complete.call_args_list[0][1].get('result') - assert result == json.dumps(resp) + boto_client.respond_activity_task_completed.assert_called_with( + result=json.dumps(result), taskToken=poll.get('taskToken')) -def test_task_failure(monkeypatch, poll): +def test_task_failure(monkeypatch, boto_client, poll): """Run an activity that has a bad task. """ resp = dict(foo='bar') mock = MagicMock(return_value=resp) - current_activity = activity_run(monkeypatch, poll=poll, execute=mock) - current_activity.execute_activity.side_effect = Exception('fail') + reason = 'fail' + current_activity = activity_run(monkeypatch, boto_client, poll=poll, + execute=mock) + current_activity.execute_activity.side_effect = Exception(reason) current_activity.run() - assert current_activity.fail.called + boto_client.respond_activity_task_failed.assert_called_with( + taskToken=poll.get('taskToken'), + reason=reason) -def test_task_failure_on_close_activity(monkeypatch, poll): +def test_task_failure_on_close_activity(monkeypatch, boto_client, poll): """Run an activity failure when the task is already closed. """ resp = dict(foo='bar') mock = MagicMock(return_value=resp) - current_activity = activity_run(monkeypatch, poll=poll, execute=mock) + current_activity = activity_run(monkeypatch, boto_client, poll=poll, + execute=mock) current_activity.execute_activity.side_effect = Exception('fail') - current_activity.fail.side_effect = Exception('fail') + boto_client.respond_activity_task_failed.side_effect = Exception('fail') current_activity.unset_log_context = MagicMock() current_activity.run() assert current_activity.unset_log_context.called -def test_execute_activity(monkeypatch): +def test_execute_activity(monkeypatch, boto_client): """Test the execution of an activity. """ - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) - monkeypatch.setattr(activity.Activity, 'heartbeat', lambda self: None) + monkeypatch.setattr(activity.ActivityExecution, 'heartbeat', + lambda self: None) resp = dict(task_resp='something') custom_task = MagicMock(return_value=resp) - current_activity = activity.Activity() + current_activity = activity.Activity(boto_client) current_activity.runner = runner.Sync(custom_task) - val = current_activity.execute_activity(dict(foo='bar')) + val = current_activity.execute_activity(activity.ActivityExecution( + boto_client, 'activityId', 'taskToken', '{"context": "value"}')) assert custom_task.called assert val == resp -def test_hydrate_activity(monkeypatch): +def test_hydrate_activity(monkeypatch, boto_client): """Test the hydratation of an activity. """ - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) - current_activity = activity.Activity() + current_activity = activity.Activity(boto_client) current_activity.hydrate(dict( name='activity', domain='domain', @@ -294,26 +306,25 @@ def test_hydrate_activity(monkeypatch): tasks=[lambda: dict('val')])) -def test_create_activity(monkeypatch): +def test_create_activity(monkeypatch, boto_client): """Test the creation of an activity via `create`. """ - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) - create = activity.create('domain_name', 'flow_name') + create = activity.create(boto_client, 'domain_name', 'flow_name') current_activity = create(name='activity_name') assert isinstance(current_activity, activity.Activity) assert current_activity.name == 'flow_name_activity_name' assert current_activity.task_list == 'flow_name_activity_name' - assert current_activity.domain == 'domain_name' + assert current_activity.domain is 'domain_name' + assert current_activity.client is boto_client -def test_create_external_activity(monkeypatch): +def test_create_external_activity(monkeypatch, boto_client): """Test the creation of an external activity via `create`. """ - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) - create = activity.create('domain_name', 'flow_name') + create = activity.create(boto_client, 'domain_name', 'flow_name') current_activity = create( name='activity_name', @@ -335,7 +346,6 @@ def test_create_activity_worker(monkeypatch): """Test the creation of an activity worker. """ - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) from tests.fixtures.flows import example worker = activity.ActivityWorker(example) @@ -345,14 +355,12 @@ def test_create_activity_worker(monkeypatch): assert not worker.worker_activities -def test_instances_creation(monkeypatch, generators): +def test_instances_creation(monkeypatch, boto_client, generators): """Test the creation of an activity instance id with the use of a local context. """ - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) - - local_activity = activity.Activity() + local_activity = activity.Activity(boto_client) external_activity = activity.ExternalActivity(timeout=60) for current_activity in [local_activity, external_activity]: @@ -374,7 +382,7 @@ def test_instances_creation(monkeypatch, generators): assert not instances[0].local_context -def test_activity_timeouts(monkeypatch, generators): +def test_activity_timeouts(monkeypatch, boto_client, generators): """Test the creation of an activity timeouts. More details: the timeout of a task is 120s, the schedule to start is 1000, @@ -390,8 +398,7 @@ def test_activity_timeouts(monkeypatch, generators): def local_task(): return - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) - current_activity = activity.Activity() + current_activity = activity.Activity(boto_client) current_activity.hydrate(dict(schedule_to_start=start_timeout)) current_activity.generators = generators current_activity.runner = runner.Sync( @@ -408,14 +415,13 @@ def local_task(): schedule_to_start + instance.timeout) -def test_external_activity_timeouts(monkeypatch, generators): +def test_external_activity_timeouts(monkeypatch, boto_client, generators): """Test the creation of an external activity timeouts. """ timeout = 120 start_timeout = 1000 - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) current_activity = activity.ExternalActivity(timeout=timeout) current_activity.hydrate(dict(schedule_to_start=start_timeout)) current_activity.generators = generators @@ -430,44 +436,39 @@ def test_external_activity_timeouts(monkeypatch, generators): schedule_to_start + instance.timeout) -def test_worker_run(monkeypatch): +@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3") +def test_worker_run(monkeypatch, boto_client): """Test running the worker. """ - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) - from tests.fixtures.flows import example worker = activity.ActivityWorker(example) assert len(worker.activities) == 4 for current_activity in worker.activities: - monkeypatch.setattr( - current_activity, 'run', MagicMock(return_value=False)) + monkeypatch.setattr( + current_activity, 'run', MagicMock(return_value=False)) worker.run() assert len(worker.activities) == 4 for current_activity in worker.activities: - # this check was originally `assert current_activity.run.called` - # for some reason this fails on py2.7, so we explicitly check for - # `called == 1`. - assert current_activity.run.called == 1 + assert current_activity.run.called def test_worker_run_with_skipped_activities(monkeypatch): """Test running the worker with defined activities. """ - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) - monkeypatch.setattr( - activity.Activity, 'run', MagicMock(return_value=False)) + monkeypatch.setattr(activity.Activity, 'run', MagicMock(return_value=False)) + from tests.fixtures.flows import example worker = activity.ActivityWorker(example, activities=['activity_1']) assert len(worker.worker_activities) == 1 for current_activity in worker.activities: - monkeypatch.setattr( - current_activity, 'run', MagicMock(return_value=False)) + monkeypatch.setattr( + current_activity, 'run', MagicMock(return_value=False)) worker.run() diff --git a/tests/test_context.py b/tests/test_context.py index 96a8429..7fe9f23 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -3,24 +3,15 @@ from unittest.mock import MagicMock except: from mock import MagicMock -import boto.swf.layer2 as swf from garcon import context from tests.fixtures import decider as decider_events -def mock(monkeypatch): - for base in [swf.Decider, swf.WorkflowType, swf.ActivityType, swf.Domain]: - monkeypatch.setattr(base, '__init__', MagicMock(return_value=None)) - if base is not swf.Decider: - monkeypatch.setattr(base, 'register', MagicMock()) - - def test_context_creation_without_events(monkeypatch): """Check the basic context creation. """ - mock(monkeypatch) current_context = context.ExecutionContext() assert not current_context.current assert not current_context.workflow_input @@ -30,7 +21,6 @@ def test_context_creation_with_events(monkeypatch): """Test context creation with events. """ - mock(monkeypatch) from tests.fixtures import decider as poll current_context = context.ExecutionContext(poll.history.get('events')) @@ -40,7 +30,6 @@ def test_get_workflow_execution_info(monkeypatch): """Check that the workflow execution info are properly extracted """ - mock(monkeypatch) from tests.fixtures import decider as poll current_context = context.ExecutionContext() diff --git a/tests/test_decider.py b/tests/test_decider.py index 099e179..85df245 100755 --- a/tests/test_decider.py +++ b/tests/test_decider.py @@ -3,7 +3,6 @@ from unittest.mock import MagicMock except: from mock import MagicMock -import boto.swf.layer2 as swf import json import pytest @@ -12,21 +11,14 @@ from tests.fixtures import decider as decider_events -def mock(monkeypatch): - for base in [swf.Decider, swf.WorkflowType, swf.ActivityType, swf.Domain]: - monkeypatch.setattr(base, '__init__', MagicMock(return_value=None)) - if base is not swf.Decider: - monkeypatch.setattr(base, 'register', MagicMock()) - - def test_create_decider(monkeypatch): """Create a decider and check the behavior of the registration. """ - mock(monkeypatch) from tests.fixtures.flows import example d = decider.DeciderWorker(example) + assert d.client assert len(d.activities) == 4 assert d.flow assert d.domain @@ -45,7 +37,6 @@ def test_get_history(monkeypatch): """Test the decider history """ - mock(monkeypatch) from tests.fixtures.flows import example events = decider_events.history.get('events') @@ -53,13 +44,19 @@ def test_get_history(monkeypatch): events = events[:half * 2] pool_1 = events[half:] pool_2 = events[:half] + identity = 'identity' d = decider.DeciderWorker(example) - d.poll = MagicMock(return_value={'events': pool_2}) + d.client.poll_for_decision_task = MagicMock(return_value={'events': pool_2}) - resp = d.get_history({'events': pool_1, 'nextPageToken': 'nextPage'}) + resp = d.get_history( + identity, {'events': pool_1, 'nextPageToken': 'nextPage'}) - d.poll.assert_called_with(next_page_token='nextPage') + d.client.poll_for_decision_task.assert_called_with( + domain=example.domain, + next_page_token='nextPage', + identity=identity, + taskList=dict(name=d.task_list)) assert len(resp) == len([ evt for evt in events if evt['eventType'].startswith('Decision')]) @@ -68,12 +65,12 @@ def test_get_activity_states(monkeypatch): """Test get activity states from history. """ - mock(monkeypatch) from tests.fixtures.flows import example + identity= 'identity' events = decider_events.history.get('events') d = decider.DeciderWorker(example) - history = d.get_history({'events': events}) + history = d.get_history(identity, {'events': events}) states = d.get_activity_states(history) for activity_name, activity_instances in states.items(): @@ -85,11 +82,12 @@ def test_running_workflow(monkeypatch): """Test running a workflow. """ - mock(monkeypatch) from tests.fixtures.flows import example - d = decider.DeciderWorker(example) - d.poll = MagicMock(return_value=decider_events.history) + d = decider.DeciderWorker(example, register=False) + d.client.poll_for_decision_task = MagicMock( + return_value=decider_events.history) + d.client.respond_decision_task_completed = MagicMock() d.complete = MagicMock() d.create_decisions_from_flow = MagicMock() @@ -112,36 +110,41 @@ def test_running_workflow_identity(monkeypatch): """Test running a workflow with and without identity. """ - mock(monkeypatch) from tests.fixtures.flows import example - d = decider.DeciderWorker(example) - d.poll = MagicMock() + d = decider.DeciderWorker(example, register=False) + d.client.poll_for_decision_task = MagicMock() d.complete = MagicMock() d.create_decisions_from_flow = MagicMock() # assert running decider without identity d.run() - d.poll.assert_called_with(identity=None) + d.client.poll_for_decision_task.assert_called_with( + domain=d.domain, + taskList=dict(name=d.task_list), + identity='') # assert running decider with identity d.run('foo') - d.poll.assert_called_with(identity='foo') + d.client.poll_for_decision_task.assert_called_with( + domain=d.domain, + taskList=dict(name=d.task_list), + identity='foo') def test_running_workflow_exception(monkeypatch): """Run a decider with an exception raised during poll. """ - mock(monkeypatch) from tests.fixtures.flows import example - d = decider.DeciderWorker(example) - d.poll = MagicMock(return_value=decider_events.history) + d = decider.DeciderWorker(example, register=False) + d.client.poll_for_decision_task = MagicMock( + return_value=decider_events.history) d.complete = MagicMock() d.create_decisions_from_flow = MagicMock() exception = Exception('test') - d.poll.side_effect = exception + d.client.poll_for_decision_task.side_effect = exception d.on_exception = MagicMock() d.logger.error = MagicMock() d.run() @@ -154,10 +157,9 @@ def test_create_decisions_from_flow_exception(monkeypatch): """Test exception is raised and workflow fails when exception raised. """ - mock(monkeypatch) from tests.fixtures.flows import example - decider_worker = decider.DeciderWorker(example) + decider_worker = decider.DeciderWorker(example, register=False) decider_worker.logger.error = MagicMock() decider_worker.on_exception = MagicMock() @@ -165,14 +167,17 @@ def test_create_decisions_from_flow_exception(monkeypatch): monkeypatch.setattr(decider.activity, 'find_available_activities', MagicMock(side_effect = exception)) - mock_decisions = MagicMock() + decisions = [] mock_activity_states = MagicMock() mock_context = MagicMock() decider_worker.create_decisions_from_flow( - mock_decisions, mock_activity_states, mock_context) + decisions, mock_activity_states, mock_context) + + failed_execution = dict( + decisionType='FailWorkflowExecution', + failWorkflowExecutionDecisionAttributes=dict(reason=str(exception))) - mock_decisions.fail_workflow_execution.assert_called_with( - reason=str(exception)) + assert failed_execution in decisions assert decider_worker.on_exception.called decider_worker.logger.error.assert_called_with(exception, exc_info=True) @@ -181,11 +186,10 @@ def test_running_workflow_without_events(monkeypatch): """Test running a workflow without having any events. """ - mock(monkeypatch) from tests.fixtures.flows import example - d = decider.DeciderWorker(example) - d.poll = MagicMock(return_value={}) + d = decider.DeciderWorker(example, register=False) + d.client.poll_for_decision_task = MagicMock(return_value={}) d.get_history = MagicMock() d.run() @@ -207,7 +211,6 @@ def test_schedule_with_unscheduled_activity(monkeypatch): """Test the scheduling of an activity. """ - mock(monkeypatch) from tests.fixtures.flows import example monkeypatch.setattr(decider, 'schedule_activity_task', MagicMock()) @@ -229,7 +232,6 @@ def test_schedule_with_scheduled_activity(monkeypatch): """Test the scheduling of an activity. """ - mock(monkeypatch) from tests.fixtures.flows import example monkeypatch.setattr(decider, 'schedule_activity_task', MagicMock()) @@ -261,7 +263,6 @@ def test_schedule_with_completed_activity(monkeypatch): """Test the scheduling of an activity. """ - mock(monkeypatch) from tests.fixtures.flows import example monkeypatch.setattr(decider, 'schedule_activity_task', MagicMock()) @@ -306,65 +307,74 @@ def test_schedule_activity_task(monkeypatch): """Test scheduling an activity task. """ - mock(monkeypatch) from tests.fixtures.flows import example instance = list(example.activity_1.instances({}))[0] - decisions = MagicMock() + decisions = [] decider.schedule_activity_task(decisions, instance) - decisions.schedule_activity_task.assert_called_with( - instance.id, - instance.activity_name, - '1.0', - task_list=instance.activity_worker.task_list, - input=json.dumps(instance.create_execution_input()), - heartbeat_timeout=str(instance.heartbeat_timeout), - start_to_close_timeout=str(instance.timeout), - schedule_to_start_timeout=str(instance.schedule_to_start), - schedule_to_close_timeout=str(instance.schedule_to_close)) + expects = dict( + decisionType='ScheduleActivityTask', + scheduleActivityTaskDecisionAttributes=dict( + activityId=instance.id, + activityType=dict( + name=instance.activity_name, + version='1.0'), + taskList=dict(name=instance.activity_worker.task_list), + input=json.dumps(instance.create_execution_input()), + heartbeatTimeout=str(instance.heartbeat_timeout), + startToCloseTimeout=str(instance.timeout), + scheduleToStartTimeout=str(instance.schedule_to_start), + scheduleToCloseTimeout=str(instance.schedule_to_close))) + assert expects in decisions def test_schedule_activity_task_with_version(monkeypatch): """Test scheduling an activity task with a version. """ - mock(monkeypatch) from tests.fixtures.flows import example instance = list(example.activity_1.instances({}))[0] - decisions = MagicMock() + decisions = [] version = '2.0' decider.schedule_activity_task(decisions, instance, version=version) - decisions.schedule_activity_task.assert_called_with( - instance.id, - instance.activity_name, - version, - task_list=instance.activity_worker.task_list, - input=json.dumps(instance.create_execution_input()), - heartbeat_timeout=str(instance.heartbeat_timeout), - start_to_close_timeout=str(instance.timeout), - schedule_to_start_timeout=str(instance.schedule_to_start), - schedule_to_close_timeout=str(instance.schedule_to_close)) - - -def test_schedule_activity_task_with_version(monkeypatch): + expects = dict( + decisionType='ScheduleActivityTask', + scheduleActivityTaskDecisionAttributes=dict( + activityId=instance.id, + activityType=dict( + name=instance.activity_name, + version=version), + taskList=dict(name=instance.activity_worker.task_list), + input=json.dumps(instance.create_execution_input()), + heartbeatTimeout=str(instance.heartbeat_timeout), + startToCloseTimeout=str(instance.timeout), + scheduleToStartTimeout=str(instance.schedule_to_start), + scheduleToCloseTimeout=str(instance.schedule_to_close))) + assert expects in decisions + + +def test_schedule_activity_task_with_custom_id(monkeypatch): """Test scheduling an activity task with a custom id. """ - mock(monkeypatch) from tests.fixtures.flows import example instance = list(example.activity_1.instances({}))[0] - decisions = MagicMock() + decisions = [] custom_id = 'special_id' decider.schedule_activity_task(decisions, instance, id=custom_id) - decisions.schedule_activity_task.assert_called_with( - custom_id, - instance.activity_name, - '1.0', - task_list=instance.activity_worker.task_list, - input=json.dumps(instance.create_execution_input()), - heartbeat_timeout=str(instance.heartbeat_timeout), - start_to_close_timeout=str(instance.timeout), - schedule_to_start_timeout=str(instance.schedule_to_start), - schedule_to_close_timeout=str(instance.schedule_to_close)) + expects = dict( + decisionType='ScheduleActivityTask', + scheduleActivityTaskDecisionAttributes=dict( + activityId=custom_id, + activityType=dict( + name=instance.activity_name, + version='1.0'), + taskList=dict(name=instance.activity_worker.task_list), + input=json.dumps(instance.create_execution_input()), + heartbeatTimeout=str(instance.heartbeat_timeout), + startToCloseTimeout=str(instance.timeout), + scheduleToStartTimeout=str(instance.schedule_to_start), + scheduleToCloseTimeout=str(instance.schedule_to_close))) + assert expects in decisions diff --git a/tests/test_runner.py b/tests/test_runner.py index 0fe408c..c2bc4cc 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -23,18 +23,18 @@ def test_execute_default_task_runner(): current_runner.execute(None, None) -def test_synchronous_tasks(monkeypatch): +def test_synchronous_tasks(monkeypatch, boto_client): """Test synchronous tasks. """ - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) - monkeypatch.setattr(activity.Activity, 'heartbeat', lambda self: None) + monkeypatch.setattr(activity.ActivityExecution, 'heartbeat', + lambda self: None) resp = dict(foo='bar') current_runner = runner.Sync( MagicMock(), MagicMock(return_value=resp)) - current_activity = activity.Activity() - current_activity.hydrate(dict(runner=current_runner)) + current_activity = activity.ActivityExecution( + boto_client, 'activityId', 'taskToken', '{"context": "value"}') result = current_runner.execute(current_activity, EMPTY_CONTEXT) @@ -46,12 +46,12 @@ def test_synchronous_tasks(monkeypatch): assert resp == result -def test_aynchronous_tasks(monkeypatch): +def test_aynchronous_tasks(monkeypatch, boto_client): """Test asynchronous tasks. """ - monkeypatch.setattr(activity.Activity, '__init__', lambda self: None) - monkeypatch.setattr(activity.Activity, 'heartbeat', lambda self: None) + monkeypatch.setattr(activity.ActivityExecution, 'heartbeat', + lambda self: None) tasks = [MagicMock() for i in range(5)] tasks[2].return_value = dict(oi='mondo') @@ -66,11 +66,10 @@ def test_aynchronous_tasks(monkeypatch): assert current_runner.max_workers == workers assert len(current_runner.tasks) == len(tasks) - current_activity = activity.Activity() - current_activity.hydrate(dict(runner=current_runner)) + current_activity = activity.ActivityExecution(boto_client, + 'activityId', 'taskToken', '{"hello": "world"}') - context = dict(hello='world') - resp = current_runner.execute(current_activity, context) + resp = current_runner.execute(current_activity, current_activity.context) for current_task in tasks: assert current_task.called diff --git a/tests/test_utils.py b/tests/test_utils.py index 44564c7..333dac2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock except: from mock import MagicMock -import boto.exception as boto_exception +from botocore import exceptions import datetime import pytest import json @@ -53,23 +53,15 @@ def test_non_throttle_error(): """Assert SWF error is evaluated as non-throttle error properly. """ - response_status = 400 - response_reason = 'Bad Request' - reponse_body = ( - '{"__type": "com.amazon.coral.availability#ThrottlingException",' - '"message": "Rate exceeded"}') - json_body = json.loads(reponse_body) - exception = boto_exception.SWFResponseError( - response_status, response_reason, body=json_body) + exception = exceptions.ClientError( + {'Error': {'Code': 'ThrottlingException'}}, + 'operationName') result = utils.non_throttle_error(exception) assert not utils.non_throttle_error(exception) - reponse_body = ( - '{"__type": "com.amazon.coral.availability#OtheException",' - '"message": "Rate exceeded"}') - json_body = json.loads(reponse_body) - exception = boto_exception.SWFResponseError( - response_status, response_reason, body=json_body) + exception = exceptions.ClientError( + {'Error': {'Code': 'RateExceeded'}}, + 'operationName') assert utils.non_throttle_error(exception) def test_throttle_backoff_handler(): diff --git a/tox.ini b/tox.ini index 8d7ccb0..fe679c1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,py36 +envlist = py27,py34,py35,py36,py37 [testenv] deps=pytest From e0421969ba1b3dd01bcf7485556ce3c213cf8e6a Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sun, 22 Jul 2018 10:07:10 -0700 Subject: [PATCH 04/31] Refactor: split requirements in two files, one for library and one for tests --- .travis.yml | 1 + requirements-tests.txt | 4 ++++ requirements.txt | 4 ---- tox.ini | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 requirements-tests.txt diff --git a/.travis.yml b/.travis.yml index 897cfb3..34b5e84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ env: install: - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt; fi - "pip install -r requirements.txt" + - "pip install -r requirements-tests.txt" script: "py.test tests/ --cov garcon --cov-report term-missing" after_success: - coveralls diff --git a/requirements-tests.txt b/requirements-tests.txt new file mode 100644 index 0000000..fd809fd --- /dev/null +++ b/requirements-tests.txt @@ -0,0 +1,4 @@ +pytest-cov==2.5.1 +pytest==3.6.3 +python-coveralls==2.4.3 +tox==2.9.1 diff --git a/requirements.txt b/requirements.txt index 102ce9c..2ab4b98 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,2 @@ backoff==1.4.3 boto3==1.7.58 -pytest-cov==2.5.1 -pytest==3.6.3 -python-coveralls==2.4.3 -tox==2.9.1 diff --git a/tox.ini b/tox.ini index fe679c1..cb40329 100644 --- a/tox.ini +++ b/tox.ini @@ -2,8 +2,8 @@ envlist = py27,py34,py35,py36,py37 [testenv] -deps=pytest -commands=py.test +deps = -rrequirements-tests.txt +commands = py.test [testenv:py27] deps = -rrequirements-py2.txt From 959285742eeb6cf94c9e5a081e40e48daeb016d8 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sat, 2 May 2020 11:50:10 -0400 Subject: [PATCH 05/31] Add Github Workflows. --- .github/workflows/testing.yml | 36 +++++++++++++++++++++++++++++++++++ .travis.yml | 30 ----------------------------- requirements-tests.txt | 4 +++- setup.py | 6 ++---- tests/test_activity.py | 4 ++-- tests/test_task.py | 7 ++++--- 6 files changed, 47 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/testing.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 0000000..e1b1f27 --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,36 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Garcon / Pull Request / Tests + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-tests.txt + - name: Lint with flake8 + run: | + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + env: + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} + run: | + pytest --cov=. + coveralls diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 34b5e84..0000000 --- a/.travis.yml +++ /dev/null @@ -1,30 +0,0 @@ -language: python -matrix: - include: - - python: 2.7 - dist: trusty - sudo: false - - python: 3.3 - dist: trusty - sudo: false - - python: 3.4 - dist: trusty - sudo: false - - python: 3.5 - dist: trusty - sudo: false - - python: 3.6 - dist: trusty - sudo: false - - python: 3.7 - dist: xenial - sudo: true -env: - - BOTO_CONFIG=/tmp/nowhere -install: - - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt; fi - - "pip install -r requirements.txt" - - "pip install -r requirements-tests.txt" -script: "py.test tests/ --cov garcon --cov-report term-missing" -after_success: - - coveralls diff --git a/requirements-tests.txt b/requirements-tests.txt index fd809fd..ce57371 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,4 +1,6 @@ +flake8 +wheel pytest-cov==2.5.1 pytest==3.6.3 -python-coveralls==2.4.3 tox==2.9.1 +coveralls==1.10.0 diff --git a/setup.py b/setup.py index 9dc1ea5..0e7f1a2 100755 --- a/setup.py +++ b/setup.py @@ -15,8 +15,7 @@ url='https://github.com/xethorn/garcon/', author='Michael Ortali', author_email='hello@xethorn.net', - description=( - 'Lightweight library for AWS SWF.'), + description=('Lightweight library for AWS SWF.'), long_description=long_description, license='MIT', packages=find_packages(), @@ -24,9 +23,8 @@ install_requires=['boto', 'backoff'], zip_safe=False, classifiers=[ - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', ],) diff --git a/tests/test_activity.py b/tests/test_activity.py index ec9c92a..da768fd 100755 --- a/tests/test_activity.py +++ b/tests/test_activity.py @@ -316,8 +316,8 @@ def test_create_activity(monkeypatch, boto_client): assert isinstance(current_activity, activity.Activity) assert current_activity.name == 'flow_name_activity_name' assert current_activity.task_list == 'flow_name_activity_name' - assert current_activity.domain is 'domain_name' - assert current_activity.client is boto_client + assert current_activity.domain == 'domain_name' + assert current_activity.client == boto_client def test_create_external_activity(monkeypatch, boto_client): diff --git a/tests/test_task.py b/tests/test_task.py index 7cb9ba6..0c3da09 100755 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -64,7 +64,7 @@ def test(): assert test.__garcon__.get('foo') is None task._decorate(test, 'foo', 'bar') - assert test.__garcon__.get('foo') is 'bar' + assert test.__garcon__.get('foo') == 'bar' def test_generator_decorator(): @@ -108,7 +108,7 @@ def testB(): task._link_decorator(testA, testB) assert not getattr(testA, '__garcon__', None) - assert len(testB.__garcon__) is 0 + assert len(testB.__garcon__) == 0 task._decorate(testB, 'foo', 'value') assert testB.__garcon__.get('foo') == 'value' @@ -123,7 +123,7 @@ def test_task_decorator(): @task.decorate(timeout=timeout) def test(user): - assert user is userinfo + assert user == userinfo assert test.__garcon__.get('timeout') == timeout assert test.__garcon__.get('heartbeat') == timeout @@ -153,6 +153,7 @@ def test_task_decorator_with_heartbeat(): """ heartbeat = 50 + userinfo = None @task.decorate(heartbeat=heartbeat) def test(user): From f1633ab9d33aa5950445f8671c32eef665a1c207 Mon Sep 17 00:00:00 2001 From: Volodymyr Korniichuk Date: Wed, 6 May 2020 15:33:52 +0300 Subject: [PATCH 06/31] Add on_exception to run() activity method if fail() method fails (#90) Add on_exception to run() activity method if fail() method fails --- garcon/activity.py | 5 +++-- tests/test_activity.py | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/garcon/activity.py b/garcon/activity.py index 1921980..04da141 100755 --- a/garcon/activity.py +++ b/garcon/activity.py @@ -337,8 +337,9 @@ def run(self, identity=None): execution.fail(str(error)[:255]) if self.on_exception: self.on_exception(self, error) - except: - pass + except Exception as error2: # noqa: E722 + if self.on_exception: + self.on_exception(self, error2) self.unset_log_context() return True diff --git a/tests/test_activity.py b/tests/test_activity.py index da768fd..1d8ab21 100755 --- a/tests/test_activity.py +++ b/tests/test_activity.py @@ -177,6 +177,27 @@ def test_run_capture_exception(monkeypatch, poll, boto_client): assert not boto_client.respond_activity_task_completed.called +def test_run_capture_fail_exception(monkeypatch, poll, boto_client): + """Run an activity with an exception raised during failing execution. + """ + + current_activity = activity_run(monkeypatch, boto_client, poll) + current_activity.on_exception = MagicMock() + current_activity.execute_activity = MagicMock() + current_activity.complete = MagicMock() + current_activity.fail = MagicMock() + error_msg_long = "Error" * 100 + current_activity.complete.side_effect = Exception(error_msg_long) + current_activity.fail.side_effect = Exception(error_msg_long) + current_activity.run() + + assert boto_client.poll_for_activity_task.called + assert current_activity.execute_activity.called + assert not current_activity.complete.called + assert not current_activity.fail.called + assert current_activity.on_exception.called + + def test_run_capture_poll_exception(monkeypatch, boto_client, poll): """Run an activity with an exception raised during poll. """ @@ -249,6 +270,7 @@ def test_task_failure(monkeypatch, boto_client, poll): reason = 'fail' current_activity = activity_run(monkeypatch, boto_client, poll=poll, execute=mock) + current_activity.on_exception = MagicMock() current_activity.execute_activity.side_effect = Exception(reason) current_activity.run() @@ -265,6 +287,7 @@ def test_task_failure_on_close_activity(monkeypatch, boto_client, poll): mock = MagicMock(return_value=resp) current_activity = activity_run(monkeypatch, boto_client, poll=poll, execute=mock) + current_activity.on_exception = MagicMock() current_activity.execute_activity.side_effect = Exception('fail') boto_client.respond_activity_task_failed.side_effect = Exception('fail') current_activity.unset_log_context = MagicMock() From dee1b3d42b47cb0279698f40af5c65f976ac9941 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sat, 9 May 2020 15:43:40 -0400 Subject: [PATCH 07/31] Add workflow to publish to pypi. --- .github/workflows/publish.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..b7535ce --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,28 @@ +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* \ No newline at end of file From 27138fe23f4e17312be3a3657a638f11fdd0cc7d Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sat, 9 May 2020 15:47:18 -0400 Subject: [PATCH 08/31] Create 1.0.0 alpha release --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0e7f1a2..0fc5979 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='Garcon', - version='0.4.0', + version='1.0.0a1', url='https://github.com/xethorn/garcon/', author='Michael Ortali', author_email='hello@xethorn.net', From e5db053ae5cd2391271bc417024c1a3be579a2b7 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sun, 10 May 2020 11:55:54 -0400 Subject: [PATCH 09/31] Remove support for python 2.7. --- README.rst | 6 ++---- docs/index.rst | 4 +--- requirements-py2.txt | 3 --- tests/fixtures/flows/example.py | 2 -- tests/test_activity.py | 15 +++++---------- tests/test_context.py | 6 +----- tests/test_decider.py | 7 ++----- tests/test_runner.py | 6 +----- tests/test_task.py | 6 +----- 9 files changed, 13 insertions(+), 42 deletions(-) delete mode 100755 requirements-py2.txt diff --git a/README.rst b/README.rst index 5a26689..7c30d0c 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,7 @@ Lightweight library for AWS SWF. Requirements ~~~~~~~~~~~~ -- Python 2.7, 3.4, 3.5, 3.6, 3.7 (tested) +- Python 3.5, 3.6, 3.7, 3.8 (tested) - Boto3 (tested) Goal @@ -33,8 +33,6 @@ be completed. .. code:: python - from __future__ import print_function - import boto3 from garcon import activity from garcon import runner @@ -87,7 +85,7 @@ Application architecture Contributors ~~~~~~~~~~~~ -- Michael Ortali +- Michael Ortali (Author) - Adam Griffiths - Raphael Antonmattei - John Penner diff --git a/docs/index.rst b/docs/index.rst index d8aa0cc..8c9ed93 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,7 +6,7 @@ Lightweight library for AWS SWF. Requirements ------------ -* Python 2.7, 3.4 (tested) +* Python 3.5, 3.6, 3.7, 3.8 (tested) * Boto 2.34.0 (tested) @@ -20,8 +20,6 @@ Code sample The code sample shows a workflow that has 4 activities. It starts with activity_1, which after being completed schedule activity_2 and activity_3 to be ran in parallel. The workflow ends after the completion of activity_4 which requires activity_2 and activity_3 to be completed:: - from __future__ import print_function - from garcon import activity from garcon import runner diff --git a/requirements-py2.txt b/requirements-py2.txt deleted file mode 100755 index 6dc9995..0000000 --- a/requirements-py2.txt +++ /dev/null @@ -1,3 +0,0 @@ --r requirements.txt -mock==1.0.1 -futures==2.2.0 \ No newline at end of file diff --git a/tests/fixtures/flows/example.py b/tests/fixtures/flows/example.py index d3c37d8..a349072 100755 --- a/tests/fixtures/flows/example.py +++ b/tests/fixtures/flows/example.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from garcon import activity from garcon import runner diff --git a/tests/test_activity.py b/tests/test_activity.py index 1d8ab21..f0bcb99 100755 --- a/tests/test_activity.py +++ b/tests/test_activity.py @@ -1,16 +1,11 @@ -from __future__ import absolute_import -from __future__ import print_function -try: - from unittest.mock import MagicMock - from unittest.mock import ANY -except: - from mock import MagicMock - from mock import ANY -from botocore import exceptions +from unittest.mock import MagicMock +from unittest.mock import ANY import json -import pytest import sys +from botocore import exceptions +import pytest + from garcon import activity from garcon import event from garcon import runner diff --git a/tests/test_context.py b/tests/test_context.py index 7fe9f23..8e95698 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,8 +1,4 @@ -from __future__ import absolute_import -try: - from unittest.mock import MagicMock -except: - from mock import MagicMock +from unittest.mock import MagicMock from garcon import context from tests.fixtures import decider as decider_events diff --git a/tests/test_decider.py b/tests/test_decider.py index 85df245..5ad1076 100755 --- a/tests/test_decider.py +++ b/tests/test_decider.py @@ -1,9 +1,6 @@ -from __future__ import absolute_import -try: - from unittest.mock import MagicMock -except: - from mock import MagicMock +from unittest.mock import MagicMock import json + import pytest from garcon import decider diff --git a/tests/test_runner.py b/tests/test_runner.py index c2bc4cc..8f0d42e 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,8 +1,4 @@ -from __future__ import absolute_import -try: - from unittest.mock import MagicMock -except: - from mock import MagicMock +from unittest.mock import MagicMock import pytest diff --git a/tests/test_task.py b/tests/test_task.py index 0c3da09..b060685 100755 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -1,9 +1,5 @@ -from __future__ import absolute_import import functools -try: - from unittest.mock import MagicMock -except: - from mock import MagicMock +from unittest.mock import MagicMock import pytest From 5ae3839ee7fa334e98d7abf918044545f5082179 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sun, 10 May 2020 11:59:34 -0400 Subject: [PATCH 10/31] Issue 1.0.0 Alpha 2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0fc5979..ec84972 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='Garcon', - version='1.0.0a1', + version='1.0.0a2', url='https://github.com/xethorn/garcon/', author='Michael Ortali', author_email='hello@xethorn.net', From 784ec3593f2c29369bc06e35b1e9606a871be598 Mon Sep 17 00:00:00 2001 From: Vkorniychuk Date: Tue, 9 Jun 2020 20:22:24 +0300 Subject: [PATCH 11/31] Added log.GarconLogger as ancestor for ActivityExecution, fixed heartbeat argument --- garcon/activity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/garcon/activity.py b/garcon/activity.py index 04da141..b64560b 100755 --- a/garcon/activity.py +++ b/garcon/activity.py @@ -462,7 +462,7 @@ def run(self): return False -class ActivityExecution: +class ActivityExecution(log.GarconLogger): def __init__(self, client, activity_id, task_token, context): """Create an an activity execution. @@ -486,7 +486,7 @@ def heartbeat(self, details=None): details (str): details to add to the heartbeat. """ - self.client.record_activity_task_heartbeat(self.task_token, + self.client.record_activity_task_heartbeat(taskToken=self.task_token, details=details or '') def fail(self, reason=None): From 39f4297e8eb5d7662d4927089ef6c97190b63618 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Tue, 9 Jun 2020 18:12:45 -0400 Subject: [PATCH 12/31] Update testing.yml Adding support for pull requests. --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e1b1f27..74d6bcb 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -3,7 +3,7 @@ name: Garcon / Pull Request / Tests -on: [push] +on: [push, pull_request] jobs: build: From c46e85e1557eeae80948981c1329f6f3f777a51c Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Tue, 9 Jun 2020 18:16:46 -0400 Subject: [PATCH 13/31] Update setup.py Bump alpha version to include #96. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ec84972..84b0ea6 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='Garcon', - version='1.0.0a2', + version='1.0.0a3', url='https://github.com/xethorn/garcon/', author='Michael Ortali', author_email='hello@xethorn.net', From 8464ba71efdc9e9b2d6369fa3092adf617e0b739 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Tue, 9 Jun 2020 18:16:47 -0400 Subject: [PATCH 14/31] Update setup.py Bump alpha version to include #96. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ec84972..84b0ea6 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='Garcon', - version='1.0.0a2', + version='1.0.0a3', url='https://github.com/xethorn/garcon/', author='Michael Ortali', author_email='hello@xethorn.net', From 40f7e9d0dd4908d060df90b6b40093f3f30285e2 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Fri, 12 Jun 2020 11:25:48 -0400 Subject: [PATCH 15/31] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 84b0ea6..14f2436 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='Garcon', - version='1.0.0a3', + version='1.0.0', url='https://github.com/xethorn/garcon/', author='Michael Ortali', author_email='hello@xethorn.net', From 1d446b46acc0650a04aeefd30ee06741fae218b6 Mon Sep 17 00:00:00 2001 From: Vkorniychuk Date: Fri, 19 Jun 2020 15:04:06 +0300 Subject: [PATCH 16/31] Fix for decider param --- garcon/decider.py | 2 +- tests/test_decider.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/garcon/decider.py b/garcon/decider.py index 30eda46..51c2b6f 100755 --- a/garcon/decider.py +++ b/garcon/decider.py @@ -56,7 +56,7 @@ def get_history(self, identity, poll): domain=self.domain, identity=identity, taskList=dict(name=self.task_list), - next_page_token=poll['nextPageToken']) + nextPageToken=poll['nextPageToken']) if 'events' in poll: events += poll['events'] diff --git a/tests/test_decider.py b/tests/test_decider.py index 5ad1076..20baf59 100755 --- a/tests/test_decider.py +++ b/tests/test_decider.py @@ -51,7 +51,7 @@ def test_get_history(monkeypatch): d.client.poll_for_decision_task.assert_called_with( domain=example.domain, - next_page_token='nextPage', + nextPageToken='nextPage', identity=identity, taskList=dict(name=d.task_list)) assert len(resp) == len([ From 1b1a490fa89d0b862c1875fa9bb7c7ffc6a21c00 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Fri, 19 Jun 2020 08:15:38 -0400 Subject: [PATCH 17/31] v1.0.1 Fix decider next page token. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 14f2436..ce69e85 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='Garcon', - version='1.0.0', + version='1.0.1', url='https://github.com/xethorn/garcon/', author='Michael Ortali', author_email='hello@xethorn.net', From 52395dc9cc1209e9cd52786358ff055c7b85207d Mon Sep 17 00:00:00 2001 From: Vkorniychuk Date: Fri, 19 Jun 2020 19:36:35 +0300 Subject: [PATCH 18/31] Fixed another decider param --- garcon/decider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/garcon/decider.py b/garcon/decider.py index 51c2b6f..b8aeb30 100755 --- a/garcon/decider.py +++ b/garcon/decider.py @@ -238,7 +238,7 @@ def run(self, identity=None): if 'events' not in poll: return True - history = self.get_history(identity, poll) + history = self.get_history(identity or '', poll) activity_states = self.get_activity_states(history) current_context = event.get_current_context(history) current_context.set_workflow_execution_info(poll, self.domain) From 586eebec865637265fe107859765154b343c5082 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sat, 20 Jun 2020 11:02:05 -0400 Subject: [PATCH 19/31] Update setup.py Fixes: * #102 : fix: retrieving history for a given identity only accepts strings, `None` was provided. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ce69e85..0045f7b 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='Garcon', - version='1.0.1', + version='1.0.2', url='https://github.com/xethorn/garcon/', author='Michael Ortali', author_email='hello@xethorn.net', From 80a3ef565b7883a3f04385cc011b4d345b43b908 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Tue, 5 Jan 2021 15:44:40 -0500 Subject: [PATCH 20/31] Update testing.yml --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 74d6bcb..ada1e1e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Garcon / Pull Request / Tests +name: Build on: [push, pull_request] From e44db3080e34ff102bfe137f05980ade4342cafb Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Tue, 5 Jan 2021 15:45:24 -0500 Subject: [PATCH 21/31] Update README.rst --- README.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 7c30d0c..112dcb0 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -|Build Status| |Coverage Status| +|BuildStatus| |Downloads| |CoverageStatus| Lightweight library for AWS SWF. @@ -94,7 +94,11 @@ Contributors .. _rantonmattei: github.com/rantonmattei .. _someboredkiddo: github.com/someboredkiddo -.. |Build Status| image:: https://travis-ci.org/xethorn/garcon.svg - :target: https://travis-ci.org/xethorn/garcon -.. |Coverage Status| image:: https://coveralls.io/repos/xethorn/garcon/badge.svg?branch=master +.. |BuildStatus| image:: https://github.com/xethorn/garcon/workflows/Build/badge.svg + :target: https://github.com/xethorn/garcon/actions?query=workflow%3ABuild+branch%3Amaster + +.. |Downloads| image:: https://img.shields.io/pypi/dm/garcon.svg + :target: https://coveralls.io/r/xethorn/garcon?branch=master + +.. |CoverageStatus| image:: https://coveralls.io/repos/xethorn/garcon/badge.svg?branch=master :target: https://coveralls.io/r/xethorn/garcon?branch=master From ece65a3ea3dfadbf8ac7a909e7c242e6da287944 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sun, 2 May 2021 09:59:23 -0400 Subject: [PATCH 22/31] Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..b9e55a0 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '39 1 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 5bd4b3b1b98f132251ae4ca0f1f0608e6294d567 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sun, 9 May 2021 16:42:06 -0400 Subject: [PATCH 23/31] Update example in README --- README.rst | 80 +++++++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/README.rst b/README.rst index 112dcb0..9812f6d 100644 --- a/README.rst +++ b/README.rst @@ -25,47 +25,38 @@ case, you might want to use directly boto. Code sample ~~~~~~~~~~~ -The code sample shows a workflow that has 4 activities. It starts with -activity\_1, which after being completed schedule activity\_2 and -activity\_3 to be ran in parallel. The workflow ends after the -completion of activity\_4 which requires activity\_2 and activity\_3 to -be completed. +The code sample shows a workflow where a user enters a coffee shop, orders +a coffee and a chocolate chip cookie. All ordered items are prepared and +completed, the user pays the order, receives the ordered items, then leave +the shop. -.. code:: python - - import boto3 - from garcon import activity - from garcon import runner - - client = boto3.client('swf', region_name='us-east-1') - - domain = 'dev' - name = 'workflow_sample' - create = activity.create(client, domain, name) +The code below represents the workflow decider. For the full code sample, +see the `example`_. - test_activity_1 = create( - name='activity_1', - run=runner.Sync( - lambda activity, context: print('activity_1'))) - - test_activity_2 = create( - name='activity_2', - requires=[test_activity_1], - run=runner.Async( - lambda activity, context: print('activity_2_task_1'), - lambda activity, context: print('activity_2_task_2'))) +.. code:: python - test_activity_3 = create( - name='activity_3', - requires=[test_activity_1], - run=runner.Sync( - lambda activity, context: print('activity_3'))) + enter = schedule('enter', self.create_enter_coffee_activity) + enter.wait() + + total = 0 + for item in ['coffee', 'chocolate_chip_cookie']: + activity_name = 'order_{item}'.format(item=item) + activity = schedule(activity_name, + self.create_order_activity, + input={'item': item}) + total += activity.result.get('price') + + pay_activity = schedule( + 'pay', self.create_payment_activity, + input={'total': total}) + + get_order = schedule('get_order', self.create_get_order_activity) + + # Waiting for paying and getting the order to complete before + # we let the user leave the coffee shop. + pay_activity.wait(), get_order.wait() + schedule('leave_coffee_shop', self.create_leave_coffee_shop) - test_activity_4 = create( - name='activity_4', - requires=[test_activity_3, test_activity_2], - run=runner.Sync( - lambda activity, context: print('activity_4'))) Application architecture ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -82,6 +73,11 @@ Application architecture │ └── s3.py # Task that focuses on s3 files. └── task_example.py # Your different tasks. +Trusted by +~~~~~~~~~~ + +|The Orchard| |Sony Music| |DataArt| + Contributors ~~~~~~~~~~~~ @@ -93,6 +89,7 @@ Contributors .. _xethorn: github.com/xethorn .. _rantonmattei: github.com/rantonmattei .. _someboredkiddo: github.com/someboredkiddo +.. _example: https://github.com/xethorn/garcon/tree/master/example/custom_decider .. |BuildStatus| image:: https://github.com/xethorn/garcon/workflows/Build/badge.svg :target: https://github.com/xethorn/garcon/actions?query=workflow%3ABuild+branch%3Amaster @@ -102,3 +99,12 @@ Contributors .. |CoverageStatus| image:: https://coveralls.io/repos/xethorn/garcon/badge.svg?branch=master :target: https://coveralls.io/r/xethorn/garcon?branch=master + +.. |The Orchard| image:: https://media-exp1.licdn.com/dms/image/C4E0BAQGi7o5g9l4JWg/company-logo_200_200/0/1519855981606?e=2159024400&v=beta&t=WBe-gOK2b30vUTGKbA025i9NFVDyOrS4Fotx9fMEZWo + :target: https://theorchard.com + +.. |Sony Music| image:: https://media-exp1.licdn.com/dms/image/C4D0BAQE9rvU-3ig-jg/company-logo_200_200/0/1604099587507?e=2159024400&v=beta&t=eAAubphf_fI-5GEb0ak1QnmtRHmc8466Qj4sGrCsWYc + :target: https://www.sonymusic.com/ + +.. |DataArt| image:: https://media-exp1.licdn.com/dms/image/C4E0BAQGRi6OIlNQG8Q/company-logo_200_200/0/1519856519357?e=2159024400&v=beta&t=oi6HQpzoeTKA082s-8Ft75vGTvAkEp4VHRyMLeOHXoo + :target: https://www.dataart.com/ From 7b7b203fac57864bb528dd7cfce98a718c7c2df7 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Tue, 10 Sep 2024 10:39:16 -0400 Subject: [PATCH 24/31] Update testing.yml Remove coveralls --- .github/workflows/testing.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ada1e1e..33870e1 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -32,5 +32,4 @@ jobs: env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} run: | - pytest --cov=. - coveralls + pytest From f485e369a2bbb5b29e91fc140b263ac91fd6fb30 Mon Sep 17 00:00:00 2001 From: Egor Fedorov Date: Tue, 10 Sep 2024 20:50:30 +0700 Subject: [PATCH 25/31] Bump requirements versions, tested with actual environments --- .github/workflows/testing.yml | 2 +- README.rst | 4 ++-- requirements-tests.txt | 8 ++++---- requirements.txt | 4 ++-- setup.py | 11 ++++++----- tox.ini | 5 +---- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ada1e1e..d994c47 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v2 diff --git a/README.rst b/README.rst index 9812f6d..c740a12 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,7 @@ Lightweight library for AWS SWF. Requirements ~~~~~~~~~~~~ -- Python 3.5, 3.6, 3.7, 3.8 (tested) +- Python 3.8, 3.9, 3.10, 3.11, 3.12 (tested) - Boto3 (tested) Goal @@ -20,7 +20,7 @@ The goal of this library is to allow the creation of Amazon Simple Workflow without the need to worry about the orchestration of the different activities and building out the different workers. This framework aims to help simple workflows. If you have a more complex -case, you might want to use directly boto. +case, you might want to use directly boto3. Code sample ~~~~~~~~~~~ diff --git a/requirements-tests.txt b/requirements-tests.txt index ce57371..0ded7db 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,6 +1,6 @@ flake8 wheel -pytest-cov==2.5.1 -pytest==3.6.3 -tox==2.9.1 -coveralls==1.10.0 +pytest-cov==5.0.0 +pytest==8.3.2 +tox==4.18.1 +coveralls==4.0.0 diff --git a/requirements.txt b/requirements.txt index 2ab4b98..d3bd936 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -backoff==1.4.3 -boto3==1.7.58 +backoff==2.2.1 +boto3==1.35.0 diff --git a/setup.py b/setup.py index 0045f7b..494a901 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='Garcon', - version='1.0.2', + version='1.0.3', url='https://github.com/xethorn/garcon/', author='Michael Ortali', author_email='hello@xethorn.net', @@ -20,11 +20,12 @@ license='MIT', packages=find_packages(), include_package_data=True, - install_requires=['boto', 'backoff'], + install_requires=['boto3', 'backoff'], zip_safe=False, classifiers=[ - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ],) diff --git a/tox.ini b/tox.ini index cb40329..2c7d2b4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,9 @@ [tox] -envlist = py27,py34,py35,py36,py37 +envlist = py38,py39,py310,py311,py312 [testenv] deps = -rrequirements-tests.txt commands = py.test -[testenv:py27] -deps = -rrequirements-py2.txt - [testenv:py3] deps = -rrequirements.txt From 173525e1a9cc859e7573bac8c5ba00b1dda00f06 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sat, 14 Sep 2024 18:06:26 -0400 Subject: [PATCH 26/31] Update publish.yml --- .github/workflows/publish.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b7535ce..0f328ef 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,10 +19,8 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + - name: Build run: | python setup.py sdist bdist_wheel - twine upload dist/* \ No newline at end of file + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 From 1b0591dc4d5455dadd388c41dfa5c556588a1642 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sat, 14 Sep 2024 18:09:17 -0400 Subject: [PATCH 27/31] Update publish.yml --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0f328ef..7fedacb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,9 +6,9 @@ on: jobs: deploy: - runs-on: ubuntu-latest - + permissions: + id-token: write steps: - uses: actions/checkout@v2 - name: Set up Python From 5144c4f25a5b7f257ac5a2369f9fdf9e62df3c83 Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Sat, 14 Sep 2024 18:20:44 -0400 Subject: [PATCH 28/31] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 494a901..95a26c4 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='Garcon', - version='1.0.3', + version='1.1.0', url='https://github.com/xethorn/garcon/', author='Michael Ortali', author_email='hello@xethorn.net', From aea0b534097dd4881aafb9e929cbc846febad846 Mon Sep 17 00:00:00 2001 From: Egor Fedorov Date: Thu, 26 Sep 2024 03:14:14 +0700 Subject: [PATCH 29/31] fix threading --- garcon/activity.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/garcon/activity.py b/garcon/activity.py index b64560b..772f164 100755 --- a/garcon/activity.py +++ b/garcon/activity.py @@ -537,13 +537,19 @@ def __init__(self, flow, activities=None): def run(self): """Run the activities. """ - + threads = [] for activity in self.activities: if (self.worker_activities and activity.name not in self.worker_activities): continue - threading.Thread(target=worker_runner, args=(activity,)).start() - + thread = threading.Thread( + target=worker_runner, + args=(activity,)) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() class ActivityState: """ From aa3d2b34bf95dbabfded86cc219d1c8296051f57 Mon Sep 17 00:00:00 2001 From: Egor Fedorov Date: Thu, 26 Sep 2024 10:31:49 +0700 Subject: [PATCH 30/31] fix threading --- garcon/activity.py | 1 + 1 file changed, 1 insertion(+) diff --git a/garcon/activity.py b/garcon/activity.py index 772f164..b9fe701 100755 --- a/garcon/activity.py +++ b/garcon/activity.py @@ -551,6 +551,7 @@ def run(self): for thread in threads: thread.join() + class ActivityState: """ Activity State From a21c89eae175e89c6deade06c420e5ca5dc357fa Mon Sep 17 00:00:00 2001 From: Michael Ortali Date: Thu, 26 Sep 2024 05:09:52 -0400 Subject: [PATCH 31/31] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 95a26c4..cd03f1b 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='Garcon', - version='1.1.0', + version='1.1.1', url='https://github.com/xethorn/garcon/', author='Michael Ortali', author_email='hello@xethorn.net',