Task Consumer¶
+The task consumer is a TaskManager which is also a Workers that consumes tasks from the task queue and executes them. It can be imported from fluid.scheduler:
from fastapi.scheduler import TaskConsumer
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+TaskConsumer(**config)
+
+
+
+
+ Bases: TaskManager, Workers
The Task Consumer is a Task Manager responsible for consuming tasks +from a task queue
+ + + + + + +
+ Source code in
+
+
+
+
+
+ Source code in fluid/scheduler/consumer.py
+ 163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 | |
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+instance-attribute
+
+
+¶
+config = TaskManagerConfig(**kwargs)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+property
+
+
+¶
+num_concurrent_tasks
+
+
+
+
+The number of concurrent_tasks running in the consumer
+
+
+
+
+
+
+
+
+
+
+async
+
+
+¶
+status()
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 288 +289 | |
+
+
+
+
+
+
+
+
+
+
+gracefully_stop()
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 285 +286 | |
+
+
+
+
+
+
+
+
+
+
+is_running()
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 79 +80 | |
+
+
+
+
+
+
+
+
+
+
+is_stopping()
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 275 +276 | |
+
+
+
+
+
+
+
+
+
+async
+
+
+¶
+run()
+
+
+
+
+run the workers
+ +
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 414 +415 +416 +417 +418 +419 +420 +421 +422 +423 +424 | |
+
+
+
+
+
+
+
+
+
+
+start_running()
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 | |
+
+
+
+
+
+
+
+
+
+
+add_workers(*workers)
+
+
+
+
+add workers to the workers
+ +
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 407 +408 +409 +410 +411 +412 | |
+
+
+
+
+
+
+
+
+
+async
+
+
+¶
+wait_for_exit()
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 426 +427 +428 | |
+
+
+
+
+
+
+
+
+
+
+create_task(worker)
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 291 +292 +293 +294 | |
+
+
+
+
+
+
+
+
+
+async
+
+
+¶
+on_shutdown()
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 82 +83 | |
+
+
+
+
+
+
+
+
+
+async
+
+
+¶
+shutdown()
+
+
+
+
+shutdown the workers
+ +
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 | |
+
+
+
+
+
+
+
+
+
+
+bail_out(reason, code=1)
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 334 +335 | |
+
+
+
+
+
+
+
+
+
+async
+
+
+¶
+safe_run()
+
+
+
+
+Context manager to run a worker safely
+ +
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 | |
+
+
+
+
+
+
+
+
+
+
+remove_workers(*workers)
+
+
+
+
+remove workers from the workers
+ +
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 430 +431 +432 +433 +434 +435 +436 +437 | |
+
+
+
+
+
+
+
+
+
+async
+
+
+¶
+startup()
+
+
+
+
+start the workers
+ +
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 439 +440 +441 +442 +443 +444 +445 | |
+
+
+
+
+
+
+
+
+
+
+register_callback(
+ callback, seconds, jitter=0.0, periodic=False
+)
+
+
+
+
+Register a callback
+The callback can be periodic or not.
+ +
+ Source code in
+
+
+ Source code in fluid/utils/worker.py
+ 447 +448 +449 +450 +451 +452 +453 +454 +455 +456 +457 +458 +459 +460 +461 +462 +463 +464 +465 +466 +467 | |
+
+
+
+
+
+
+
+
+
+async
+
+
+¶
+enter_async_context(cm)
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 65 +66 | |
+
+
+
+
+
+
+
+
+
+async
+
+
+¶
+execute(task, **params)
+
+
+
+
+Execute a task and wait for it to finish
+ +
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 76 +77 +78 +79 +80 | |
+
+
+
+
+
+
+
+
+
+
+execute_sync(task, **params)
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 85 +86 +87 +88 | |
+
+
+
+
+
+
+
+
+
+
+register_task(task)
+
+
+
+
+Register a task with the task manager
+Only tasks registered can be executed by a task manager
+ +
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 90 +91 +92 +93 +94 +95 | |
+
+
+
+
+
+
+
+
+
+async
+
+
+¶
+queue(task, priority=None, **params)
+
+
+
+
+Queue a task for execution
+This methods fires two events:
+-
+
- queue: when the task is about to be queued +
- queued: after the task is queued +
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 97 + 98 + 99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 | |
+
+
+
+
+
+
+
+
+
+
+create_task_run(task, run_id='', priority=None, **params)
+
+
+
+
+Create a TaskRun in init state
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 | |
+
+
+
+
+
+
+
+
+
+
+register_from_module(module)
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 135 +136 +137 +138 +139 +140 | |
+
+
+
+
+
+
+
+
+
+
+cli(**kwargs)
+
+
+
+
+Create the task manager command line interface
+ +
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 142 +143 +144 +145 +146 +147 +148 +149 +150 +151 | |
+
+
+
+
+
+
+
+
+
+
+sync_queue(task)
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 186 +187 | |
+
+
+
+
+
+
+
+
+
+
+sync_priority_queue(task)
+
+
+
+
+
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 189 +190 | |
+
+
+
+
+
+
+
+
+
+
+num_concurrent_tasks_for(task_name)
+
+
+
+
+The number of concurrent tasks for a given task_name
+ +
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 192 +193 +194 | |
+
+
+
+
+
+
+
+
+
+
+
+async
+
+
+¶
+queue_and_wait(task, *, timeout=2, **params)
+
+
+
+
+Queue a task and wait for it to finish
+ +
+ Source code in
+
+
+ Source code in fluid/scheduler/consumer.py
+ 196 +197 +198 +199 +200 +201 | |