From ce63b2c3ccb126bf299b8c4069ccf676e7e04e4a Mon Sep 17 00:00:00 2001 From: Roman Zlobin Date: Wed, 20 Dec 2023 00:54:34 +0300 Subject: [PATCH 1/2] replace forwardref with type_checking --- dff/__init__.py | 5 ++++ dff/pipeline/__init__.py | 2 ++ dff/pipeline/conditions.py | 6 ++-- dff/pipeline/pipeline/actor.py | 6 ++-- dff/pipeline/pipeline/component.py | 6 ++-- dff/pipeline/service/extra.py | 6 ++-- dff/pipeline/service/group.py | 6 ++-- dff/pipeline/service/service.py | 6 ++-- dff/pipeline/types.py | 46 ++++++++++++++---------------- dff/script/core/context.py | 13 ++++----- dff/script/core/normalization.py | 14 ++++----- dff/script/core/script.py | 17 ++++++----- dff/script/labels/std_labels.py | 6 ++-- 13 files changed, 77 insertions(+), 62 deletions(-) diff --git a/dff/__init__.py b/dff/__init__.py index cdc329558..f3cfbd963 100644 --- a/dff/__init__.py +++ b/dff/__init__.py @@ -9,3 +9,8 @@ import nest_asyncio nest_asyncio.apply() + +from dff.pipeline import Pipeline +from dff.script import Context, Script + +Script.model_rebuild() diff --git a/dff/pipeline/__init__.py b/dff/pipeline/__init__.py index 95f85e82b..1b345f647 100644 --- a/dff/pipeline/__init__.py +++ b/dff/pipeline/__init__.py @@ -32,3 +32,5 @@ from .service.extra import BeforeHandler, AfterHandler from .service.group import ServiceGroup from .service.service import Service, to_service + +ExtraHandlerRuntimeInfo.model_rebuild() diff --git a/dff/pipeline/conditions.py b/dff/pipeline/conditions.py index fe7e65ce5..6b10d7099 100644 --- a/dff/pipeline/conditions.py +++ b/dff/pipeline/conditions.py @@ -5,7 +5,8 @@ are attached should be executed or not. The standard set of them allows user to setup dependencies between pipeline components. """ -from typing import Optional, ForwardRef +from __future__ import annotations +from typing import Optional, TYPE_CHECKING from dff.script import Context @@ -16,7 +17,8 @@ StartConditionCheckerAggregationFunction, ) -Pipeline = ForwardRef("Pipeline") +if TYPE_CHECKING: + from dff.pipeline.pipeline.pipeline import Pipeline def always_start_condition(_: Context, __: Pipeline) -> bool: diff --git a/dff/pipeline/pipeline/actor.py b/dff/pipeline/pipeline/actor.py index c567c554a..f62daabbf 100644 --- a/dff/pipeline/pipeline/actor.py +++ b/dff/pipeline/pipeline/actor.py @@ -22,9 +22,10 @@ .. figure:: /_static/drawio/dfe/user_actor.png """ +from __future__ import annotations import logging import asyncio -from typing import Union, Callable, Optional, Dict, List, ForwardRef +from typing import Union, Callable, Optional, Dict, List, TYPE_CHECKING import copy from dff.utils.turn_caching import cache_clear @@ -39,7 +40,8 @@ logger = logging.getLogger(__name__) -Pipeline = ForwardRef("Pipeline") +if TYPE_CHECKING: + from dff.pipeline.pipeline.pipeline import Pipeline def error_handler(error_msgs: list, msg: str, exception: Optional[Exception] = None, logging_flag: bool = True): diff --git a/dff/pipeline/pipeline/component.py b/dff/pipeline/pipeline/component.py index d58ddb39d..587b39c80 100644 --- a/dff/pipeline/pipeline/component.py +++ b/dff/pipeline/pipeline/component.py @@ -8,11 +8,12 @@ The PipelineComponent class can be a group or a service. It is designed to be reusable and composable, allowing developers to create complex processing pipelines by combining multiple components. """ +from __future__ import annotations import logging import abc import asyncio import copy -from typing import Optional, Awaitable, ForwardRef +from typing import Optional, Awaitable, TYPE_CHECKING from dff.script import Context @@ -31,7 +32,8 @@ logger = logging.getLogger(__name__) -Pipeline = ForwardRef("Pipeline") +if TYPE_CHECKING: + from dff.pipeline.pipeline.pipeline import Pipeline class PipelineComponent(abc.ABC): diff --git a/dff/pipeline/service/extra.py b/dff/pipeline/service/extra.py index dc1242fba..1212743e6 100644 --- a/dff/pipeline/service/extra.py +++ b/dff/pipeline/service/extra.py @@ -5,10 +5,11 @@ beyond the core functionality. Extra handlers is an input converting addition to :py:class:`.PipelineComponent`. For example, it is used to grep statistics from components, timing, logging, etc. """ +from __future__ import annotations import asyncio import logging import inspect -from typing import Optional, List, ForwardRef +from typing import Optional, List, TYPE_CHECKING from dff.script import Context @@ -23,7 +24,8 @@ logger = logging.getLogger(__name__) -Pipeline = ForwardRef("Pipeline") +if TYPE_CHECKING: + from dff.pipeline.pipeline.pipeline import Pipeline class _ComponentExtraHandler: diff --git a/dff/pipeline/service/group.py b/dff/pipeline/service/group.py index 1f845cde1..2d977103a 100644 --- a/dff/pipeline/service/group.py +++ b/dff/pipeline/service/group.py @@ -7,9 +7,10 @@ allowing for easier management and organization of the services within the pipeline. The :py:class:`~.ServiceGroup` serves the important function of grouping services to work together in parallel. """ +from __future__ import annotations import asyncio import logging -from typing import Optional, List, Union, Awaitable, ForwardRef +from typing import Optional, List, Union, Awaitable, TYPE_CHECKING from dff.script import Context @@ -29,7 +30,8 @@ logger = logging.getLogger(__name__) -Pipeline = ForwardRef("Pipeline") +if TYPE_CHECKING: + from dff.pipeline.pipeline.pipeline import Pipeline class ServiceGroup(PipelineComponent): diff --git a/dff/pipeline/service/service.py b/dff/pipeline/service/service.py index 76787143e..0895380f8 100644 --- a/dff/pipeline/service/service.py +++ b/dff/pipeline/service/service.py @@ -9,9 +9,10 @@ Service can be asynchronous only if its handler is a coroutine. Actor wrapping service is asynchronous. """ +from __future__ import annotations import logging import inspect -from typing import Optional, ForwardRef +from typing import Optional, TYPE_CHECKING from dff.script import Context @@ -27,7 +28,8 @@ logger = logging.getLogger(__name__) -Pipeline = ForwardRef("Pipeline") +if TYPE_CHECKING: + from dff.pipeline.pipeline.pipeline import Pipeline class Service(PipelineComponent): diff --git a/dff/pipeline/types.py b/dff/pipeline/types.py index ef7bef9ae..a31038cc5 100644 --- a/dff/pipeline/types.py +++ b/dff/pipeline/types.py @@ -5,24 +5,22 @@ The classes and special types in this module can include data models, data structures, and other types that are defined for type hinting. """ +from __future__ import annotations from abc import ABC from enum import unique, Enum -from typing import Callable, Union, Awaitable, Dict, List, Optional, NewType, Iterable, Any, Protocol, Hashable +from typing import Callable, Union, Awaitable, Dict, List, Optional, Iterable, Any, Protocol, Hashable, TYPE_CHECKING from dff.context_storages import DBContextStorage from dff.script import Context, ActorStage, NodeLabel2Type, Script, Message from typing_extensions import NotRequired, TypedDict, TypeAlias from pydantic import BaseModel - -_ForwardPipeline = NewType("Pipeline", Any) -_ForwardPipelineComponent = NewType("PipelineComponent", Any) -_ForwardService = NewType("Service", _ForwardPipelineComponent) -_ForwardServiceBuilder = NewType("ServiceBuilder", Any) -_ForwardServiceGroup = NewType("ServiceGroup", _ForwardPipelineComponent) -_ForwardComponentExtraHandler = NewType("_ComponentExtraHandler", Any) -_ForwardProvider = NewType("ABCProvider", ABC) -_ForwardExtraHandlerRuntimeInfo = NewType("ExtraHandlerRuntimeInfo", Any) +if TYPE_CHECKING: + from dff.pipeline.pipeline.pipeline import Pipeline + from dff.pipeline.service.service import Service + from dff.pipeline.service.group import ServiceGroup + from dff.pipeline.service.extra import _ComponentExtraHandler + from dff.messengers.common.interface import MessengerInterface class PipelineRunnerFunction(Protocol): @@ -112,7 +110,7 @@ class ExtraHandlerType(str, Enum): """ -StartConditionCheckerFunction: TypeAlias = Callable[[Context, _ForwardPipeline], bool] +StartConditionCheckerFunction: TypeAlias = Callable[[Context, "Pipeline"], bool] """ A function type for components `start_conditions`. Accepts context and pipeline, returns boolean (whether service can be launched). @@ -152,8 +150,8 @@ class ServiceRuntimeInfo(BaseModel): ExtraHandlerFunction: TypeAlias = Union[ Callable[[Context], Any], - Callable[[Context, _ForwardPipeline], Any], - Callable[[Context, _ForwardPipeline, _ForwardExtraHandlerRuntimeInfo], Any], + Callable[[Context, "Pipeline"], Any], + Callable[[Context, "Pipeline", "ExtraHandlerRuntimeInfo"], Any], ] """ A function type for creating wrappers (before and after functions). @@ -177,10 +175,10 @@ class ExtraHandlerRuntimeInfo(BaseModel): ServiceFunction: TypeAlias = Union[ Callable[[Context], None], Callable[[Context], Awaitable[None]], - Callable[[Context, _ForwardPipeline], None], - Callable[[Context, _ForwardPipeline], Awaitable[None]], - Callable[[Context, _ForwardPipeline, ServiceRuntimeInfo], None], - Callable[[Context, _ForwardPipeline, ServiceRuntimeInfo], Awaitable[None]], + Callable[[Context, "Pipeline"], None], + Callable[[Context, "Pipeline"], Awaitable[None]], + Callable[[Context, "Pipeline", ServiceRuntimeInfo], None], + Callable[[Context, "Pipeline", ServiceRuntimeInfo], Awaitable[None]], ] """ A function type for creating service handlers. @@ -190,7 +188,7 @@ class ExtraHandlerRuntimeInfo(BaseModel): ExtraHandlerBuilder: TypeAlias = Union[ - _ForwardComponentExtraHandler, + "_ComponentExtraHandler", TypedDict( "WrapperDict", { @@ -205,19 +203,19 @@ class ExtraHandlerRuntimeInfo(BaseModel): A type, representing anything that can be transformed to ExtraHandlers. It can be: -- _ForwardComponentExtraHandler object +- ExtraHandlerFunction object - Dictionary, containing keys `timeout`, `asynchronous`, `functions` """ ServiceBuilder: TypeAlias = Union[ ServiceFunction, - _ForwardService, + "Service", str, TypedDict( "ServiceDict", { - "handler": _ForwardServiceBuilder, + "handler": "ServiceBuilder", "before_handler": NotRequired[Optional[ExtraHandlerBuilder]], "after_handler": NotRequired[Optional[ExtraHandlerBuilder]], "timeout": NotRequired[Optional[float]], @@ -239,8 +237,8 @@ class ExtraHandlerRuntimeInfo(BaseModel): ServiceGroupBuilder: TypeAlias = Union[ - List[Union[ServiceBuilder, List[ServiceBuilder], _ForwardServiceGroup]], - _ForwardServiceGroup, + List[Union[ServiceBuilder, List[ServiceBuilder], "ServiceGroup"]], + "ServiceGroup", ] """ A type, representing anything that can be transformed to service group. @@ -254,7 +252,7 @@ class ExtraHandlerRuntimeInfo(BaseModel): PipelineBuilder: TypeAlias = TypedDict( "PipelineBuilder", { - "messenger_interface": NotRequired[Optional[_ForwardProvider]], + "messenger_interface": NotRequired[Optional["MessengerInterface"]], "context_storage": NotRequired[Optional[Union[DBContextStorage, Dict]]], "components": ServiceGroupBuilder, "before_handler": NotRequired[Optional[ExtraHandlerBuilder]], diff --git a/dff/script/core/context.py b/dff/script/core/context.py index 730beec2f..849a0651d 100644 --- a/dff/script/core/context.py +++ b/dff/script/core/context.py @@ -16,18 +16,20 @@ The context can be easily serialized to a format that can be stored or transmitted, such as JSON. This allows developers to save the context data and resume the conversation later. """ +from __future__ import annotations import logging from uuid import UUID, uuid4 -from typing import Any, Optional, Union, Dict, List, Set +from typing import Any, Optional, Union, Dict, List, Set, TYPE_CHECKING from pydantic import BaseModel, Field, field_validator from .types import NodeLabel2Type, ModuleName from .message import Message -logger = logging.getLogger(__name__) +if TYPE_CHECKING: + from dff.script.core.script import Node -Node = BaseModel +logger = logging.getLogger(__name__) def get_last_index(dictionary: dict) -> int: @@ -120,7 +122,7 @@ def sort_dict_keys(cls, dictionary: dict) -> dict: return {key: dictionary[key] for key in sorted(dictionary)} @classmethod - def cast(cls, ctx: Optional[Union["Context", dict, str]] = None, *args, **kwargs) -> "Context": + def cast(cls, ctx: Optional[Union[Context, dict, str]] = None, *args, **kwargs) -> Context: """ Transform different data types to the objects of the :py:class:`~.Context` class. @@ -277,6 +279,3 @@ def current_node(self) -> Optional[Node]: ) return node - - -Context.model_rebuild() diff --git a/dff/script/core/normalization.py b/dff/script/core/normalization.py index ef9f75419..302b1f33b 100644 --- a/dff/script/core/normalization.py +++ b/dff/script/core/normalization.py @@ -5,21 +5,20 @@ that is suitable for script and actor execution process. This module contains a basic set of functions for normalizing data in a dialog script. """ +from __future__ import annotations import logging - -from typing import Union, Callable, Optional, ForwardRef +from typing import Union, Callable, Optional, TYPE_CHECKING from .keywords import Keywords from .context import Context from .types import NodeLabel3Type, NodeLabelType, ConditionType, LabelType from .message import Message -from pydantic import validate_call +if TYPE_CHECKING: + from dff.pipeline.pipeline.pipeline import Pipeline logger = logging.getLogger(__name__) -Pipeline = ForwardRef("Pipeline") - def normalize_label( label: NodeLabelType, default_flow_label: LabelType = "" @@ -83,10 +82,9 @@ def callable_condition_handler(ctx: Context, pipeline: Pipeline) -> bool: return callable_condition_handler -@validate_call def normalize_response( - response: Optional[Union[Message, Callable[[Context, Pipeline], Message]]] -) -> Callable[[Context, Pipeline], Message]: + response: Optional[Union[Message, Callable[[Context, "Pipeline"], Message]]] +) -> Callable[[Context, "Pipeline"], Message]: """ This function is used to normalize response. If the response is a Callable, it is returned, otherwise the response is wrapped in an asynchronous function and this function is returned. diff --git a/dff/script/core/script.py b/dff/script/core/script.py index 7896c415e..25c60cc5c 100644 --- a/dff/script/core/script.py +++ b/dff/script/core/script.py @@ -6,23 +6,22 @@ the user's input and the current state of the conversation. """ # %% - +from __future__ import annotations import logging -from typing import Callable, Optional, Any, Dict, Union +from typing import Callable, Optional, Any, Dict, Union, TYPE_CHECKING -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, field_validator, validate_call from .types import LabelType, NodeLabelType, ConditionType, NodeLabel3Type from .message import Message from .keywords import Keywords -from .normalization import normalize_condition, normalize_label, validate_call -from typing import ForwardRef - -logger = logging.getLogger(__name__) +from .normalization import normalize_condition, normalize_label +if TYPE_CHECKING: + from dff.script.core.context import Context + from dff.pipeline.pipeline.pipeline import Pipeline -Pipeline = ForwardRef("Pipeline") -Context = ForwardRef("Context") +logger = logging.getLogger(__name__) class Node(BaseModel, extra="forbid", validate_assignment=True): diff --git a/dff/script/labels/std_labels.py b/dff/script/labels/std_labels.py index 1c2c03322..7409fc21b 100644 --- a/dff/script/labels/std_labels.py +++ b/dff/script/labels/std_labels.py @@ -10,10 +10,12 @@ This module contains a standard set of scripting :py:const:`labels ` that can be used by developers to define the conversation flow. """ -from typing import Optional, Callable, ForwardRef +from __future__ import annotations +from typing import Optional, Callable, TYPE_CHECKING from dff.script import Context, NodeLabel3Type -Pipeline = ForwardRef("Pipeline") +if TYPE_CHECKING: + from dff.pipeline.pipeline.pipeline import Pipeline def repeat(priority: Optional[float] = None) -> Callable: From b9af077b55615d65d9d014b75eb6f12c9a0f7056 Mon Sep 17 00:00:00 2001 From: Roman Zlobin Date: Wed, 20 Dec 2023 12:08:17 +0300 Subject: [PATCH 2/2] codestyle --- dff/pipeline/types.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dff/pipeline/types.py b/dff/pipeline/types.py index a31038cc5..4409cc87b 100644 --- a/dff/pipeline/types.py +++ b/dff/pipeline/types.py @@ -6,7 +6,6 @@ data structures, and other types that are defined for type hinting. """ from __future__ import annotations -from abc import ABC from enum import unique, Enum from typing import Callable, Union, Awaitable, Dict, List, Optional, Iterable, Any, Protocol, Hashable, TYPE_CHECKING