diff --git a/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py b/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py index 7cfbf13df11..f8efe0f321b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py +++ b/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py @@ -82,21 +82,19 @@ def _is_model_cls(obj) -> bool: for model_name, model_cls in inspect.getmembers(module, _is_model_cls): assert model_name # nosec if ( - (config_cls := model_cls.Config) - and inspect.isclass(config_cls) - and is_strict_inner(model_cls, config_cls) - and (schema_extra := getattr(config_cls, "schema_extra", {})) - and isinstance(schema_extra, dict) + (config_dict := model_cls.model_config) + and (json_schema_extra := config_dict.get("json_schema_extra", {})) + and isinstance(json_schema_extra, dict) ): - if "example" in schema_extra: + if "example" in json_schema_extra: yield ModelExample( model_cls=model_cls, example_name="example", - example_data=schema_extra["example"], + example_data=json_schema_extra["example"], ) - elif "examples" in schema_extra: - for index, example in enumerate(schema_extra["examples"]): + elif "examples" in json_schema_extra: + for index, example in enumerate(json_schema_extra["examples"]): yield ModelExample( model_cls=model_cls, example_name=f"examples_{index}", diff --git a/packages/settings-library/requirements/_base.in b/packages/settings-library/requirements/_base.in index ec1d848cc85..01734738bcb 100644 --- a/packages/settings-library/requirements/_base.in +++ b/packages/settings-library/requirements/_base.in @@ -3,8 +3,8 @@ # --constraint ../../../requirements/constraints.txt -pydantic>=1.9 - +pydantic +pydantic-settings # extra rich diff --git a/packages/settings-library/requirements/_base.txt b/packages/settings-library/requirements/_base.txt index 900c4fea2aa..0adc42f67e7 100644 --- a/packages/settings-library/requirements/_base.txt +++ b/packages/settings-library/requirements/_base.txt @@ -1,15 +1,24 @@ +annotated-types==0.7.0 + # via pydantic click==8.1.7 # via typer markdown-it-py==3.0.0 # via rich mdurl==0.1.2 # via markdown-it-py -pydantic==1.10.17 +pydantic==2.8.2 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in + # pydantic-settings +pydantic-core==2.20.1 + # via pydantic +pydantic-settings==2.4.0 + # via -r requirements/_base.in pygments==2.18.0 # via rich +python-dotenv==1.0.1 + # via pydantic-settings rich==13.7.1 # via # -r requirements/_base.in @@ -21,4 +30,5 @@ typer==0.12.4 typing-extensions==4.12.2 # via # pydantic + # pydantic-core # typer diff --git a/packages/settings-library/requirements/_test.txt b/packages/settings-library/requirements/_test.txt index 1ca7d43dd3c..56bf15d9c2d 100644 --- a/packages/settings-library/requirements/_test.txt +++ b/packages/settings-library/requirements/_test.txt @@ -2,8 +2,6 @@ coverage==7.6.1 # via # -r requirements/_test.in # pytest-cov -exceptiongroup==1.2.2 - # via pytest faker==27.0.0 # via -r requirements/_test.in iniconfig==2.0.0 @@ -34,12 +32,10 @@ pytest-sugar==1.0.0 python-dateutil==2.9.0.post0 # via faker python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in six==1.16.0 # via python-dateutil termcolor==2.4.0 # via pytest-sugar -tomli==2.0.1 - # via - # coverage - # pytest diff --git a/packages/settings-library/requirements/_tools.txt b/packages/settings-library/requirements/_tools.txt index a75c5397d80..d14257822b0 100644 --- a/packages/settings-library/requirements/_tools.txt +++ b/packages/settings-library/requirements/_tools.txt @@ -67,21 +67,11 @@ ruff==0.6.1 # via -r requirements/../../../requirements/devenv.txt setuptools==73.0.1 # via pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 # via # -c requirements/_base.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/packages/settings-library/src/settings_library/base.py b/packages/settings-library/src/settings_library/base.py index 296b453e26c..f098c12e1a3 100644 --- a/packages/settings-library/src/settings_library/base.py +++ b/packages/settings-library/src/settings_library/base.py @@ -1,19 +1,13 @@ +from functools import cached_property import logging from collections.abc import Sequence -from functools import cached_property -from typing import Final, get_args, get_origin - -from pydantic import ( - BaseConfig, - BaseSettings, - ConfigError, - Extra, - ValidationError, - validator, -) -from pydantic.error_wrappers import ErrorList, ErrorWrapper -from pydantic.fields import ModelField, Undefined -from pydantic.typing import is_literal_type +#from functools import cached_property +from typing import Final + +from pydantic import ConfigDict, ValidationError, ValidationInfo, field_validator +#from pydantic.error_wrappers import ErrorList, ErrorWrapper +#from pydantic.typing import is_literal_type +from pydantic_settings import BaseSettings _logger = logging.getLogger(__name__) @@ -22,11 +16,11 @@ ] = "%s auto_default_from_env unresolved, defaulting to None" -class DefaultFromEnvFactoryError(ValidationError): +class DefaultFromEnvFactoryError(ValueError): ... -def create_settings_from_env(field: ModelField): +def create_settings_from_env(field): # NOTE: Cannot pass only field.type_ because @prepare_field (when this function is called) # this value is still not resolved (field.type_ at that moment has a weak_ref). # Therefore we keep the entire 'field' but MUST be treated here as read-only @@ -46,9 +40,9 @@ def _default_factory(): ) return None - def _prepend_field_name(ee: ErrorList): - if isinstance(ee, ErrorWrapper): - return ErrorWrapper(ee.exc, (field.name, *ee.loc_tuple())) + def _prepend_field_name(ee): + if isinstance(ee): + return ValidationError(ee.exc, (field.name, *ee.loc_tuple())) assert isinstance(ee, Sequence) # nosec return [_prepend_field_name(e) for e in ee] @@ -70,66 +64,72 @@ class BaseCustomSettings(BaseSettings): SEE tests for details. """ - @validator("*", pre=True) + model_config = ConfigDict( + case_sensitive = True, # All must be capitalized + extra = "forbid", + allow_mutation = False, + frozen = True, + validate_all = True, + keep_untouched = (cached_property,) + ) + + @field_validator("*", mode="before") @classmethod - def parse_none(cls, v, field: ModelField): + def parse_none(cls, v): # WARNING: In nullable fields, envs equal to null or none are parsed as None !! - if field.allow_none and isinstance(v, str) and v.lower() in ("null", "none"): + if isinstance(v, str) and v.lower() in ("null", "none"): return None return v - class Config(BaseConfig): - case_sensitive = True # All must be capitalized - extra = Extra.forbid - allow_mutation = False - frozen = True - validate_all = True - keep_untouched = (cached_property,) - - @classmethod - def prepare_field(cls, field: ModelField) -> None: - super().prepare_field(field) - - auto_default_from_env = field.field_info.extra.get( - "auto_default_from_env", False - ) - - field_type = field.type_ - if args := get_args(field_type): - field_type = next(a for a in args if a != type(None)) - - # Avoids issubclass raising TypeError. SEE test_issubclass_type_error_with_pydantic_models - is_not_composed = ( - get_origin(field_type) is None - ) # is not composed as dict[str, Any] or Generic[Base] - # avoid literals raising TypeError - is_not_literal = is_literal_type(field.type_) is False - - if ( - is_not_literal - and is_not_composed - and issubclass(field_type, BaseCustomSettings) - ): - if auto_default_from_env: - assert field.field_info.default is Undefined - assert field.field_info.default_factory is None - - # Transform it into something like `Field(default_factory=create_settings_from_env(field))` - field.default_factory = create_settings_from_env(field) - field.default = None - field.required = False # has a default now - - elif ( - is_not_literal - and is_not_composed - and issubclass(field_type, BaseSettings) - ): - msg = f"{cls}.{field.name} of type {field_type} must inherit from BaseCustomSettings" - raise ConfigError(msg) - - elif auto_default_from_env: - msg = f"auto_default_from_env=True can only be used in BaseCustomSettings subclassesbut field {cls}.{field.name} is {field_type} " - raise ConfigError(msg) + # # TODO[pydantic]: The `Config` class inherits from another class, please create the `model_config` manually. + # # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. + # class Config(BaseConfig): + + + # @classmethod + # def prepare_field(cls, field: ModelField) -> None: + # super().prepare_field(field) + + # auto_default_from_env = field.field_info.extra.get( + # "auto_default_from_env", False + # ) + + # field_type = field.type_ + # if args := get_args(field_type): + # field_type = next(a for a in args if a != type(None)) + + # # Avoids issubclass raising TypeError. SEE test_issubclass_type_error_with_pydantic_models + # is_not_composed = ( + # get_origin(field_type) is None + # ) # is not composed as dict[str, Any] or Generic[Base] + # # avoid literals raising TypeError + # is_not_literal = is_literal_type(field.type_) is False + + # if ( + # is_not_literal + # and is_not_composed + # and issubclass(field_type, BaseCustomSettings) + # ): + # if auto_default_from_env: + # assert field.field_info.default is Undefined + # assert field.field_info.default_factory is None + + # # Transform it into something like `Field(default_factory=create_settings_from_env(field))` + # field.default_factory = create_settings_from_env(field) + # field.default = None + # field.required = False # has a default now + + # elif ( + # is_not_literal + # and is_not_composed + # and issubclass(field_type, BaseSettings) + # ): + # msg = f"{cls}.{field.name} of type {field_type} must inherit from BaseCustomSettings" + # raise ConfigError(msg) + + # elif auto_default_from_env: + # msg = f"auto_default_from_env=True can only be used in BaseCustomSettings subclassesbut field {cls}.{field.name} is {field_type} " + # raise ConfigError(msg) @classmethod def create_from_envs(cls, **overrides): diff --git a/packages/settings-library/src/settings_library/basic_types.py b/packages/settings-library/src/settings_library/basic_types.py index 277832669e1..e863874c7b8 100644 --- a/packages/settings-library/src/settings_library/basic_types.py +++ b/packages/settings-library/src/settings_library/basic_types.py @@ -6,18 +6,15 @@ import re from enum import Enum -from pydantic import ConstrainedInt, ConstrainedStr - +from pydantic import Field, StringConstraints +from typing_extensions import Annotated # port number range -class PortInt(ConstrainedInt): - gt = 0 - lt = 65535 +PortInt = Annotated[int, Field(gt=0, lt=65535)] # e.g. 'v5' -class VersionTag(ConstrainedStr): - regex = re.compile(r"^v\d$") +VersionTag = Annotated[str, StringConstraints(pattern=re.compile(r"^v\d$"))] class LogLevel(str, Enum): @@ -55,7 +52,6 @@ class BuildTargetEnum(str, Enum): # non-empty bounded string used as identifier # e.g. "123" or "name_123" or "fa327c73-52d8-462a-9267-84eeaf0f90e3" but NOT "" -class IDStr(ConstrainedStr): - strip_whitespace = True - min_length = 1 - max_length = 50 +IDStr = Annotated[ + str, StringConstraints(strip_whitespace=True, min_length=1, max_length=50) +] diff --git a/packages/settings-library/src/settings_library/catalog.py b/packages/settings-library/src/settings_library/catalog.py index e5f44f29269..743ad4859d3 100644 --- a/packages/settings-library/src/settings_library/catalog.py +++ b/packages/settings-library/src/settings_library/catalog.py @@ -1,6 +1,6 @@ from functools import cached_property -from pydantic import parse_obj_as +from pydantic import TypeAdapter from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag from settings_library.utils_service import ( @@ -13,7 +13,7 @@ class CatalogSettings(BaseCustomSettings, MixinServiceSettings): CATALOG_HOST: str = "catalog" CATALOG_PORT: PortInt = DEFAULT_FASTAPI_PORT - CATALOG_VTAG: VersionTag = parse_obj_as(VersionTag, "v0") + CATALOG_VTAG: VersionTag = TypeAdapter(VersionTag).validate_strings("v0") @cached_property def api_base_url(self) -> str: diff --git a/packages/settings-library/src/settings_library/comp_services.py b/packages/settings-library/src/settings_library/comp_services.py index e3cb628f7b7..ef55013d828 100644 --- a/packages/settings-library/src/settings_library/comp_services.py +++ b/packages/settings-library/src/settings_library/comp_services.py @@ -1,5 +1,4 @@ -from pydantic import ByteSize, NonNegativeInt, validator -from pydantic.tools import parse_raw_as +from pydantic import ByteSize, NonNegativeInt, TypeAdapter, field_validator from settings_library.base import BaseCustomSettings from ._constants import GB @@ -10,19 +9,20 @@ class ComputationalServices(BaseCustomSettings): DEFAULT_MAX_NANO_CPUS: NonNegativeInt = _DEFAULT_MAX_NANO_CPUS_VALUE - DEFAULT_MAX_MEMORY: ByteSize = parse_raw_as( - ByteSize, f"{_DEFAULT_MAX_MEMORY_VALUE}" + DEFAULT_MAX_MEMORY: ByteSize = TypeAdapter(ByteSize).validate_strings( + f"{_DEFAULT_MAX_MEMORY_VALUE}" ) + DEFAULT_RUNTIME_TIMEOUT: NonNegativeInt = 0 - @validator("DEFAULT_MAX_NANO_CPUS", pre=True) + @field_validator("DEFAULT_MAX_NANO_CPUS", mode="before") @classmethod def _set_default_cpus_if_negative(cls, v): if v is None or v == "" or int(v) <= 0: v = _DEFAULT_MAX_NANO_CPUS_VALUE return v - @validator("DEFAULT_MAX_MEMORY", pre=True) + @field_validator("DEFAULT_MAX_MEMORY", mode="before") @classmethod def _set_default_memory_if_negative(cls, v): if v is None or v == "" or int(v) <= 0: diff --git a/packages/settings-library/src/settings_library/director_v2.py b/packages/settings-library/src/settings_library/director_v2.py index 78c5edd78c6..cab7637bbf8 100644 --- a/packages/settings-library/src/settings_library/director_v2.py +++ b/packages/settings-library/src/settings_library/director_v2.py @@ -1,6 +1,6 @@ from functools import cached_property -from pydantic import parse_obj_as +from pydantic import TypeAdapter from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag from settings_library.utils_service import ( @@ -13,7 +13,7 @@ class DirectorV2Settings(BaseCustomSettings, MixinServiceSettings): DIRECTOR_V2_HOST: str = "director-v2" DIRECTOR_V2_PORT: PortInt = DEFAULT_FASTAPI_PORT - DIRECTOR_V2_VTAG: VersionTag = parse_obj_as(VersionTag, "v2") + DIRECTOR_V2_VTAG: VersionTag = TypeAdapter(VersionTag).validate_python("v2") @cached_property def api_base_url(self) -> str: diff --git a/packages/settings-library/src/settings_library/docker_registry.py b/packages/settings-library/src/settings_library/docker_registry.py index bb365cb9785..17a426e32dd 100644 --- a/packages/settings-library/src/settings_library/docker_registry.py +++ b/packages/settings-library/src/settings_library/docker_registry.py @@ -1,7 +1,7 @@ from functools import cached_property -from typing import Any, ClassVar +from typing import Any -from pydantic import Field, SecretStr, validator +from pydantic import ConfigDict, Field, SecretStr, field_validator from .base import BaseCustomSettings @@ -23,7 +23,21 @@ class RegistrySettings(BaseCustomSettings): ) REGISTRY_SSL: bool = Field(..., description="access to registry through ssl") - @validator("REGISTRY_PATH", pre=True) + model_config = ConfigDict( + json_schema_extra={ + "examples": [ + { + "REGISTRY_AUTH": "True", + "REGISTRY_USER": "theregistryuser", + "REGISTRY_PW": "some_secret_value", + "REGISTRY_SSL": "True", + "REGISTRY_URL": "registry.osparc-master.speag.com", + } + ], + } + ) + + @field_validator("REGISTRY_PATH", mode="before") @classmethod def _escape_none_string(cls, v) -> Any | None: return None if v == "None" else v @@ -35,16 +49,3 @@ def resolved_registry_url(self) -> str: @cached_property def api_url(self) -> str: return f"{self.REGISTRY_URL}/v2" - - class Config(BaseCustomSettings.Config): - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] - "examples": [ - { - "REGISTRY_AUTH": "True", - "REGISTRY_USER": "theregistryuser", - "REGISTRY_PW": "some_secret_value", - "REGISTRY_SSL": "True", - "REGISTRY_URL": "registry.osparc-master.speag.com", - } - ], - } diff --git a/packages/settings-library/src/settings_library/ec2.py b/packages/settings-library/src/settings_library/ec2.py index 2cd7cf0b9a6..a28fd9335c8 100644 --- a/packages/settings-library/src/settings_library/ec2.py +++ b/packages/settings-library/src/settings_library/ec2.py @@ -1,6 +1,4 @@ -from typing import Any, ClassVar - -from pydantic import Field +from pydantic import ConfigDict, Field from .base import BaseCustomSettings @@ -13,8 +11,8 @@ class EC2Settings(BaseCustomSettings): EC2_REGION_NAME: str = "us-east-1" EC2_SECRET_ACCESS_KEY: str - class Config(BaseCustomSettings.Config): - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "EC2_ACCESS_KEY_ID": "my_access_key_id", @@ -24,3 +22,4 @@ class Config(BaseCustomSettings.Config): } ], } + ) diff --git a/packages/settings-library/src/settings_library/efs.py b/packages/settings-library/src/settings_library/efs.py index d09b8abb20f..34c48f9dca6 100644 --- a/packages/settings-library/src/settings_library/efs.py +++ b/packages/settings-library/src/settings_library/efs.py @@ -8,7 +8,7 @@ class AwsEfsSettings(BaseCustomSettings): EFS_DNS_NAME: str = Field( description="AWS Elastic File System DNS name", - example="fs-xxx.efs.us-east-1.amazonaws.com", + examples=["fs-xxx.efs.us-east-1.amazonaws.com"], ) EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: str EFS_MOUNTED_PATH: Path = Field( @@ -16,7 +16,7 @@ class AwsEfsSettings(BaseCustomSettings): ) EFS_ONLY_ENABLED_FOR_USERIDS: list[int] = Field( description="This is temporary solution so we can enable it for specific users for testing purpose", - example=[1], + examples=[[1]], ) diff --git a/packages/settings-library/src/settings_library/email.py b/packages/settings-library/src/settings_library/email.py index b15bf209405..6ec1d346a25 100644 --- a/packages/settings-library/src/settings_library/email.py +++ b/packages/settings-library/src/settings_library/email.py @@ -1,6 +1,6 @@ from enum import Enum -from pydantic import root_validator +from pydantic import model_validator from pydantic.fields import Field from pydantic.types import SecretStr @@ -31,7 +31,7 @@ class SMTPSettings(BaseCustomSettings): SMTP_USERNAME: str | None = Field(None, min_length=1) SMTP_PASSWORD: SecretStr | None = Field(None, min_length=1) - @root_validator + @model_validator(mode="before") @classmethod def _both_credentials_must_be_set(cls, values): username = values.get("SMTP_USERNAME") @@ -43,7 +43,7 @@ def _both_credentials_must_be_set(cls, values): return values - @root_validator + @model_validator(mode="before") @classmethod def _enabled_tls_required_authentication(cls, values): smtp_protocol = values.get("SMTP_PROTOCOL") diff --git a/packages/settings-library/src/settings_library/node_ports.py b/packages/settings-library/src/settings_library/node_ports.py index 2a5d12f1bd7..e600daed2f4 100644 --- a/packages/settings-library/src/settings_library/node_ports.py +++ b/packages/settings-library/src/settings_library/node_ports.py @@ -1,7 +1,7 @@ from datetime import timedelta from typing import Final -from pydantic import Field, NonNegativeInt, PositiveInt, SecretStr, root_validator +from pydantic import Field, NonNegativeInt, PositiveInt, SecretStr, model_validator from .base import BaseCustomSettings from .postgres import PostgresSettings @@ -21,7 +21,7 @@ def auth_required(self) -> bool: # for details see https://github.com/ITISFoundation/osparc-issues/issues/1264 return self.STORAGE_USERNAME is not None and self.STORAGE_PASSWORD is not None - @root_validator + @model_validator(mode="before") @classmethod def _validate_auth_fields(cls, values): username = values["STORAGE_USERNAME"] diff --git a/packages/settings-library/src/settings_library/postgres.py b/packages/settings-library/src/settings_library/postgres.py index f8335bbeed2..3aaabf54e33 100644 --- a/packages/settings-library/src/settings_library/postgres.py +++ b/packages/settings-library/src/settings_library/postgres.py @@ -1,8 +1,15 @@ +from typing import Self import urllib.parse from functools import cached_property -from typing import Any, ClassVar -from pydantic import Field, PostgresDsn, SecretStr, validator +from pydantic import ( + AliasChoices, + ConfigDict, + Field, + PostgresDsn, + SecretStr, + model_validator, +) from .base import BaseCustomSettings from .basic_types import PortInt @@ -31,21 +38,35 @@ class PostgresSettings(BaseCustomSettings): POSTGRES_CLIENT_NAME: str | None = Field( default=None, description="Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)", - env=[ + validation_alias=AliasChoices( "POSTGRES_CLIENT_NAME", # This is useful when running inside a docker container, then the hostname is set each client gets a different name "HOST", "HOSTNAME", - ], + ), ) - @validator("POSTGRES_MAXSIZE") - @classmethod - def _check_size(cls, v, values): - if not (values["POSTGRES_MINSIZE"] <= v): - msg = f"assert POSTGRES_MINSIZE={values['POSTGRES_MINSIZE']} <= POSTGRES_MAXSIZE={v}" + model_config = ConfigDict( + json_schema_extra={ + "examples": [ + # minimal required + { + "POSTGRES_HOST": "localhost", + "POSTGRES_PORT": "5432", + "POSTGRES_USER": "usr", + "POSTGRES_PASSWORD": "secret", + "POSTGRES_DB": "db", + } + ], + } + ) + + @model_validator(mode='after') + def _check_size(self) -> Self: + if not (self.POSTGRES_MINSIZE <= self.POSTGRES_MAXSIZE): + msg = f"assert POSTGRES_MINSIZE={self.POSTGRES_MINSIZE} <= POSTGRES_MAXSIZE={self.POSTGRES_MAXSIZE}" raise ValueError(msg) - return v + return self @cached_property def dsn(self) -> str: @@ -80,17 +101,3 @@ def dsn_with_query(self) -> str: {"application_name": self.POSTGRES_CLIENT_NAME} ) return dsn - - class Config(BaseCustomSettings.Config): - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] - "examples": [ - # minimal required - { - "POSTGRES_HOST": "localhost", - "POSTGRES_PORT": "5432", - "POSTGRES_USER": "usr", - "POSTGRES_PASSWORD": "secret", - "POSTGRES_DB": "db", - } - ], - } diff --git a/packages/settings-library/src/settings_library/rabbit.py b/packages/settings-library/src/settings_library/rabbit.py index 19c6af0b656..81d1ac01049 100644 --- a/packages/settings-library/src/settings_library/rabbit.py +++ b/packages/settings-library/src/settings_library/rabbit.py @@ -1,6 +1,6 @@ from functools import cached_property -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pydantic.networks import AnyUrl from pydantic.types import SecretStr @@ -15,7 +15,7 @@ class RabbitDsn(AnyUrl): class RabbitSettings(BaseCustomSettings): # host RABBIT_HOST: str - RABBIT_PORT: PortInt = parse_obj_as(PortInt, 5672) + RABBIT_PORT: PortInt = TypeAdapter(PortInt).validate_python(5672) RABBIT_SECURE: bool # auth diff --git a/packages/settings-library/src/settings_library/redis.py b/packages/settings-library/src/settings_library/redis.py index ecccad69c10..546c77e7752 100644 --- a/packages/settings-library/src/settings_library/redis.py +++ b/packages/settings-library/src/settings_library/redis.py @@ -1,6 +1,6 @@ from enum import Enum -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pydantic.networks import RedisDsn from pydantic.types import SecretStr @@ -22,7 +22,7 @@ class RedisDatabase(int, Enum): class RedisSettings(BaseCustomSettings): # host REDIS_HOST: str = "redis" - REDIS_PORT: PortInt = parse_obj_as(PortInt, 6789) + REDIS_PORT: PortInt = TypeAdapter(PortInt).validate_python(6789) # auth REDIS_USER: str | None = None diff --git a/packages/settings-library/src/settings_library/resource_usage_tracker.py b/packages/settings-library/src/settings_library/resource_usage_tracker.py index dc696fab76c..480d48a3798 100644 --- a/packages/settings-library/src/settings_library/resource_usage_tracker.py +++ b/packages/settings-library/src/settings_library/resource_usage_tracker.py @@ -1,7 +1,7 @@ from datetime import timedelta from functools import cached_property -from pydantic import parse_obj_as +from pydantic import TypeAdapter from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag from settings_library.utils_service import ( @@ -16,7 +16,7 @@ class ResourceUsageTrackerSettings(BaseCustomSettings, MixinServiceSettings): RESOURCE_USAGE_TRACKER_HOST: str = "resource-usage-tracker" RESOURCE_USAGE_TRACKER_PORT: PortInt = DEFAULT_FASTAPI_PORT - RESOURCE_USAGE_TRACKER_VTAG: VersionTag = parse_obj_as(VersionTag, "v1") + RESOURCE_USAGE_TRACKER_VTAG: VersionTag = TypeAdapter(VersionTag).validate_strings("v1") @cached_property def api_base_url(self) -> str: diff --git a/packages/settings-library/src/settings_library/s3.py b/packages/settings-library/src/settings_library/s3.py index cef1bf11be5..5e971283d46 100644 --- a/packages/settings-library/src/settings_library/s3.py +++ b/packages/settings-library/src/settings_library/s3.py @@ -1,6 +1,4 @@ -from typing import Any, ClassVar - -from pydantic import AnyHttpUrl, Field +from pydantic import AnyHttpUrl, ConfigDict, Field from .base import BaseCustomSettings from .basic_types import IDStr @@ -15,8 +13,8 @@ class S3Settings(BaseCustomSettings): S3_REGION: IDStr S3_SECRET_KEY: IDStr - class Config(BaseCustomSettings.Config): - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + model_config = ConfigDict( + json_schema_extra={ "examples": [ { # non AWS use-case @@ -35,3 +33,4 @@ class Config(BaseCustomSettings.Config): }, ], } + ) diff --git a/packages/settings-library/src/settings_library/ssm.py b/packages/settings-library/src/settings_library/ssm.py index 32b965fa123..05c5200a0b5 100644 --- a/packages/settings-library/src/settings_library/ssm.py +++ b/packages/settings-library/src/settings_library/ssm.py @@ -1,6 +1,4 @@ -from typing import Any, ClassVar - -from pydantic import AnyHttpUrl, Field, SecretStr +from pydantic import AnyHttpUrl, ConfigDict, Field, SecretStr from .base import BaseCustomSettings @@ -13,8 +11,8 @@ class SSMSettings(BaseCustomSettings): SSM_REGION_NAME: str = "us-east-1" SSM_SECRET_ACCESS_KEY: SecretStr - class Config(BaseCustomSettings.Config): - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "SSM_ACCESS_KEY_ID": "my_access_key_id", @@ -24,3 +22,4 @@ class Config(BaseCustomSettings.Config): } ], } + ) diff --git a/packages/settings-library/src/settings_library/storage.py b/packages/settings-library/src/settings_library/storage.py index 92ec0301257..16d97a7a61a 100644 --- a/packages/settings-library/src/settings_library/storage.py +++ b/packages/settings-library/src/settings_library/storage.py @@ -1,6 +1,6 @@ from functools import cached_property -from pydantic import parse_obj_as +from pydantic import TypeAdapter from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag from settings_library.utils_service import ( @@ -13,7 +13,7 @@ class StorageSettings(BaseCustomSettings, MixinServiceSettings): STORAGE_HOST: str = "storage" STORAGE_PORT: PortInt = DEFAULT_AIOHTTP_PORT - STORAGE_VTAG: VersionTag = parse_obj_as(VersionTag, "v0") + STORAGE_VTAG: VersionTag = TypeAdapter(VersionTag).validate_strings("v0") @cached_property def base_url(self) -> str: diff --git a/packages/settings-library/src/settings_library/tracing.py b/packages/settings-library/src/settings_library/tracing.py index 28a11cbbf6a..a5f1c8202af 100644 --- a/packages/settings-library/src/settings_library/tracing.py +++ b/packages/settings-library/src/settings_library/tracing.py @@ -1,4 +1,4 @@ -from pydantic import AnyUrl, Field, parse_obj_as +from pydantic import AliasChoices, AnyUrl, Field, TypeAdapter from .base import BaseCustomSettings @@ -7,15 +7,15 @@ class TracingSettings(BaseCustomSettings): TRACING_ZIPKIN_ENDPOINT: AnyUrl = Field( - default=parse_obj_as(AnyUrl, "http://jaeger:9411"), + default=TypeAdapter(AnyUrl).validate_strings("http://jaeger:9411"), description="Zipkin compatible endpoint", ) TRACING_THRIFT_COMPACT_ENDPOINT: AnyUrl = Field( - default=parse_obj_as(AnyUrl, "http://jaeger:5775"), + default=TypeAdapter(AnyUrl).validate_strings("http://jaeger:5775"), description="accept zipkin.thrift over compact thrift protocol (deprecated, used by legacy clients only)", ) TRACING_CLIENT_NAME: str = Field( default=UNDEFINED_CLIENT_NAME, description="Name of the application connecting the tracing service", - env=["HOST", "HOSTNAME", "TRACING_CLIENT_NAME"], + validation_alias=AliasChoices("HOST", "HOSTNAME", "TRACING_CLIENT_NAME"), ) diff --git a/packages/settings-library/src/settings_library/twilio.py b/packages/settings-library/src/settings_library/twilio.py index eb4ec0c707a..2f27e2a59ff 100644 --- a/packages/settings-library/src/settings_library/twilio.py +++ b/packages/settings-library/src/settings_library/twilio.py @@ -7,28 +7,23 @@ import re -from re import Pattern -from pydantic import ConstrainedStr, Field, parse_obj_as +from pydantic import Field, StringConstraints, TypeAdapter +from typing_extensions import Annotated from .base import BaseCustomSettings - -class CountryCodeStr(ConstrainedStr): - # Based on https://countrycode.org/ - strip_whitespace: bool = True - regex: Pattern[str] | None = re.compile(r"^\d{1,4}") - - class Config: - frozen = True +# Based on https://countrycode.org/ +CountryCodeStr = Annotated[ + str, StringConstraints(strip_whitespace=True, pattern=re.compile(r"^\d{1,4}")) +] class TwilioSettings(BaseCustomSettings): TWILIO_ACCOUNT_SID: str = Field(..., description="Twilio account String Identifier") TWILIO_AUTH_TOKEN: str = Field(..., description="API tokens") TWILIO_COUNTRY_CODES_W_ALPHANUMERIC_SID_SUPPORT: list[CountryCodeStr] = Field( - default=parse_obj_as( - list[CountryCodeStr], + default=TypeAdapter(list[CountryCodeStr]).validate_python( [ "41", ], diff --git a/packages/settings-library/src/settings_library/utils_cli.py b/packages/settings-library/src/settings_library/utils_cli.py index 79d0e1ac145..c4f286ee91d 100644 --- a/packages/settings-library/src/settings_library/utils_cli.py +++ b/packages/settings-library/src/settings_library/utils_cli.py @@ -7,7 +7,7 @@ import rich import typer from pydantic import ValidationError -from pydantic.env_settings import BaseSettings +from pydantic_settings import BaseSettings from ._constants import HEADER_STR from .base import BaseCustomSettings diff --git a/packages/settings-library/src/settings_library/utils_service.py b/packages/settings-library/src/settings_library/utils_service.py index e7bb66057c5..2e4eaa9de7b 100644 --- a/packages/settings-library/src/settings_library/utils_service.py +++ b/packages/settings-library/src/settings_library/utils_service.py @@ -4,14 +4,14 @@ """ from enum import Enum, auto -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pydantic.networks import AnyUrl from pydantic.types import SecretStr from .basic_types import PortInt -DEFAULT_AIOHTTP_PORT: PortInt = parse_obj_as(PortInt, 8080) -DEFAULT_FASTAPI_PORT: PortInt = parse_obj_as(PortInt, 8000) +DEFAULT_AIOHTTP_PORT: PortInt = TypeAdapter(PortInt).validate_python(8080) +DEFAULT_FASTAPI_PORT: PortInt = TypeAdapter(PortInt).validate_python(8000) class URLPart(Enum): diff --git a/packages/settings-library/src/settings_library/webserver.py b/packages/settings-library/src/settings_library/webserver.py index 4da2c41d699..730cb619720 100644 --- a/packages/settings-library/src/settings_library/webserver.py +++ b/packages/settings-library/src/settings_library/webserver.py @@ -1,6 +1,6 @@ from functools import cached_property -from pydantic import parse_obj_as +from pydantic import TypeAdapter from .base import BaseCustomSettings from .basic_types import PortInt, VersionTag @@ -10,7 +10,7 @@ class WebServerSettings(BaseCustomSettings, MixinServiceSettings): WEBSERVER_HOST: str = "webserver" WEBSERVER_PORT: PortInt = DEFAULT_AIOHTTP_PORT - WEBSERVER_VTAG: VersionTag = parse_obj_as(VersionTag, "v0") + WEBSERVER_VTAG: VersionTag = TypeAdapter(VersionTag).validate_strings("v0") @cached_property def base_url(self) -> str: diff --git a/packages/settings-library/tests/test__models_examples.py b/packages/settings-library/tests/test__models_examples.py index c60a6c08261..96ffc7135b2 100644 --- a/packages/settings-library/tests/test__models_examples.py +++ b/packages/settings-library/tests/test__models_examples.py @@ -14,6 +14,6 @@ def test_all_settings_library_models_config_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): - assert model_cls.parse_obj( + assert model_cls.model_validate( example_data ), f"Failed {example_name} : {json.dumps(example_data)}" diff --git a/packages/settings-library/tests/test__pydantic_settings.py b/packages/settings-library/tests/test__pydantic_settings.py index 8cf3eadc30f..885f72be72c 100644 --- a/packages/settings-library/tests/test__pydantic_settings.py +++ b/packages/settings-library/tests/test__pydantic_settings.py @@ -13,14 +13,14 @@ """ -from pydantic import BaseSettings, validator -from pydantic.fields import ModelField, Undefined +from pydantic import ValidationInfo, field_validator +from pydantic_settings import BaseSettings def assert_field_specs( model_cls, name, is_required, is_nullable, explicit_default, defaults ): - field: ModelField = model_cls.__fields__[name] + field = model_cls.__fields__[name] print(field, field.field_info) assert field.required == is_required @@ -46,11 +46,12 @@ class Settings(BaseSettings): # Other ways to write down "required" is using ... VALUE_ALSO_REQUIRED: int = ... # type: ignore - @validator("*", pre=True) + + @field_validator("*", mode="before") @classmethod - def parse_none(cls, v, values, field: ModelField): + def parse_none(cls, v, info: ValidationInfo): # WARNING: In nullable fields, envs equal to null or none are parsed as None !! - if field.allow_none: + if info.allow_none: if isinstance(v, str) and v.lower() in ("null", "none"): return None return v @@ -134,7 +135,7 @@ def test_construct(monkeypatch): settings_from_init = Settings( VALUE=1, VALUE_ALSO_REQUIRED=10, VALUE_NULLABLE_REQUIRED=None ) - print(settings_from_init.json(exclude_unset=True, indent=1)) + print(settings_from_init.model_dump_json(exclude_unset=True, indent=1)) # from env vars monkeypatch.setenv("VALUE", "1") @@ -144,13 +145,13 @@ def test_construct(monkeypatch): ) # WARNING: set this env to None would not work w/o ``parse_none`` validator! bug??? settings_from_env = Settings() - print(settings_from_env.json(exclude_unset=True, indent=1)) + print(settings_from_env.model_dump_json(exclude_unset=True, indent=1)) assert settings_from_init == settings_from_env # mixed settings_from_both = Settings(VALUE_NULLABLE_REQUIRED=3) - print(settings_from_both.json(exclude_unset=True, indent=1)) + print(settings_from_both.model_dump_json(exclude_unset=True, indent=1)) assert settings_from_both == settings_from_init.copy( update={"VALUE_NULLABLE_REQUIRED": 3} diff --git a/packages/settings-library/tests/test_base.py b/packages/settings-library/tests/test_base.py index 7cbd9fa8773..da428b602a2 100644 --- a/packages/settings-library/tests/test_base.py +++ b/packages/settings-library/tests/test_base.py @@ -10,8 +10,9 @@ import pytest import settings_library.base -from pydantic import BaseModel, BaseSettings, ValidationError +from pydantic import BaseModel, ValidationError from pydantic.fields import Field +from pydantic_settings import BaseSettings from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_envfile from settings_library.base import ( @@ -38,7 +39,7 @@ def _get_attrs_tree(obj: Any) -> dict[str, Any]: def _print_defaults(model_cls: type[BaseModel]): - for field in model_cls.__fields__.values(): + for field in model_cls.model_fields.values(): print(field.name, ":", end="") try: default = field.get_default() @@ -48,7 +49,9 @@ def _print_defaults(model_cls: type[BaseModel]): def _dumps_model_class(model_cls: type[BaseModel]): - d = {field.name: _get_attrs_tree(field) for field in model_cls.__fields__.values()} + d = { + field.name: _get_attrs_tree(field) for field in model_cls.model_fields.values() + } return json.dumps(d, indent=1) @@ -101,14 +104,14 @@ def test_create_settings_class( # DEV: Path("M1.ignore.json").write_text(dumps_model_class(M)) - assert M.__fields__["VALUE_NULLABLE_DEFAULT_ENV"].default_factory + assert M.model_fields["VALUE_NULLABLE_DEFAULT_ENV"].default_factory - assert M.__fields__["VALUE_NULLABLE_DEFAULT_ENV"].get_default() is None + assert M.model_fields["VALUE_NULLABLE_DEFAULT_ENV"].get_default() is None - assert M.__fields__["VALUE_DEFAULT_ENV"].default_factory + assert M.model_fields["VALUE_DEFAULT_ENV"].default_factory with pytest.raises(DefaultFromEnvFactoryError): - M.__fields__["VALUE_DEFAULT_ENV"].get_default() + M.model_fields["VALUE_DEFAULT_ENV"].get_default() def test_create_settings_class_with_environment( @@ -136,15 +139,15 @@ def test_create_settings_class_with_environment( instance = SettingsClass() - print(instance.json(indent=2)) + print(instance.model_dump_json(indent=2)) # checks - assert instance.dict(exclude_unset=True) == { + assert instance.model_dump(exclude_unset=True) == { "VALUE": {"S_VALUE": 2}, "VALUE_NULLABLE_REQUIRED": {"S_VALUE": 3}, } - assert instance.dict() == { + assert instance.model_dump() == { "VALUE": {"S_VALUE": 2}, "VALUE_DEFAULT": {"S_VALUE": 42}, "VALUE_CONFUSING": None, diff --git a/packages/settings-library/tests/test_base_w_postgres.py b/packages/settings-library/tests/test_base_w_postgres.py index d54d40bf925..f6f90ae0999 100644 --- a/packages/settings-library/tests/test_base_w_postgres.py +++ b/packages/settings-library/tests/test_base_w_postgres.py @@ -49,7 +49,7 @@ class _FakePostgresSettings(BaseCustomSettings): POSTGRES_CLIENT_NAME: str | None = Field( None, - env=["HOST", "HOSTNAME", "POSTGRES_CLIENT_NAME"], + validation_alias=["HOST", "HOSTNAME", "POSTGRES_CLIENT_NAME"], ) # diff --git a/packages/settings-library/tests/test_email.py b/packages/settings-library/tests/test_email.py index 1cd3978503e..acb9d607c89 100644 --- a/packages/settings-library/tests/test_email.py +++ b/packages/settings-library/tests/test_email.py @@ -67,7 +67,7 @@ def all_env_devel_undefined( ], ) def test_smtp_configuration_ok(cfg: dict[str, Any], all_env_devel_undefined: None): - assert SMTPSettings.parse_obj(cfg) + assert SMTPSettings.model_validate(cfg) @pytest.mark.parametrize( diff --git a/packages/settings-library/tests/test_postgres.py b/packages/settings-library/tests/test_postgres.py index 1708acc7808..be16ffd1c78 100644 --- a/packages/settings-library/tests/test_postgres.py +++ b/packages/settings-library/tests/test_postgres.py @@ -20,12 +20,12 @@ def test_cached_property_dsn(mock_environment: dict): assert all(key == key.upper() for key in settings.dict()) # dsn is computed from the other fields - assert "dsn" not in settings.dict() + assert "dsn" not in settings.model_dump() # causes cached property to be computed and stored on the instance assert settings.dsn - assert "dsn" in settings.dict() + assert "dsn" in settings.model_dump() def test_dsn_with_query(mock_environment: dict, monkeypatch): diff --git a/packages/settings-library/tests/test_twilio.py b/packages/settings-library/tests/test_twilio.py index 6f2830ea4aa..1989fbe6a9f 100644 --- a/packages/settings-library/tests/test_twilio.py +++ b/packages/settings-library/tests/test_twilio.py @@ -20,7 +20,7 @@ def test_twilio_settings_within_envdevel( }, ) settings = TwilioSettings.create_from_envs() - print(settings.json(indent=2)) + print(settings.model_dump_json(indent=2)) assert settings diff --git a/packages/settings-library/tests/test_utils_logging.py b/packages/settings-library/tests/test_utils_logging.py index 9054b391333..ce043184a21 100644 --- a/packages/settings-library/tests/test_utils_logging.py +++ b/packages/settings-library/tests/test_utils_logging.py @@ -1,6 +1,6 @@ import logging -from pydantic import Field, validator +from pydantic import Field, field_validator from settings_library.base import BaseCustomSettings from settings_library.basic_types import BootMode from settings_library.utils_logging import MixinLoggingSettings @@ -19,7 +19,7 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): # LOGGING LOG_LEVEL: str = Field( "WARNING", - env=[ + validation_alias=[ "APPNAME_LOG_LEVEL", "LOG_LEVEL", ], @@ -27,7 +27,7 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): APPNAME_DEBUG: bool = Field(False, description="Starts app in debug mode") - @validator("LOG_LEVEL") + @field_validator("LOG_LEVEL") @classmethod def _v(cls, value) -> str: return cls.validate_log_level(value) @@ -40,7 +40,7 @@ def _v(cls, value) -> str: assert settings.LOG_LEVEL == "DEBUG" assert ( - settings.json() + settings.model_dump_json() == '{"SC_BOOT_MODE": null, "LOG_LEVEL": "DEBUG", "APPNAME_DEBUG": false}' ) @@ -48,6 +48,6 @@ def _v(cls, value) -> str: assert settings.log_level == logging.DEBUG # log_level is cached-property (notice that is lower-case!), and gets added after first use assert ( - settings.json() + settings.model_dump_json() == '{"SC_BOOT_MODE": null, "LOG_LEVEL": "DEBUG", "APPNAME_DEBUG": false, "log_level": 10}' ) diff --git a/packages/settings-library/tests/test_utils_service.py b/packages/settings-library/tests/test_utils_service.py index a3638f9b31e..5802d5889aa 100644 --- a/packages/settings-library/tests/test_utils_service.py +++ b/packages/settings-library/tests/test_utils_service.py @@ -5,7 +5,7 @@ from functools import cached_property import pytest -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from pydantic.types import SecretStr from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag @@ -88,8 +88,10 @@ def test_service_settings_base_urls(service_settings_cls: type): settings_with_defaults = service_settings_cls() - base_url = parse_obj_as(AnyHttpUrl, settings_with_defaults.base_url) - api_base_url = parse_obj_as(AnyHttpUrl, settings_with_defaults.api_base_url) + base_url = TypeAdapter(AnyHttpUrl).validate_strings(settings_with_defaults.base_url) + api_base_url = TypeAdapter(AnyHttpUrl).validate_strings( + settings_with_defaults.api_base_url + ) assert base_url.path != api_base_url.path assert (base_url.scheme, base_url.host, base_url.port) == ( diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 3316f4276ed..aa123429313 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -34,8 +34,7 @@ urllib3>=1.26.5 # https://github.com/advisories/GH # SEE https://github.com/ITISFoundation/osparc-simcore/issues/4481 -fastapi<0.100.0 -pydantic<2.0 +pydantic<3 # with new released version 1.0.0 (https://github.com/aio-libs/aiozipkin/releases). # TODO: includes async features https://docs.sqlalchemy.org/en/14/changelog/migration_20.html diff --git a/services/web/server/tests/unit/isolated/test_storage_schemas.py b/services/web/server/tests/unit/isolated/test_storage_schemas.py index c11ce1f1345..31ea4260bb4 100644 --- a/services/web/server/tests/unit/isolated/test_storage_schemas.py +++ b/services/web/server/tests/unit/isolated/test_storage_schemas.py @@ -20,4 +20,4 @@ def test_model_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): print(example_name, ":", json.dumps(example_data)) - assert model_cls.parse_obj(example_data) + assert model_cls.model_validate(example_data)