From b5ce2172143a1833116911875c58d406b8623c9b Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:53:56 +0200 Subject: [PATCH 01/15] chore: expand gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6a72a76f..92be4e5a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ requests.egg-info/ *.pyc *.swp *.egg +.env/ env/ .venv/ venv/ From edb59071d2570b4062695e7c9b50afbfb34c7897 Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Wed, 17 Jul 2024 08:05:17 +0200 Subject: [PATCH 02/15] test: remove skip on remote tests --- tests/models/test_base_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/models/test_base_model.py b/tests/models/test_base_model.py index 2d231ba4..cf487302 100644 --- a/tests/models/test_base_model.py +++ b/tests/models/test_base_model.py @@ -90,7 +90,7 @@ def test_basemodellist_execute(self, model, filename, modelname): output_folder = Path(TestUtils.get_output_test_data_dir(modelname)) / "multiple" ml = BaseModelList(models=[a, b]) - for i, modelinstance in enumerate(ml.models): + for modelinstance in ml.models: modelinstance.parse(benchmark_fn) fn = "test" ml.models.append(model(filename=Path(fn))) @@ -120,7 +120,6 @@ def test_basemodellist_execute(self, model, filename, modelname): (DFoundationsModel, "bm1-1a.foi", "dfoundations/benchmarks"), ], ) - @pytest.mark.skip(reason="Failing after pydantic 2 update. Cause of failure should be investigated in more detail.") def test_basemodellist_execute_remote(self, _, __, model, filename, modelname): # Setup models a = model() @@ -129,7 +128,7 @@ def test_basemodellist_execute_remote(self, _, __, model, filename, modelname): benchmark_fn = input_folder / filename ml = BaseModelList(models=[a, b]) - for i, modelinstance in enumerate(ml.models): + for modelinstance in ml.models: modelinstance.parse(benchmark_fn) fn = "test" ml.models.append(model(filename=Path(fn))) @@ -170,6 +169,7 @@ def test_basemodel_execute_remote(self, _, __, model, filename, modelname): model = modelinstance.execute_remote("/") # no url is needed with the TestClient assert model.output + class TestBool: @pytest.mark.unittest def test_init(self): From 7ac27446e16d7faec4ca88c90147cdeed00cacc3 Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Wed, 17 Jul 2024 08:05:30 +0200 Subject: [PATCH 03/15] chore: typo --- geolib/models/dseries_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geolib/models/dseries_parser.py b/geolib/models/dseries_parser.py index 6ac6238e..2d1a500e 100644 --- a/geolib/models/dseries_parser.py +++ b/geolib/models/dseries_parser.py @@ -43,7 +43,7 @@ class DSeriesStructure(BaseModelStructure): def __init__(self, *args, **kwargs): """The base class for all DSerie structures. - It's parent is a BaseModel from Pydantic, which expects + Its parent is a BaseModel from Pydantic, which expects all its fields named as kwargs. Here we check, using the type annotations stored for each From c3318a4712a1eb2a8bbf1dc3318e6e676158fd0f Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Wed, 17 Jul 2024 14:09:41 +0200 Subject: [PATCH 04/15] chore: small improvement --- geolib/models/base_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geolib/models/base_model.py b/geolib/models/base_model.py index 48139160..9ec77688 100644 --- a/geolib/models/base_model.py +++ b/geolib/models/base_model.py @@ -273,7 +273,7 @@ def execute( process.wait(timeout=timeout_in_seconds) # Iterate over the models - for i, models in enumerate(split_models): + for models in split_models: for model in models: model = model.copy(deep=True) # prevent aliasing output_filename = output_filename_from_input(model) From 627cbb1def9f384e38f94e432d886fc461e02447 Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Wed, 17 Jul 2024 14:53:44 +0200 Subject: [PATCH 05/15] chore: add test to show problem --- tests/models/test_base_model.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/models/test_base_model.py b/tests/models/test_base_model.py index cf487302..227eb465 100644 --- a/tests/models/test_base_model.py +++ b/tests/models/test_base_model.py @@ -19,6 +19,17 @@ class TestBaseModel: + def test_model_dump_issues(self): + a = DSettlementModel(filename="a.txt") + assert a.datastructure.input_data + b = DSettlementModel(filename="b.txt") + ml = BaseModelList(models=[a, b]) + assert ml.models[0].datastructure.input_data + + _dump = ml.model_dump() + with pytest.raises(KeyError): + _dump["models"][0]["datastructure"]["input_data"] + @pytest.fixture def default_base_model(self): return BaseModel() @@ -70,7 +81,7 @@ def test_Model_execute_timeout_after_giventime(self, default_base_model): ) @pytest.mark.acceptance - @only_teamcity + # @only_teamcity @pytest.mark.parametrize( "model,filename,modelname", [ @@ -105,7 +116,7 @@ def test_basemodellist_execute(self, model, filename, modelname): assert fn in output.errors[-1] @pytest.mark.acceptance - @only_teamcity + # @only_teamcity @mock.patch("geolib.models.base_model.requests.post", side_effect=client.post) @mock.patch( "geolib.models.base_model.requests.compat.urljoin", @@ -143,7 +154,7 @@ def test_basemodellist_execute_remote(self, _, __, model, filename, modelname): assert fn in output.errors[-1] @pytest.mark.acceptance - @only_teamcity + # @only_teamcity @mock.patch("geolib.models.base_model.requests.post", side_effect=client.post) @mock.patch( "geolib.models.base_model.requests.compat.urljoin", From 32f701098b29d5b1606f8083017fd0318857e148 Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:42:11 +0200 Subject: [PATCH 06/15] chore: add dummy_model to reproduce --- geolib/dummy_model.py | 51 +++++++++++++++++++++++++++++++++ tests/models/test_base_model.py | 13 +++++---- 2 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 geolib/dummy_model.py diff --git a/geolib/dummy_model.py b/geolib/dummy_model.py new file mode 100644 index 00000000..62af6abf --- /dev/null +++ b/geolib/dummy_model.py @@ -0,0 +1,51 @@ +from pathlib import Path +from typing import List, Optional, Type, Union + +from pydantic import DirectoryPath, FilePath + +from geolib.models.base_model import BaseModel +from geolib.models.base_model_structure import BaseDataClass +from geolib.models.dseries_parser import DSeriesStructure +from geolib.models.dsettlement.dsettlement_parserprovider import ( + DSettlementParserProvider, +) + + +class Results(DSeriesStructure): + dummy: str = "" + + +class DummyInputStructure(DSeriesStructure): + dummy: str = "" + + +class DummyOutputStructure(DSeriesStructure): + results: Results + input_data: DummyInputStructure + + +class DummyStructure(DSeriesStructure): + input_data: DummyInputStructure = DummyInputStructure() + output_data: Optional[Results] = None + + +class DummyModel(BaseModel): + filename: Optional[Path] = None + datastructure: Union[DummyStructure, DummyOutputStructure] = DummyStructure() + + @property + def parser_provider_type(self) -> Type[DSettlementParserProvider]: + pass + + def serialize( + self, filename: Union[FilePath, DirectoryPath, None] + ) -> Union[FilePath, DirectoryPath, None]: + pass + + +class DummyModelList(BaseDataClass): + dummy_models: List[ + DummyModel + ] # Changing this into List[BaseModel] will cause the test to fail + base_models: List[BaseModel] + errors: List[str] = [] diff --git a/tests/models/test_base_model.py b/tests/models/test_base_model.py index 227eb465..6f04189e 100644 --- a/tests/models/test_base_model.py +++ b/tests/models/test_base_model.py @@ -6,6 +6,7 @@ from fastapi.testclient import TestClient from teamcity import is_running_under_teamcity +from geolib.dummy_model import DummyModel, DummyModelList from geolib.models import BaseDataClass, DSettlementModel from geolib.models.base_model import BaseModel, BaseModelList, MetaData from geolib.models.dfoundations.dfoundations_model import DFoundationsModel @@ -20,15 +21,17 @@ class TestBaseModel: def test_model_dump_issues(self): - a = DSettlementModel(filename="a.txt") + a = DummyModel(filename="a.txt") assert a.datastructure.input_data - b = DSettlementModel(filename="b.txt") - ml = BaseModelList(models=[a, b]) - assert ml.models[0].datastructure.input_data + b = DummyModel(filename="b.txt") + ml = DummyModelList(dummy_models=[a, b], base_models=[a, b]) + assert ml.dummy_models[0].datastructure.input_data + assert ml.base_models[0].datastructure.input_data _dump = ml.model_dump() + _dump["dummy_models"][0]["datastructure"]["input_data"] with pytest.raises(KeyError): - _dump["models"][0]["datastructure"]["input_data"] + _dump["base_models"][0]["datastructure"]["input_data"] @pytest.fixture def default_base_model(self): From d1caf2d164ad4ba91794c8ed8344443dfbbe7e1d Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:42:34 +0200 Subject: [PATCH 07/15] chore: add dummy_model to reproduce --- tests/models/test_base_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/models/test_base_model.py b/tests/models/test_base_model.py index 6f04189e..9ae3c263 100644 --- a/tests/models/test_base_model.py +++ b/tests/models/test_base_model.py @@ -29,9 +29,9 @@ def test_model_dump_issues(self): assert ml.base_models[0].datastructure.input_data _dump = ml.model_dump() - _dump["dummy_models"][0]["datastructure"]["input_data"] + _ = _dump["dummy_models"][0]["datastructure"]["input_data"] with pytest.raises(KeyError): - _dump["base_models"][0]["datastructure"]["input_data"] + _ = _dump["base_models"][0]["datastructure"]["input_data"] @pytest.fixture def default_base_model(self): From 01b4d9aa0f557a04d5b3be1989c6230671b6eaab Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Wed, 17 Jul 2024 17:11:25 +0200 Subject: [PATCH 08/15] chore: isolate issue --- geolib/dummy_model.py | 72 +++++++++++++++++++++++++++++---- tests/models/test_base_model.py | 4 ++ 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/geolib/dummy_model.py b/geolib/dummy_model.py index 62af6abf..9d15bf65 100644 --- a/geolib/dummy_model.py +++ b/geolib/dummy_model.py @@ -1,14 +1,67 @@ +import abc from pathlib import Path from typing import List, Optional, Type, Union -from pydantic import DirectoryPath, FilePath +from pydantic import BaseModel, DirectoryPath, FilePath -from geolib.models.base_model import BaseModel -from geolib.models.base_model_structure import BaseDataClass -from geolib.models.dseries_parser import DSeriesStructure +from geolib._compat import IS_PYDANTIC_V2 from geolib.models.dsettlement.dsettlement_parserprovider import ( DSettlementParserProvider, ) +from geolib.models.meta import MetaData + +if IS_PYDANTIC_V2: + from pydantic import ConfigDict +settings = MetaData() + + +class BaseValidator: + def __init__(self, ds): + self.ds = ds + + @property + def is_valid(self) -> bool: + return all( + [ + getattr(self, func)() + for func in dir(self) + if (func.startswith("is_valid_")) + ] + ) + + +class BaseDataClass(BaseModel): + """Base class for *all* pydantic classes in GEOLib.""" + + if IS_PYDANTIC_V2: + model_config = ConfigDict( + validate_assignment=True, + arbitrary_types_allowed=True, + validate_default=True, + extra=settings.extra_fields, + ) + else: + + class Config: + validate_assignment = True + arbitrary_types_allowed = True + validate_all = True + extra = settings.extra_fields + + +class BaseModelStructure(BaseDataClass, abc.ABC): + @property + def is_valid(self) -> bool: + """Validates the current model structure.""" + return self.validator().is_valid + + def validator(self) -> BaseValidator: + """Set the Validator class.""" + return BaseValidator(self) + + +class DSeriesStructure(BaseModelStructure): + pass class Results(DSeriesStructure): @@ -29,7 +82,12 @@ class DummyStructure(DSeriesStructure): output_data: Optional[Results] = None -class DummyModel(BaseModel): +class DummyBaseModel(BaseDataClass, abc.ABC): + filename: Optional[Path] = None + datastructure: Optional[BaseModelStructure] = None + + +class DummyModel(DummyBaseModel): filename: Optional[Path] = None datastructure: Union[DummyStructure, DummyOutputStructure] = DummyStructure() @@ -46,6 +104,6 @@ def serialize( class DummyModelList(BaseDataClass): dummy_models: List[ DummyModel - ] # Changing this into List[BaseModel] will cause the test to fail - base_models: List[BaseModel] + ] # Changing this into List[DummyBaseModel] will cause the test to fail + base_models: List[DummyBaseModel] errors: List[str] = [] diff --git a/tests/models/test_base_model.py b/tests/models/test_base_model.py index 9ae3c263..fdf44b79 100644 --- a/tests/models/test_base_model.py +++ b/tests/models/test_base_model.py @@ -21,6 +21,10 @@ class TestBaseModel: def test_model_dump_issues(self): + """ + Test to reproduce the issue with the model_dump method + If BaseModels are added to the list, the model_dump method will fail + """ a = DummyModel(filename="a.txt") assert a.datastructure.input_data b = DummyModel(filename="b.txt") From 1af962d07f504e3d5dafb0ebff460ef70f5d7f4b Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Wed, 17 Jul 2024 17:21:37 +0200 Subject: [PATCH 09/15] chore: further reduce case --- geolib/dummy_model.py | 37 ++++--------------------------------- 1 file changed, 4 insertions(+), 33 deletions(-) diff --git a/geolib/dummy_model.py b/geolib/dummy_model.py index 9d15bf65..98496209 100644 --- a/geolib/dummy_model.py +++ b/geolib/dummy_model.py @@ -5,9 +5,6 @@ from pydantic import BaseModel, DirectoryPath, FilePath from geolib._compat import IS_PYDANTIC_V2 -from geolib.models.dsettlement.dsettlement_parserprovider import ( - DSettlementParserProvider, -) from geolib.models.meta import MetaData if IS_PYDANTIC_V2: @@ -60,26 +57,9 @@ def validator(self) -> BaseValidator: return BaseValidator(self) -class DSeriesStructure(BaseModelStructure): - pass - - -class Results(DSeriesStructure): - dummy: str = "" - - -class DummyInputStructure(DSeriesStructure): - dummy: str = "" - - -class DummyOutputStructure(DSeriesStructure): - results: Results - input_data: DummyInputStructure - - -class DummyStructure(DSeriesStructure): - input_data: DummyInputStructure = DummyInputStructure() - output_data: Optional[Results] = None +class DummyStructure(BaseModelStructure): + input_data: BaseModelStructure = BaseModelStructure() + output_data: Optional[BaseModelStructure] = None class DummyBaseModel(BaseDataClass, abc.ABC): @@ -89,16 +69,7 @@ class DummyBaseModel(BaseDataClass, abc.ABC): class DummyModel(DummyBaseModel): filename: Optional[Path] = None - datastructure: Union[DummyStructure, DummyOutputStructure] = DummyStructure() - - @property - def parser_provider_type(self) -> Type[DSettlementParserProvider]: - pass - - def serialize( - self, filename: Union[FilePath, DirectoryPath, None] - ) -> Union[FilePath, DirectoryPath, None]: - pass + datastructure: DummyStructure = DummyStructure() class DummyModelList(BaseDataClass): From 420878ff8a995db70c292f162506a2bf764fa2c0 Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Fri, 19 Jul 2024 09:39:57 +0200 Subject: [PATCH 10/15] chore: further reduce case --- geolib/dummy_model.py | 45 +++++++++++++++---------------------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/geolib/dummy_model.py b/geolib/dummy_model.py index 98496209..41fa8ed3 100644 --- a/geolib/dummy_model.py +++ b/geolib/dummy_model.py @@ -1,8 +1,8 @@ import abc from pathlib import Path -from typing import List, Optional, Type, Union +from typing import List, Optional, Union -from pydantic import BaseModel, DirectoryPath, FilePath +from pydantic import BaseModel from geolib._compat import IS_PYDANTIC_V2 from geolib.models.meta import MetaData @@ -11,20 +11,7 @@ from pydantic import ConfigDict settings = MetaData() - -class BaseValidator: - def __init__(self, ds): - self.ds = ds - - @property - def is_valid(self) -> bool: - return all( - [ - getattr(self, func)() - for func in dir(self) - if (func.startswith("is_valid_")) - ] - ) +### BASE MODEL STRUCTURE class BaseDataClass(BaseModel): @@ -47,14 +34,17 @@ class Config: class BaseModelStructure(BaseDataClass, abc.ABC): - @property - def is_valid(self) -> bool: - """Validates the current model structure.""" - return self.validator().is_valid + pass + + +class GeolibBaseModel(BaseDataClass, abc.ABC): + filename: Optional[Path] = None + datastructure: Optional[ + Union[BaseModelStructure] + ] = None # Adding DummyStructure in Union here would cause circular dependencies in the real application + - def validator(self) -> BaseValidator: - """Set the Validator class.""" - return BaseValidator(self) +### DUMMY MODEL STRUCTURE class DummyStructure(BaseModelStructure): @@ -62,12 +52,7 @@ class DummyStructure(BaseModelStructure): output_data: Optional[BaseModelStructure] = None -class DummyBaseModel(BaseDataClass, abc.ABC): - filename: Optional[Path] = None - datastructure: Optional[BaseModelStructure] = None - - -class DummyModel(DummyBaseModel): +class DummyModel(GeolibBaseModel): filename: Optional[Path] = None datastructure: DummyStructure = DummyStructure() @@ -76,5 +61,5 @@ class DummyModelList(BaseDataClass): dummy_models: List[ DummyModel ] # Changing this into List[DummyBaseModel] will cause the test to fail - base_models: List[DummyBaseModel] + base_models: List[GeolibBaseModel] errors: List[str] = [] From b9d1109a7b749b28d261a41c0ab973d55e5ff2fe Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Fri, 19 Jul 2024 09:45:46 +0200 Subject: [PATCH 11/15] chore: resturcture imports/classes --- geolib/__init__.py | 3 - geolib/models/__init__.py | 3 +- geolib/models/base_data_class.py | 29 ++++ geolib/models/base_model.py | 130 +---------------- geolib/models/base_model_list.py | 131 ++++++++++++++++++ geolib/models/base_model_structure.py | 30 +--- geolib/models/dgeoflow/internal.py | 52 ++----- .../models/dsettlement/dsettlement_model.py | 2 +- geolib/models/dsettlement/internal.py | 3 +- tests/models/dsheetpiling/test_bug_fixes.py | 12 +- .../test_dstability_state_correlations.py | 2 +- tests/models/test_base_model.py | 3 +- tests/service/test_service.py | 2 +- 13 files changed, 196 insertions(+), 206 deletions(-) create mode 100644 geolib/models/base_data_class.py create mode 100644 geolib/models/base_model_list.py diff --git a/geolib/__init__.py b/geolib/__init__.py index fc0f3ee9..11c8047c 100644 --- a/geolib/__init__.py +++ b/geolib/__init__.py @@ -3,6 +3,3 @@ """ __version__ = "2.3.0" - -from . import utils -from .models import * diff --git a/geolib/models/__init__.py b/geolib/models/__init__.py index d3f6f7cd..c5eb5115 100644 --- a/geolib/models/__init__.py +++ b/geolib/models/__init__.py @@ -3,7 +3,8 @@ """ from .base_model_structure import BaseDataClass, BaseModelStructure # isort:skip -from .base_model import BaseModel, BaseModelList +from .base_model import BaseModel +from .base_model_list import BaseModelList from .dfoundations import DFoundationsModel from .dsettlement import DSettlementModel from .dsheetpiling import DSheetPilingModel diff --git a/geolib/models/base_data_class.py b/geolib/models/base_data_class.py new file mode 100644 index 00000000..500c2ef4 --- /dev/null +++ b/geolib/models/base_data_class.py @@ -0,0 +1,29 @@ +from pydantic import BaseModel + +from geolib._compat import IS_PYDANTIC_V2 + +if IS_PYDANTIC_V2: + from pydantic import ConfigDict + +from .meta import MetaData + +settings = MetaData() + + +class BaseDataClass(BaseModel): + """Base class for *all* pydantic classes in GEOLib.""" + + if IS_PYDANTIC_V2: + model_config = ConfigDict( + validate_assignment=True, + arbitrary_types_allowed=True, + validate_default=True, + extra=settings.extra_fields, + ) + else: + + class Config: + validate_assignment = True + arbitrary_types_allowed = True + validate_all = True + extra = settings.extra_fields diff --git a/geolib/models/base_model.py b/geolib/models/base_model.py index 9ec77688..67e30a06 100644 --- a/geolib/models/base_model.py +++ b/geolib/models/base_model.py @@ -5,15 +5,13 @@ """ import abc import logging -import os from abc import abstractmethod, abstractproperty -from pathlib import Path, PosixPath, WindowsPath -from subprocess import Popen, run -from types import CoroutineType +from pathlib import Path +from subprocess import run from typing import List, Optional, Type, Union import requests -from pydantic import DirectoryPath, FilePath, HttpUrl, conlist +from pydantic import DirectoryPath, FilePath, HttpUrl from geolib._compat import IS_PYDANTIC_V2 @@ -26,10 +24,9 @@ from geolib.errors import CalculationError from geolib.models import BaseDataClass - -from .base_model_structure import BaseModelStructure -from .meta import MetaData -from .parsers import BaseParserProvider +from geolib.models.base_model_structure import BaseModelStructure +from geolib.models.meta import MetaData +from geolib.models.parsers import BaseParserProvider logger = logging.getLogger(__name__) meta = MetaData() @@ -209,121 +206,6 @@ def set_meta_property(self, key: str, value: str) -> None: raise ValueError(f"Metadata property {key} does not exist.") -class BaseModelList(BaseDataClass): - """Hold multiple models that can be executed in parallel. - - Note that all models need to have a unique filename - otherwise they will overwrite eachother. This also helps with - identifying them later.""" - - models: List[BaseModel] - errors: List[str] = [] - - def execute( - self, - calculation_folder: DirectoryPath, - timeout_in_seconds: int = meta.timeout, - nprocesses: Optional[int] = os.cpu_count(), - ) -> "BaseModelList": - """Execute all models in this class in parallel. - - We split the list to separate folders and call a batch processes on each folder. - Note that the order of models will change. - """ - - # manual check as remote execution could result in zero models - if len(self.models) == 0: - raise ValueError("Can't execute with zero models.") - - lead_model = self.models[0] - processes = [] - output_models = [] - errors = [] - - # Divide the models over n processes and make sure to copy them to prevent aliasing - split_models = [self.models[i::nprocesses] for i in range(nprocesses)] - for i, models in enumerate(split_models): - if len(models) == 0: - continue - unique_folder = calculation_folder / str(i) - unique_folder.mkdir(parents=True, exist_ok=True) - - for model in models: - fn = unique_folder / model.filename.name - model.serialize(fn.resolve()) - - executable = meta.console_folder / lead_model.default_console_path - if not executable.exists(): - logger.error( - f"Please make sure the `geolib.env` file points to the console folder. GEOLib now can't find it at `{executable}`" - ) - raise CalculationError( - -1, f"Console executable not found at {executable}." - ) - - process = Popen( - [str(executable)] + lead_model.console_flags + [str(i)], - cwd=str(calculation_folder.resolve()), - ) - processes.append(process) - - # Wait for all processes to be done - for process in processes: - logger.debug(f"Executed with {process.args}") - process.wait(timeout=timeout_in_seconds) - - # Iterate over the models - for models in split_models: - for model in models: - model = model.copy(deep=True) # prevent aliasing - output_filename = output_filename_from_input(model) - if output_filename.exists(): - try: - model.parse(output_filename) - output_models.append(model) - - except ValidationError: - logger.warning( - f"Ouput file generated but parsing of {output_filename.name} failed." - ) - error = model.get_error_context() - errors.append(error) - else: - logger.warning( - f"Model @ {output_filename.name} failed. Please check the .err file and batchlog.txt in its folder." - ) - error = model.get_error_context() - errors.append(error) - - return self.__class__(models=output_models, errors=errors) - - def execute_remote(self, endpoint: HttpUrl) -> "BaseModelList": - """Execute all models in this class in parallel on a remote endpoint. - - Note that the order of models will change. - """ - lead_model = self.models[0] - - response = requests.post( - requests.compat.urljoin( - endpoint, f"calculate/{lead_model.__class__.__name__.lower()}s" - ), - data="[" + ",".join((model.json() for model in self.models)) + "]", - auth=HTTPBasicAuth(meta.gl_username, meta.gl_password), - ) - if response.status_code == 200: - models = response.json()["models"] - errors = response.json()["errors"] - stripped_models = [] - for model in models: - # remove possibly invalid external metadata - model.get("meta", {}).pop("console_folder", None) - stripped_models.append(lead_model.__class__(**model)) - return self.__class__(models=stripped_models, errors=errors) - else: - raise CalculationError(response.status_code, response.text) - - def output_filename_from_input(model: BaseModel, extension: str = None) -> Path: if not extension: extension = model.parser_provider_type().output_parsers[-1].suffix_list[0] diff --git a/geolib/models/base_model_list.py b/geolib/models/base_model_list.py new file mode 100644 index 00000000..d590010c --- /dev/null +++ b/geolib/models/base_model_list.py @@ -0,0 +1,131 @@ +import logging +import os +from subprocess import Popen +from typing import List, Optional + +import requests +from pydantic import DirectoryPath, HttpUrl +from requests.auth import HTTPBasicAuth + +from geolib.errors import CalculationError +from geolib.models import meta +from geolib.models.base_data_class import BaseDataClass +from geolib.models.base_model import BaseModel, output_filename_from_input + +logger = logging.getLogger(__name__) +meta = meta.MetaData() + + +class BaseModelList(BaseDataClass): + """Hold multiple models that can be executed in parallel. + + Note that all models need to have a unique filename + otherwise they will overwrite eachother. This also helps with + identifying them later.""" + + models: List[BaseModel] + errors: List[str] = [] + + def execute( + self, + calculation_folder: DirectoryPath, + timeout_in_seconds: int = meta.timeout, + nprocesses: Optional[int] = os.cpu_count(), + ) -> "BaseModelList": + """Execute all models in this class in parallel. + + We split the list to separate folders and call a batch processes on each folder. + Note that the order of models will change. + """ + + # manual check as remote execution could result in zero models + if len(self.models) == 0: + raise ValueError("Can't execute with zero models.") + + lead_model = self.models[0] + processes = [] + output_models = [] + errors = [] + + # Divide the models over n processes and make sure to copy them to prevent aliasing + split_models = [self.models[i::nprocesses] for i in range(nprocesses)] + for i, models in enumerate(split_models): + if len(models) == 0: + continue + unique_folder = calculation_folder / str(i) + unique_folder.mkdir(parents=True, exist_ok=True) + + for model in models: + fn = unique_folder / model.filename.name + model.serialize(fn.resolve()) + + executable = meta.console_folder / lead_model.default_console_path + if not executable.exists(): + logger.error( + f"Please make sure the `geolib.env` file points to the console folder. GEOLib now can't find it at `{executable}`" + ) + raise CalculationError( + -1, f"Console executable not found at {executable}." + ) + + process = Popen( + [str(executable)] + lead_model.console_flags + [str(i)], + cwd=str(calculation_folder.resolve()), + ) + processes.append(process) + + # Wait for all processes to be done + for process in processes: + logger.debug(f"Executed with {process.args}") + process.wait(timeout=timeout_in_seconds) + + # Iterate over the models + for models in split_models: + for model in models: + model = model.copy(deep=True) # prevent aliasing + output_filename = output_filename_from_input(model) + if output_filename.exists(): + try: + model.parse(output_filename) + output_models.append(model) + + except ValidationError: + logger.warning( + f"Ouput file generated but parsing of {output_filename.name} failed." + ) + error = model.get_error_context() + errors.append(error) + else: + logger.warning( + f"Model @ {output_filename.name} failed. Please check the .err file and batchlog.txt in its folder." + ) + error = model.get_error_context() + errors.append(error) + + return self.__class__(models=output_models, errors=errors) + + def execute_remote(self, endpoint: HttpUrl) -> "BaseModelList": + """Execute all models in this class in parallel on a remote endpoint. + + Note that the order of models will change. + """ + lead_model = self.models[0] + + response = requests.post( + requests.compat.urljoin( + endpoint, f"calculate/{lead_model.__class__.__name__.lower()}s" + ), + data="[" + ",".join((model.json() for model in self.models)) + "]", + auth=HTTPBasicAuth(meta.gl_username, meta.gl_password), + ) + if response.status_code == 200: + models = response.json()["models"] + errors = response.json()["errors"] + stripped_models = [] + for model in models: + # remove possibly invalid external metadata + model.get("meta", {}).pop("console_folder", None) + stripped_models.append(lead_model.__class__(**model)) + return self.__class__(models=stripped_models, errors=errors) + else: + raise CalculationError(response.status_code, response.text) diff --git a/geolib/models/base_model_structure.py b/geolib/models/base_model_structure.py index c6e50896..dff79417 100644 --- a/geolib/models/base_model_structure.py +++ b/geolib/models/base_model_structure.py @@ -1,37 +1,9 @@ import abc -from math import isfinite -from pydantic import BaseModel +from geolib.models.base_data_class import BaseDataClass -from geolib._compat import IS_PYDANTIC_V2 - -if IS_PYDANTIC_V2: - from pydantic import ConfigDict - -from .meta import MetaData from .validators import BaseValidator -settings = MetaData() - - -class BaseDataClass(BaseModel): - """Base class for *all* pydantic classes in GEOLib.""" - - if IS_PYDANTIC_V2: - model_config = ConfigDict( - validate_assignment=True, - arbitrary_types_allowed=True, - validate_default=True, - extra=settings.extra_fields, - ) - else: - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - validate_all = True - extra = settings.extra_fields - class BaseModelStructure(BaseDataClass, abc.ABC): @property diff --git a/geolib/models/dgeoflow/internal.py b/geolib/models/dgeoflow/internal.py index 02d04490..c8ef8596 100644 --- a/geolib/models/dgeoflow/internal.py +++ b/geolib/models/dgeoflow/internal.py @@ -17,9 +17,9 @@ else: from pydantic import conlist, root_validator, validator -from geolib import BaseModelStructure from geolib import __version__ as version from geolib.geometry import Point +from geolib.models import BaseModelStructure from geolib.soils import Soil, StorageParameters from geolib.utils import snake_to_camel @@ -38,6 +38,7 @@ def dict(_, *args, **kwargs): for k, v in data.items() } + def transform_id_to_str(value) -> str: if value is None: return None @@ -90,9 +91,7 @@ class PersistableSoilVisualization(DGeoFlowBaseModelStructure): SoilId: Optional[str] if IS_PYDANTIC_V2: - id_validator = field_validator("SoilId", mode="before")( - transform_id_to_str - ) + id_validator = field_validator("SoilId", mode="before")(transform_id_to_str) class SoilVisualisation(DGeoFlowBaseModelStructure): @@ -130,9 +129,7 @@ def structure_group(cls) -> str: SoilLayers: List[PersistableSoilLayer] = [] if IS_PYDANTIC_V2: - id_validator = field_validator("Id", mode="before")( - transform_id_to_str - ) + id_validator = field_validator("Id", mode="before")(transform_id_to_str) def add_soillayer(self, layer_id: str, soil_id: str) -> PersistableSoilLayer: psl = PersistableSoilLayer(LayerId=layer_id, SoilId=soil_id) @@ -158,9 +155,7 @@ class PersistableSoil(DGeoFlowBaseModelStructure): VerticalPermeability: float = 0.001 if IS_PYDANTIC_V2: - id_validator = field_validator("Id", mode="before")( - transform_id_to_str - ) + id_validator = field_validator("Id", mode="before")(transform_id_to_str) class SoilCollection(DGeoFlowSubStructure): @@ -419,9 +414,7 @@ class PersistableLayer(DGeoFlowBaseModelStructure): Points: conlist(PersistablePoint, min_items=3) if IS_PYDANTIC_V2: - id_validator = field_validator("Id", mode="before")( - transform_id_to_str - ) + id_validator = field_validator("Id", mode="before")(transform_id_to_str) @classmethod def polygon_checks(cls, points): @@ -462,9 +455,7 @@ def structure_name(cls) -> str: Layers: List[PersistableLayer] = [] if IS_PYDANTIC_V2: - id_validator = field_validator("Id", mode="before")( - transform_id_to_str - ) + id_validator = field_validator("Id", mode="before")(transform_id_to_str) def contains_point(self, point: Point) -> bool: """ @@ -537,9 +528,7 @@ class PersistableBoundaryCondition(DGeoFlowBaseModelStructure): FixedHeadBoundaryConditionProperties: PersistableFixedHeadBoundaryConditionProperties if IS_PYDANTIC_V2: - id_validator = field_validator("Id", mode="before")( - transform_id_to_str - ) + id_validator = field_validator("Id", mode="before")(transform_id_to_str) class BoundaryConditionCollection(DGeoFlowSubStructure): @@ -550,9 +539,7 @@ class BoundaryConditionCollection(DGeoFlowSubStructure): BoundaryConditions: List[PersistableBoundaryCondition] = [] if IS_PYDANTIC_V2: - id_validator = field_validator("Id", mode="before")( - transform_id_to_str - ) + id_validator = field_validator("Id", mode="before")(transform_id_to_str) @classmethod def structure_group(cls) -> str: @@ -666,9 +653,7 @@ class GroundwaterFlowResult(DGeoFlowSubStructure): ContentVersion: Optional[str] = "2" if IS_PYDANTIC_V2: - id_validator = field_validator("Id", mode="before")( - transform_id_to_str - ) + id_validator = field_validator("Id", mode="before")(transform_id_to_str) @classmethod def structure_group(cls) -> str: @@ -683,9 +668,7 @@ class PipeLengthResult(DGeoFlowSubStructure): ContentVersion: Optional[str] = "2" if IS_PYDANTIC_V2: - id_validator = field_validator("Id", mode="before")( - transform_id_to_str - ) + id_validator = field_validator("Id", mode="before")(transform_id_to_str) @classmethod def structure_group(cls) -> str: @@ -701,9 +684,7 @@ class CriticalHeadResult(DGeoFlowSubStructure): ContentVersion: Optional[str] = "2" if IS_PYDANTIC_V2: - id_validator = field_validator("Id", mode="before")( - transform_id_to_str - ) + id_validator = field_validator("Id", mode="before")(transform_id_to_str) @classmethod def structure_group(cls) -> str: @@ -765,9 +746,7 @@ class PersistableMeshProperties(DGeoFlowBaseModelStructure): ElementSize: Optional[float] = 1 if IS_PYDANTIC_V2: - id_validator = field_validator("LayerId", mode="before")( - transform_id_to_str - ) + id_validator = field_validator("LayerId", mode="before")(transform_id_to_str) class MeshProperty(DGeoFlowSubStructure): @@ -778,9 +757,7 @@ class MeshProperty(DGeoFlowSubStructure): MeshProperties: Optional[List[PersistableMeshProperties]] = [] if IS_PYDANTIC_V2: - id_validator = field_validator("Id", mode="before")( - transform_id_to_str - ) + id_validator = field_validator("Id", mode="before")(transform_id_to_str) @classmethod def structure_name(cls) -> str: @@ -883,6 +860,7 @@ def list_has_id(values, id): if entry.Id == id: return True return False + # print(self) for _, scenario in enumerate(self.scenarios): for _, stage in enumerate(scenario.Stages): diff --git a/geolib/models/dsettlement/dsettlement_model.py b/geolib/models/dsettlement/dsettlement_model.py index f0c22482..304dadab 100644 --- a/geolib/models/dsettlement/dsettlement_model.py +++ b/geolib/models/dsettlement/dsettlement_model.py @@ -9,7 +9,7 @@ from pydantic.types import PositiveInt, confloat, conint, constr from geolib.geometry import Point -from geolib.models import BaseDataClass, BaseModel, BaseModelStructure +from geolib.models.base_model import BaseModel from geolib.models.dsettlement.internal_soil import SoilInternal from geolib.models.dsettlement.loads import ( CircularLoad, diff --git a/geolib/models/dsettlement/internal.py b/geolib/models/dsettlement/internal.py index 640148bf..eb9ba32d 100644 --- a/geolib/models/dsettlement/internal.py +++ b/geolib/models/dsettlement/internal.py @@ -3,14 +3,13 @@ from inspect import cleandoc from math import isclose from operator import attrgetter -from typing import Any, Dict, List, Optional, Tuple, Type, Union +from typing import Dict, List, Optional, Tuple, Type, Union from pydantic.types import PositiveInt, confloat, conint, conlist, constr from geolib._compat import IS_PYDANTIC_V2 from geolib.geometry.one import Point from geolib.models import BaseDataClass -from geolib.models.base_model_structure import BaseModelStructure from geolib.models.dseries_parser import ( DSerieListStructure, DSerieMatrixStructure, diff --git a/tests/models/dsheetpiling/test_bug_fixes.py b/tests/models/dsheetpiling/test_bug_fixes.py index 12efdd16..8a732524 100644 --- a/tests/models/dsheetpiling/test_bug_fixes.py +++ b/tests/models/dsheetpiling/test_bug_fixes.py @@ -24,7 +24,7 @@ def test_consistent_model_setting(self): # should be consistent with what is written in the input file # 1. Build model. - model = gl.DSheetPilingModel() + model = gl.models.DSheetPilingModel() test_folder = Path(TestUtils.get_output_test_data_dir(test_file_directory)) output_test_file = test_folder / Path("test_consistent_model_setting.shi") @@ -59,7 +59,7 @@ def test_geolib191(self): # The expectation is that the load is defined once but added to all the stages. # 1. Build model. - model = gl.DSheetPilingModel() + model = gl.models.DSheetPilingModel() test_folder = Path(TestUtils.get_output_test_data_dir(test_file_directory)) output_test_file = test_folder / Path("GEOLIB191.shi") @@ -140,7 +140,7 @@ def test_geolib191(self): def test_parse_field_verify_sheet_piling_calculation_type_standard(self): # Former test of test_geolib_173 # 1. Define test data - model = gl.DSheetPilingModel() + model = gl.models.DSheetPilingModel() test_folder = Path(TestUtils.get_local_test_data_dir(benchmarks_directory)) test_file = test_folder / Path("bm4-5a.shd") @@ -160,7 +160,7 @@ def test_parse_field_verify_sheet_piling_calculation_type_allowable_anchor_force ): # Former test of test_geolib_173 # 1. Define test data - model = gl.DSheetPilingModel() + model = gl.models.DSheetPilingModel() test_folder = Path(TestUtils.get_local_test_data_dir(benchmarks_directory)) test_file = test_folder / Path("bm3-1b.shd") @@ -178,7 +178,7 @@ def test_parse_field_verify_sheet_piling_calculation_type_allowable_anchor_force def test_parse_field_verify_sheet_piling_according_to_cur_method_a(self): # Former test of test_geolib_173 # 1. Define test data - model = gl.DSheetPilingModel() + model = gl.models.DSheetPilingModel() test_folder = Path(TestUtils.get_local_test_data_dir(benchmarks_directory)) test_file = test_folder / Path("bm3-1d.shd") @@ -203,7 +203,7 @@ def test_parse_field_verify_sheet_piling_according_to_cur_method_a(self): def test_parse_field_verify_sheet_piling_according_to_cur_method_b(self): # Former test of test_geolib_173 # 1. Define test data - model = gl.DSheetPilingModel() + model = gl.models.DSheetPilingModel() test_folder = Path(TestUtils.get_local_test_data_dir(benchmarks_directory)) test_file = test_folder / Path("bm4-17a.shd") diff --git a/tests/models/dstability/test_dstability_state_correlations.py b/tests/models/dstability/test_dstability_state_correlations.py index 3c4cad1c..f5d4f745 100644 --- a/tests/models/dstability/test_dstability_state_correlations.py +++ b/tests/models/dstability/test_dstability_state_correlations.py @@ -1,7 +1,7 @@ import pytest -from geolib import DStabilityModel from geolib.geometry import Point +from geolib.models import DStabilityModel from geolib.models.dstability.internal import StateCorrelation from geolib.models.dstability.states import DStabilityStatePoint, DStabilityStress diff --git a/tests/models/test_base_model.py b/tests/models/test_base_model.py index fdf44b79..4af13b5a 100644 --- a/tests/models/test_base_model.py +++ b/tests/models/test_base_model.py @@ -8,7 +8,8 @@ from geolib.dummy_model import DummyModel, DummyModelList from geolib.models import BaseDataClass, DSettlementModel -from geolib.models.base_model import BaseModel, BaseModelList, MetaData +from geolib.models.base_model import BaseModel, MetaData +from geolib.models.base_model_list import BaseModelList from geolib.models.dfoundations.dfoundations_model import DFoundationsModel from geolib.models.dsheetpiling.dsheetpiling_model import DSheetPilingModel from geolib.models.dstability.dstability_model import DStabilityModel diff --git a/tests/service/test_service.py b/tests/service/test_service.py index 81a4831a..033dbd8d 100644 --- a/tests/service/test_service.py +++ b/tests/service/test_service.py @@ -11,7 +11,7 @@ from fastapi.testclient import TestClient from requests.auth import HTTPBasicAuth -from geolib import BaseModelList, DFoundationsModel, DSettlementModel +from geolib.models import BaseModelList, DFoundationsModel, DSettlementModel from geolib.service.main import app from tests.utils import TestUtils, only_teamcity From db2a8a559ba67c645c08ee471873a2ff02462be1 Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Fri, 19 Jul 2024 14:29:17 +0200 Subject: [PATCH 12/15] chore: replace abstractproperty --- geolib/models/base_model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/geolib/models/base_model.py b/geolib/models/base_model.py index 67e30a06..9dabcf54 100644 --- a/geolib/models/base_model.py +++ b/geolib/models/base_model.py @@ -5,7 +5,7 @@ """ import abc import logging -from abc import abstractmethod, abstractproperty +from abc import abstractmethod from pathlib import Path from subprocess import run from typing import List, Optional, Type, Union @@ -150,7 +150,8 @@ def console_flags(self) -> List[str]: def console_flags_post(self) -> List[str]: return [] - @abstractproperty + @property + @abstractmethod def parser_provider_type(self) -> Type[BaseParserProvider]: """Returns the parser provider type of the current concrete class. From 00902967180446794700cfca660f8b4fbbc4f495 Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Fri, 19 Jul 2024 15:24:17 +0200 Subject: [PATCH 13/15] chore: fix core of the issue --- geolib/models/base_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geolib/models/base_model.py b/geolib/models/base_model.py index 9dabcf54..d2ea31ed 100644 --- a/geolib/models/base_model.py +++ b/geolib/models/base_model.py @@ -11,7 +11,7 @@ from typing import List, Optional, Type, Union import requests -from pydantic import DirectoryPath, FilePath, HttpUrl +from pydantic import DirectoryPath, FilePath, HttpUrl, SerializeAsAny from geolib._compat import IS_PYDANTIC_V2 @@ -34,7 +34,7 @@ class BaseModel(BaseDataClass, abc.ABC): filename: Optional[Path] = None - datastructure: Optional[BaseModelStructure] = None + datastructure: Optional[SerializeAsAny[BaseModelStructure]] = None def execute(self, timeout_in_seconds: int = meta.timeout) -> "BaseModel": """Execute a Model and wait for `timeout` seconds. From eaa23680432a601fc1ef09cdb14b92473e5b3c96 Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Fri, 19 Jul 2024 15:24:30 +0200 Subject: [PATCH 14/15] chore: adjust dummy case accordingly --- geolib/dummy_model.py | 4 ++-- tests/models/test_base_model.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/geolib/dummy_model.py b/geolib/dummy_model.py index 41fa8ed3..229755ad 100644 --- a/geolib/dummy_model.py +++ b/geolib/dummy_model.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import List, Optional, Union -from pydantic import BaseModel +from pydantic import BaseModel, SerializeAsAny from geolib._compat import IS_PYDANTIC_V2 from geolib.models.meta import MetaData @@ -40,7 +40,7 @@ class BaseModelStructure(BaseDataClass, abc.ABC): class GeolibBaseModel(BaseDataClass, abc.ABC): filename: Optional[Path] = None datastructure: Optional[ - Union[BaseModelStructure] + SerializeAsAny[BaseModelStructure] ] = None # Adding DummyStructure in Union here would cause circular dependencies in the real application diff --git a/tests/models/test_base_model.py b/tests/models/test_base_model.py index 4af13b5a..2bd32d5b 100644 --- a/tests/models/test_base_model.py +++ b/tests/models/test_base_model.py @@ -35,8 +35,7 @@ def test_model_dump_issues(self): _dump = ml.model_dump() _ = _dump["dummy_models"][0]["datastructure"]["input_data"] - with pytest.raises(KeyError): - _ = _dump["base_models"][0]["datastructure"]["input_data"] + _ = _dump["base_models"][0]["datastructure"]["input_data"] @pytest.fixture def default_base_model(self): From 46206890b76fdad89f36f6408eda43f4a1340438 Mon Sep 17 00:00:00 2001 From: Ardt Klapwijk <59741981+ArdtK@users.noreply.github.com> Date: Mon, 22 Jul 2024 09:12:32 +0200 Subject: [PATCH 15/15] chore: remove dummy model with tests --- geolib/dummy_model.py | 65 --------------------------------- tests/models/test_base_model.py | 24 ++---------- 2 files changed, 3 insertions(+), 86 deletions(-) delete mode 100644 geolib/dummy_model.py diff --git a/geolib/dummy_model.py b/geolib/dummy_model.py deleted file mode 100644 index 229755ad..00000000 --- a/geolib/dummy_model.py +++ /dev/null @@ -1,65 +0,0 @@ -import abc -from pathlib import Path -from typing import List, Optional, Union - -from pydantic import BaseModel, SerializeAsAny - -from geolib._compat import IS_PYDANTIC_V2 -from geolib.models.meta import MetaData - -if IS_PYDANTIC_V2: - from pydantic import ConfigDict -settings = MetaData() - -### BASE MODEL STRUCTURE - - -class BaseDataClass(BaseModel): - """Base class for *all* pydantic classes in GEOLib.""" - - if IS_PYDANTIC_V2: - model_config = ConfigDict( - validate_assignment=True, - arbitrary_types_allowed=True, - validate_default=True, - extra=settings.extra_fields, - ) - else: - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - validate_all = True - extra = settings.extra_fields - - -class BaseModelStructure(BaseDataClass, abc.ABC): - pass - - -class GeolibBaseModel(BaseDataClass, abc.ABC): - filename: Optional[Path] = None - datastructure: Optional[ - SerializeAsAny[BaseModelStructure] - ] = None # Adding DummyStructure in Union here would cause circular dependencies in the real application - - -### DUMMY MODEL STRUCTURE - - -class DummyStructure(BaseModelStructure): - input_data: BaseModelStructure = BaseModelStructure() - output_data: Optional[BaseModelStructure] = None - - -class DummyModel(GeolibBaseModel): - filename: Optional[Path] = None - datastructure: DummyStructure = DummyStructure() - - -class DummyModelList(BaseDataClass): - dummy_models: List[ - DummyModel - ] # Changing this into List[DummyBaseModel] will cause the test to fail - base_models: List[GeolibBaseModel] - errors: List[str] = [] diff --git a/tests/models/test_base_model.py b/tests/models/test_base_model.py index 2bd32d5b..de5c3c66 100644 --- a/tests/models/test_base_model.py +++ b/tests/models/test_base_model.py @@ -4,9 +4,7 @@ import pytest from fastapi.testclient import TestClient -from teamcity import is_running_under_teamcity -from geolib.dummy_model import DummyModel, DummyModelList from geolib.models import BaseDataClass, DSettlementModel from geolib.models.base_model import BaseModel, MetaData from geolib.models.base_model_list import BaseModelList @@ -21,22 +19,6 @@ class TestBaseModel: - def test_model_dump_issues(self): - """ - Test to reproduce the issue with the model_dump method - If BaseModels are added to the list, the model_dump method will fail - """ - a = DummyModel(filename="a.txt") - assert a.datastructure.input_data - b = DummyModel(filename="b.txt") - ml = DummyModelList(dummy_models=[a, b], base_models=[a, b]) - assert ml.dummy_models[0].datastructure.input_data - assert ml.base_models[0].datastructure.input_data - - _dump = ml.model_dump() - _ = _dump["dummy_models"][0]["datastructure"]["input_data"] - _ = _dump["base_models"][0]["datastructure"]["input_data"] - @pytest.fixture def default_base_model(self): return BaseModel() @@ -88,7 +70,7 @@ def test_Model_execute_timeout_after_giventime(self, default_base_model): ) @pytest.mark.acceptance - # @only_teamcity + @only_teamcity @pytest.mark.parametrize( "model,filename,modelname", [ @@ -123,7 +105,7 @@ def test_basemodellist_execute(self, model, filename, modelname): assert fn in output.errors[-1] @pytest.mark.acceptance - # @only_teamcity + @only_teamcity @mock.patch("geolib.models.base_model.requests.post", side_effect=client.post) @mock.patch( "geolib.models.base_model.requests.compat.urljoin", @@ -161,7 +143,7 @@ def test_basemodellist_execute_remote(self, _, __, model, filename, modelname): assert fn in output.errors[-1] @pytest.mark.acceptance - # @only_teamcity + @only_teamcity @mock.patch("geolib.models.base_model.requests.post", side_effect=client.post) @mock.patch( "geolib.models.base_model.requests.compat.urljoin",