diff --git a/src/ert/dark_storage/app.py b/src/ert/dark_storage/app.py index d093856470b..0ec82fe7ace 100644 --- a/src/ert/dark_storage/app.py +++ b/src/ert/dark_storage/app.py @@ -1,21 +1,49 @@ -from ert_storage.app import JSONResponse -from ert_storage.app import app as ert_storage_app -from ert_storage.exceptions import ErtStorageError -from fastapi import FastAPI, Request, status -from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html -from fastapi.responses import HTMLResponse, RedirectResponse +import json +from enum import Enum +from typing import Any + +from fastapi import FastAPI, Request, Response, status +from fastapi.responses import RedirectResponse from ert.dark_storage.endpoints import router as endpoints_router +from ert.dark_storage.exceptions import ErtStorageError +from ert.shared import __version__ + + +class JSONEncoder(json.JSONEncoder): + """ + Custom JSON encoder with support for Python 3.4 enums + """ + + def default(self, obj: Any) -> Any: + if isinstance(obj, Enum): + return obj.name + return super().default(obj) + + +class JSONResponse(Response): + """A replacement for Starlette's JSONResponse that permits NaNs.""" + + media_type = "application/json" + + def render(self, content: Any) -> bytes: + return ( + JSONEncoder( + ensure_ascii=False, + allow_nan=True, + indent=None, + separators=(",", ":"), + ) + .encode(content) + .encode("utf-8") + ) + app = FastAPI( - title=ert_storage_app.title, - version=ert_storage_app.version, + title="ERT Storage API (dark storage)", + version=__version__, debug=True, default_response_class=JSONResponse, - # Disable documentation so we can replace it with ERT Storage's later - openapi_url=None, - docs_url=None, - redoc_url=None, ) @@ -51,23 +79,6 @@ async def not_implemented_handler( return JSONResponse({}, status_code=status.HTTP_501_NOT_IMPLEMENTED) -@app.get("/openapi.json", include_in_schema=False) -async def get_openapi() -> JSONResponse: - return JSONResponse(ert_storage_app.openapi()) - - -@app.get("/docs", include_in_schema=False) -async def get_swagger(req: Request) -> HTMLResponse: - return get_swagger_ui_html( - openapi_url="/openapi.json", title=f"{app.title} - Swagger UI" - ) - - -@app.get("/redoc", include_in_schema=False) -async def get_redoc(req: Request) -> HTMLResponse: - return get_redoc_html(openapi_url="/openapi.json", title=f"{app.title} - Redoc") - - @app.get("/") async def root() -> RedirectResponse: return RedirectResponse("/docs") diff --git a/src/ert/dark_storage/compute/__init__.py b/src/ert/dark_storage/compute/__init__.py new file mode 100644 index 00000000000..80c0c5f8cf7 --- /dev/null +++ b/src/ert/dark_storage/compute/__init__.py @@ -0,0 +1 @@ +from .misfits import calculate_misfits_from_pandas diff --git a/src/ert/dark_storage/compute/misfits.py b/src/ert/dark_storage/compute/misfits.py new file mode 100644 index 00000000000..17615eca450 --- /dev/null +++ b/src/ert/dark_storage/compute/misfits.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Mapping, Sequence + +import numpy as np +import pandas as pd + +if TYPE_CHECKING: + import numpy.typing as npt + + +def _calculate_misfit( + obs_value: npt.NDArray[Any], + response_value: npt.NDArray[Any], + obs_std: npt.NDArray[Any], +) -> Sequence[float]: + difference = response_value - obs_value + misfit = (difference / obs_std) ** 2 + return (misfit * np.sign(difference)).tolist() + + +def calculate_misfits_from_pandas( + reponses_dict: Mapping[int, pd.DataFrame], + observation: pd.DataFrame, + summary_misfits: bool = False, +) -> pd.DataFrame: + """ + Compute misfits from reponses_dict (real_id, values in dataframe) + and observation + """ + misfits_dict = {} + for realization_index in reponses_dict: + misfits_dict[realization_index] = _calculate_misfit( + observation["values"], + reponses_dict[realization_index].loc[:, observation.index].values.flatten(), + observation["errors"], + ) + + df = pd.DataFrame(data=misfits_dict, index=observation.index) + if summary_misfits: + df = pd.DataFrame([df.abs().sum(axis=0)], columns=df.columns, index=[0]) + return df.T diff --git a/src/ert/dark_storage/endpoints/compute/misfits.py b/src/ert/dark_storage/endpoints/compute/misfits.py index a9bae8386af..44df7d10e7b 100644 --- a/src/ert/dark_storage/endpoints/compute/misfits.py +++ b/src/ert/dark_storage/endpoints/compute/misfits.py @@ -4,12 +4,12 @@ import pandas as pd from dateutil.parser import parse -from ert_storage import exceptions as exc -from ert_storage.compute import calculate_misfits_from_pandas from fastapi import APIRouter, Depends, status from fastapi.responses import Response +from ert.dark_storage import exceptions as exc from ert.dark_storage.common import data_for_key, observations_for_obs_keys +from ert.dark_storage.compute import calculate_misfits_from_pandas from ert.dark_storage.enkf import LibresFacade, get_res, get_storage from ert.storage import StorageReader diff --git a/src/ert/dark_storage/endpoints/ensembles.py b/src/ert/dark_storage/endpoints/ensembles.py index 857ebb37442..8a05f0bdbce 100644 --- a/src/ert/dark_storage/endpoints/ensembles.py +++ b/src/ert/dark_storage/endpoints/ensembles.py @@ -1,9 +1,9 @@ from typing import Any, Mapping from uuid import UUID -from ert_storage import json_schema as js from fastapi import APIRouter, Body, Depends +from ert.dark_storage import json_schema as js from ert.dark_storage.common import ensemble_parameter_names, get_response_names from ert.dark_storage.enkf import LibresFacade, get_res, get_storage from ert.storage import StorageAccessor diff --git a/src/ert/dark_storage/endpoints/experiments.py b/src/ert/dark_storage/endpoints/experiments.py index a23e72ba705..96f68711b7f 100644 --- a/src/ert/dark_storage/endpoints/experiments.py +++ b/src/ert/dark_storage/endpoints/experiments.py @@ -1,9 +1,9 @@ from typing import Any, List, Mapping from uuid import UUID -from ert_storage import json_schema as js from fastapi import APIRouter, Body, Depends +from ert.dark_storage import json_schema as js from ert.dark_storage.enkf import LibresFacade, get_res, get_storage from ert.shared.storage.extraction import create_priors from ert.storage import StorageReader diff --git a/src/ert/dark_storage/endpoints/observations.py b/src/ert/dark_storage/endpoints/observations.py index 9aaf0452b2b..830049fe5f0 100644 --- a/src/ert/dark_storage/endpoints/observations.py +++ b/src/ert/dark_storage/endpoints/observations.py @@ -1,9 +1,9 @@ from typing import Any, List, Mapping from uuid import UUID -from ert_storage import json_schema as js from fastapi import APIRouter, Body, Depends +from ert.dark_storage import json_schema as js from ert.dark_storage.enkf import LibresFacade, get_res from ert.shared.storage.extraction import create_observations diff --git a/src/ert/dark_storage/endpoints/records.py b/src/ert/dark_storage/endpoints/records.py index bff872a3d18..29827f51887 100644 --- a/src/ert/dark_storage/endpoints/records.py +++ b/src/ert/dark_storage/endpoints/records.py @@ -4,10 +4,10 @@ from uuid import UUID, uuid4 import pandas as pd -from ert_storage import json_schema as js from fastapi import APIRouter, Body, Depends, File, Header, Request, UploadFile, status from fastapi.responses import Response +from ert.dark_storage import json_schema as js from ert.dark_storage.common import ( data_for_key, ensemble_parameters, diff --git a/src/ert/dark_storage/endpoints/updates.py b/src/ert/dark_storage/endpoints/updates.py index 174e410f545..c212b600b86 100644 --- a/src/ert/dark_storage/endpoints/updates.py +++ b/src/ert/dark_storage/endpoints/updates.py @@ -1,8 +1,8 @@ from uuid import UUID -from ert_storage import json_schema as js from fastapi import APIRouter, Depends +from ert.dark_storage import json_schema as js from ert.dark_storage.enkf import LibresFacade, get_res, reset_res router = APIRouter(tags=["ensemble"]) diff --git a/src/ert/dark_storage/enkf.py b/src/ert/dark_storage/enkf.py index a73304a0220..4e74b345e32 100644 --- a/src/ert/dark_storage/enkf.py +++ b/src/ert/dark_storage/enkf.py @@ -3,10 +3,10 @@ import os from typing import Optional -from ert_storage.security import security from fastapi import Depends from ert.config import ErtConfig +from ert.dark_storage.security import security from ert.enkf_main import EnKFMain from ert.libres_facade import LibresFacade from ert.storage import StorageReader, open_storage diff --git a/src/ert/dark_storage/exceptions.py b/src/ert/dark_storage/exceptions.py new file mode 100644 index 00000000000..dff48ae43e9 --- /dev/null +++ b/src/ert/dark_storage/exceptions.py @@ -0,0 +1,30 @@ +from typing import Any + +from fastapi import status + + +class ErtStorageError(RuntimeError): + """ + Base error class for all the rest of errors + """ + + __status_code__ = status.HTTP_200_OK + + def __init__(self, message: str, **kwargs: Any): + super().__init__(message, kwargs) + + +class NotFoundError(ErtStorageError): + __status_code__ = status.HTTP_404_NOT_FOUND + + +class ConflictError(ErtStorageError): + __status_code__ = status.HTTP_409_CONFLICT + + +class ExpectationError(ErtStorageError): + __status_code__ = status.HTTP_417_EXPECTATION_FAILED + + +class UnprocessableError(ErtStorageError): + __status_code__ = status.HTTP_422_UNPROCESSABLE_ENTITY diff --git a/src/ert/dark_storage/json_schema/__init__.py b/src/ert/dark_storage/json_schema/__init__.py new file mode 100644 index 00000000000..8692ecf41f2 --- /dev/null +++ b/src/ert/dark_storage/json_schema/__init__.py @@ -0,0 +1,11 @@ +from .ensemble import EnsembleIn, EnsembleOut +from .experiment import ExperimentIn, ExperimentOut +from .observation import ( + ObservationIn, + ObservationOut, + ObservationTransformationIn, + ObservationTransformationOut, +) +from .prior import Prior +from .record import RecordOut +from .update import UpdateIn, UpdateOut diff --git a/src/ert/dark_storage/json_schema/ensemble.py b/src/ert/dark_storage/json_schema/ensemble.py new file mode 100644 index 00000000000..347b34c28db --- /dev/null +++ b/src/ert/dark_storage/json_schema/ensemble.py @@ -0,0 +1,37 @@ +from typing import Any, List, Mapping, Optional +from uuid import UUID + +from pydantic import BaseModel, Field, root_validator + + +class _Ensemble(BaseModel): + size: int + parameter_names: List[str] + response_names: List[str] + active_realizations: List[int] = [] + + +class EnsembleIn(_Ensemble): + update_id: Optional[UUID] = None + userdata: Mapping[str, Any] = {} + + @root_validator + def _check_names_no_overlap(cls, values: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Verify that `parameter_names` and `response_names` don't overlap. Ie, no + record can be both a parameter and a response. + """ + if not set(values["parameter_names"]).isdisjoint(set(values["response_names"])): + raise ValueError("parameters and responses cannot have a name in common") + return values + + +class EnsembleOut(_Ensemble): + id: UUID + children: List[UUID] = Field(alias="child_ensemble_ids") + parent: Optional[UUID] = Field(alias="parent_ensemble_id") + experiment_id: Optional[UUID] = None + userdata: Mapping[str, Any] + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/json_schema/experiment.py b/src/ert/dark_storage/json_schema/experiment.py new file mode 100644 index 00000000000..bca4983486f --- /dev/null +++ b/src/ert/dark_storage/json_schema/experiment.py @@ -0,0 +1,24 @@ +from typing import Any, Mapping, Sequence +from uuid import UUID + +from pydantic import BaseModel + +from .prior import Prior + + +class _Experiment(BaseModel): + name: str + + +class ExperimentIn(_Experiment): + priors: Mapping[str, Prior] = {} + + +class ExperimentOut(_Experiment): + id: UUID + ensemble_ids: Sequence[UUID] + priors: Mapping[str, Mapping[str, Any]] + userdata: Mapping[str, Any] + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/json_schema/observation.py b/src/ert/dark_storage/json_schema/observation.py new file mode 100644 index 00000000000..699a56d2d33 --- /dev/null +++ b/src/ert/dark_storage/json_schema/observation.py @@ -0,0 +1,43 @@ +from typing import Any, List, Mapping, Optional +from uuid import UUID + +from pydantic import BaseModel + + +class _ObservationTransformation(BaseModel): + name: str + active: List[bool] + scale: List[float] + observation_id: UUID + + +class ObservationTransformationIn(_ObservationTransformation): + pass + + +class ObservationTransformationOut(_ObservationTransformation): + id: UUID + + class Config: + orm_mode = True + + +class _Observation(BaseModel): + name: str + errors: List[float] + values: List[float] + x_axis: List[Any] + records: Optional[List[UUID]] = None + + +class ObservationIn(_Observation): + pass + + +class ObservationOut(_Observation): + id: UUID + transformation: Optional[ObservationTransformationOut] = None + userdata: Mapping[str, Any] = {} + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/json_schema/prior.py b/src/ert/dark_storage/json_schema/prior.py new file mode 100644 index 00000000000..b22bfc5b735 --- /dev/null +++ b/src/ert/dark_storage/json_schema/prior.py @@ -0,0 +1,152 @@ +import sys +from typing import Union + +from pydantic import BaseModel + +if sys.version_info < (3, 8): + from typing_extensions import Literal +else: + from typing import Literal + + +class PriorConst(BaseModel): + """ + Constant parameter prior + """ + + function: Literal["const"] = "const" + value: float + + +class PriorTrig(BaseModel): + """ + Triangular distribution parameter prior + """ + + function: Literal["trig"] = "trig" + min: float + max: float + mode: float + + +class PriorNormal(BaseModel): + """ + Normal distribution parameter prior + """ + + function: Literal["normal"] = "normal" + mean: float + std: float + + +class PriorLogNormal(BaseModel): + """ + Log-normal distribution parameter prior + """ + + function: Literal["lognormal"] = "lognormal" + mean: float + std: float + + +class PriorErtTruncNormal(BaseModel): + """ + ERT Truncated normal distribution parameter prior + + ERT differs from the usual distribution by that it simply clamps on `min` + and `max`, which gives a bias towards the extremes. + + """ + + function: Literal["ert_truncnormal"] = "ert_truncnormal" + mean: float + std: float + min: float + max: float + + +class PriorStdNormal(BaseModel): + """ + Standard normal distribution parameter prior + + Normal distribution with mean of 0 and standard deviation of 1 + """ + + function: Literal["stdnormal"] = "stdnormal" + + +class PriorUniform(BaseModel): + """ + Uniform distribution parameter prior + """ + + function: Literal["uniform"] = "uniform" + min: float + max: float + + +class PriorErtDUniform(BaseModel): + """ + ERT Discrete uniform distribution parameter prior + + This discrete uniform distribution differs from the standard by using the + `bins` parameter. Normally, `a`, and `b` are integers, and the sample space + are the integers between. ERT allows `a` and `b` to be arbitrary floats, + where the sample space is binned. + + """ + + function: Literal["ert_duniform"] = "ert_duniform" + bins: int + min: float + max: float + + +class PriorLogUniform(BaseModel): + """ + Logarithmic uniform distribution parameter prior + """ + + function: Literal["loguniform"] = "loguniform" + min: float + max: float + + +class PriorErtErf(BaseModel): + """ + ERT Error function distribution parameter prior + """ + + function: Literal["ert_erf"] = "ert_erf" + min: float + max: float + skewness: float + width: float + + +class PriorErtDErf(BaseModel): + """ + ERT Discrete error function distribution parameter prior + """ + + function: Literal["ert_derf"] = "ert_derf" + bins: int + min: float + max: float + skewness: float + width: float + + +Prior = Union[ + PriorConst, + PriorTrig, + PriorNormal, + PriorLogNormal, + PriorErtTruncNormal, + PriorStdNormal, + PriorUniform, + PriorErtDUniform, + PriorLogUniform, + PriorErtErf, + PriorErtDErf, +] diff --git a/src/ert/dark_storage/json_schema/record.py b/src/ert/dark_storage/json_schema/record.py new file mode 100644 index 00000000000..2df045abaa5 --- /dev/null +++ b/src/ert/dark_storage/json_schema/record.py @@ -0,0 +1,18 @@ +from typing import Any, Mapping, Optional +from uuid import UUID + +from pydantic import BaseModel, Field + + +class _Record(BaseModel): + pass + + +class RecordOut(_Record): + id: UUID + name: str + userdata: Mapping[str, Any] + has_observations: Optional[bool] + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/json_schema/update.py b/src/ert/dark_storage/json_schema/update.py new file mode 100644 index 00000000000..e3230312fcb --- /dev/null +++ b/src/ert/dark_storage/json_schema/update.py @@ -0,0 +1,24 @@ +from typing import List, Optional, Union +from uuid import UUID + +from pydantic import BaseModel + +from .observation import ObservationTransformationIn + + +class _Update(BaseModel): + algorithm: str + ensemble_result_id: Union[UUID, None] + ensemble_reference_id: Union[UUID, None] + + +class UpdateIn(_Update): + observation_transformations: Optional[List[ObservationTransformationIn]] = None + + +class UpdateOut(_Update): + id: UUID + experiment_id: UUID + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/security.py b/src/ert/dark_storage/security.py new file mode 100644 index 00000000000..f410b8047d9 --- /dev/null +++ b/src/ert/dark_storage/security.py @@ -0,0 +1,25 @@ +import os +from typing import Optional + +from fastapi import HTTPException, Security, status +from fastapi.security import APIKeyHeader + +DEFAULT_TOKEN = "hunter2" +_security_header = APIKeyHeader(name="Token", auto_error=False) + + +async def security(*, token: Optional[str] = Security(_security_header)) -> None: + if os.getenv("ERT_STORAGE_NO_TOKEN"): + return + if not token: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail="Not authenticated" + ) + real_token = os.getenv("ERT_STORAGE_TOKEN", DEFAULT_TOKEN) + if token == real_token: + # Success + return + + # HTTP 403 is when the user has authorized themselves, but aren't allowed to + # access this resource + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid token") diff --git a/src/ert/services/storage_service.py b/src/ert/services/storage_service.py index 0b4d5402096..452b3d11e43 100644 --- a/src/ert/services/storage_service.py +++ b/src/ert/services/storage_service.py @@ -6,7 +6,6 @@ import httpx import requests -from ert_storage.client import Client, ConnInfo from ert.services._base_service import BaseService, _Context, local_exec_args @@ -79,15 +78,14 @@ def fetch_url(self) -> str: ) @classmethod - def session(cls, timeout: Optional[int] = None) -> Client: + def session(cls, timeout: Optional[int] = None) -> httpx.Client: """ Start a HTTP transaction with the server """ inst = cls.connect(timeout=timeout) - return Client( - conn_info=ConnInfo( - base_url=inst.fetch_url(), auth_token=inst.fetch_auth()[1] - ) + return httpx.Client( + base_url=inst.fetch_url(), + headers={"X-Token": inst.fetch_auth()[1]}, ) @classmethod diff --git a/tests/unit_tests/dark_storage/conftest.py b/tests/unit_tests/dark_storage/conftest.py index de576a15094..f520039d997 100644 --- a/tests/unit_tests/dark_storage/conftest.py +++ b/tests/unit_tests/dark_storage/conftest.py @@ -60,19 +60,6 @@ def dark_storage_client(monkeypatch): yield client -@pytest.fixture -def env(monkeypatch): - monkeypatch.setenv("ERT_STORAGE_DATABASE_URL", "sqlite:///:memory:") - monkeypatch.setenv("ERT_STORAGE_NO_TOKEN", "yup") - - -@pytest.fixture -def ert_storage_app(env): - from ert_storage.app import app - - return app - - def reset_enkf(): enkf._config = None enkf._ert = None diff --git a/tests/unit_tests/dark_storage/test_api_compatibility.py b/tests/unit_tests/dark_storage/test_api_compatibility.py deleted file mode 100644 index f3dc55c38e9..00000000000 --- a/tests/unit_tests/dark_storage/test_api_compatibility.py +++ /dev/null @@ -1,18 +0,0 @@ -def test_openapi(ert_storage_app, dark_storage_app): - """ - Test that the openapi.json of Dark Storage is identical to ERT Storage - """ - expect = ert_storage_app.openapi() - actual = dark_storage_app.openapi() - - # Remove textual data (descriptions and such) from ERT Storage's API. - def _remove_text(data): - if isinstance(data, dict): - return { - key: _remove_text(val) - for key, val in data.items() - if key not in ("description", "examples") - } - return data - - assert _remove_text(expect) == _remove_text(actual)