From 7a9049be388d3b221696f0809e436a8a7386e142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Berland?= Date: Wed, 3 Jan 2024 10:17:41 +0100 Subject: [PATCH] Port sunsch to use Pydantic instead of configsuite --- pyproject.toml | 1 - src/subscript/sunsch/__init__.py | 3 + src/subscript/sunsch/sunsch.py | 270 ++++++---------------------- src/subscript/sunsch/time_vector.py | 3 +- tests/test_sunsch.py | 38 +--- 5 files changed, 62 insertions(+), 253 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9b71d8da3..b6ec1f865 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,6 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "configsuite", "resdata", "res2df", "ert>=2.38.0b7", diff --git a/src/subscript/sunsch/__init__.py b/src/subscript/sunsch/__init__.py index e69de29bb..11ce31ad5 100644 --- a/src/subscript/sunsch/__init__.py +++ b/src/subscript/sunsch/__init__.py @@ -0,0 +1,3 @@ +from .sunsch import SunschConfig, ensure_start_refdate + +__all__ = ["SunschConfig", "ensure_start_refdate"] diff --git a/src/subscript/sunsch/sunsch.py b/src/subscript/sunsch/sunsch.py index cddf6d9a1..6beac1e50 100755 --- a/src/subscript/sunsch/sunsch.py +++ b/src/subscript/sunsch/sunsch.py @@ -9,31 +9,23 @@ import argparse import datetime import logging -import sys import tempfile import textwrap import warnings from pathlib import Path -from typing import List, Union -import configsuite # lgtm [py/import-and-import-from] import dateutil.parser import yaml -from configsuite import MetaKeys as MK # lgtm [py/import-and-import-from] -from configsuite import types # lgtm [py/import-and-import-from] +from pydantic import BaseModel, FilePath from subscript import __version__, getLogger from subscript.sunsch.time_vector import TimeVector # type: ignore - -# from opm.tools import TimeVector # type: ignore - +from typing import Dict, List, Literal, Optional, Union logger = getLogger(__name__) __MAGIC_STDOUT__ = "-" # When used as a filename on the command line -SUPPORTED_DATEGRIDS = ["daily", "monthly", "yearly", "weekly", "biweekly", "bimonthly"] - DESCRIPTION = """Generate Eclipse Schedule file from merges and insertions. Reads a YAML-file specifying how a Eclipse Schedule section is to be @@ -53,30 +45,7 @@ """ -@configsuite.validator_msg("Is dategrid a supported frequency") -def _is_valid_dategrid(dategrid_str: str): - return dategrid_str in SUPPORTED_DATEGRIDS - - -@configsuite.validator_msg("Is filename an existing file") -def _is_existing_file(filename: str): - return Path(filename).exists() - - -@configsuite.transformation_msg("Defaults handling") -def _defaults_handling(config: dict): - """Handle defaults with dates.""" - return _shuffle_start_refdate(config) - - -@configsuite.transformation_msg("Convert to string") -def _to_string(element): - """Convert anything to a string""" - return str(element) - - -@configsuite.transformation_msg("Shuffle startdate vs refdate") -def _shuffle_start_refdate(config: dict) -> dict: +def ensure_start_refdate(config: dict) -> dict: """ Ensure that: * startdate is always defined, if not given, it is picked @@ -103,139 +72,26 @@ def _shuffle_start_refdate(config: dict) -> dict: return config -CONFIG_SCHEMA = { - MK.Type: types.NamedDict, - MK.Transformation: _defaults_handling, - MK.Content: { - "files": { - MK.Type: types.List, - MK.Description: "List of filenames to include in merge operation", - MK.Content: { - MK.Item: { - MK.Type: types.String, - MK.Description: "Filename to merge", - MK.ElementValidators: (_is_existing_file,), - } - }, - }, - "output": { - MK.Description: "Output filename, '-' means stdout", - MK.Type: types.String, - MK.AllowNone: True, - MK.Default: "-", - }, - "startdate": { - MK.Description: "The start date of the Eclipse run (START keyword).", - MK.Type: types.Date, - MK.AllowNone: True, - # (a transformation will provide the default value here) - }, - "starttime": { - MK.Description: ( - "The start time, used for relative " - "inserts if clock accuracy is needed" - ), - MK.Type: types.DateTime, - MK.AllowNone: True, - # (a transformation will provide/calculate a default value here) - }, - "refdate": { - MK.Description: ( - "Reference date for relative inserts. " - "Only set if it should be different than startdate." - ), - MK.Type: types.Date, - MK.AllowNone: True, - }, - "enddate": { - MK.Description: "An end date, events pass this date will be clipped", - MK.Type: types.Date, - MK.AllowNone: True, - }, - "dategrid": { - MK.Description: ( - "Set to yearly, monthly, etc to get a grid of dates included" - ), - MK.Type: types.String, - MK.AllowNone: True, - MK.ElementValidators: (_is_valid_dategrid,), - }, - "insert": { - MK.Description: ( - "List of insert statements to process into the Schedule file" - ), - MK.Type: types.List, - MK.Content: { - MK.Item: { - MK.Description: "Insert statement", - MK.Type: types.NamedDict, - MK.Content: { - "date": { - MK.Description: "Date at which to insert something", - MK.Type: types.Date, - MK.AllowNone: True, - }, - "filename": { - MK.Description: "Filename with contents to insert", - MK.Type: types.String, - MK.AllowNone: True, - MK.ElementValidators: (_is_existing_file,), - }, - "template": { - MK.Description: ( - "Template file in which substitution will " - "take place before it is inserted" - ), - MK.Type: types.String, - MK.AllowNone: True, - MK.ElementValidators: (_is_existing_file,), - }, - "days": { - MK.Description: ( - "Days after refdate/startdate at which " - "insertion should take place" - ), - MK.Type: types.Number, - MK.AllowNone: True, - }, - "string": { - MK.Description: ("A string to insert, instead of filename"), - MK.Type: types.String, - MK.AllowNone: True, - }, - "substitute": { - MK.Description: ( - "Key-value pairs for substitution in a template" - ), - MK.Type: types.Dict, - MK.Content: { - MK.Key: { - MK.Description: "Template key name", - MK.AllowNone: False, - MK.Type: types.String, - }, - MK.Value: { - MK.AllowNone: "Value to insert in template", - MK.AllowNone: False, - # Since we allow both numbers and strings here, - # it is converted to a string as configsuite - # only allows one type. - MK.Transformation: _to_string, - MK.Type: types.String, - }, - }, - }, - }, - } - }, - }, - }, -} - - -def get_schema() -> dict: - """Return the ConfigSuite schema""" - return CONFIG_SCHEMA +class InsertStatement(BaseModel): + date: Optional[datetime.date] = None + filename: Optional[FilePath] = None + template: Optional[FilePath] = None + days: Optional[float] = None + string: Optional[str] = None + substitute: Optional[Dict[str, Union[str, float, str]]] = None + + +class SunschConfig(BaseModel): + files: Optional[List[FilePath]] = None + output: Optional[str] = "-" + startdate: Optional[Union[datetime.date, datetime.datetime]] = None + starttime: Optional[datetime.datetime] = None + refdate: Optional[Union[datetime.date, datetime.datetime]] = None + enddate: Optional[datetime.date] = None + dategrid: Optional[ + Literal["daily", "monthly", "yearly", "weekly", "biweekly", "bimonthly"] + ] = None + insert: Optional[List[InsertStatement]] = None def datetime_from_date( @@ -249,7 +105,7 @@ def datetime_from_date( return datetime.datetime.combine(date, datetime.datetime.min.time()) -def process_sch_config(conf) -> TimeVector: +def process_sch_config(conf: Union[dict, SunschConfig]) -> TimeVector: """Process a Schedule configuration into a opm.tools TimeVector Recognized keys in the configuration dict: files, startdate, startime, @@ -262,14 +118,13 @@ def process_sch_config(conf) -> TimeVector: Returns: opm.io.TimeVector """ - # At least test code is calling this function with a dict as - # config - convert it to a configsuite snapshot: if isinstance(conf, dict): - conf = configsuite.ConfigSuite( - conf, CONFIG_SCHEMA, deduce_required=True - ).snapshot + conf = ensure_start_refdate(conf) + conf = SunschConfig(**conf) - # Rerun this to ensure error is caught (already done in transformation) + assert isinstance(conf, SunschConfig) + + # Rerun this to ensure error is caught datetime_from_date(conf.startdate) # Initialize the opm.tools.TimeVector class, which needs @@ -278,15 +133,15 @@ def process_sch_config(conf) -> TimeVector: if conf.files is not None: for filename in conf.files: - if sch_file_nonempty(filename): + if sch_file_nonempty(str(filename)): logger.info("Loading %s", filename) else: logger.warning("No Eclipse statements in %s, skipping", filename) continue - file_starts_with_dates = sch_file_starts_with_dates_keyword(filename) + file_starts_with_dates = sch_file_starts_with_dates_keyword(str(filename)) timevector = load_timevector_from_file( - filename, conf.startdate, file_starts_with_dates + str(filename), conf.startdate, file_starts_with_dates ) if file_starts_with_dates: schedule.load_string(str(timevector)) @@ -326,8 +181,8 @@ def process_sch_config(conf) -> TimeVector: # Do the insertion: if date >= conf.starttime: if insert_statement.string is None: - if sch_file_nonempty(filename): - schedule.load(filename, date=date) + if sch_file_nonempty(str(filename)): + schedule.load(str(filename), date=date) else: logger.warning( "No Eclipse statements in %s, skipping", filename @@ -473,7 +328,7 @@ def sch_file_starts_with_dates_keyword(filename: str) -> bool: return True -def substitute(insert_statement) -> str: +def substitute(insert_statement: InsertStatement) -> str: """ Perform key-value substitutions and generate the result as a file on disk. @@ -485,7 +340,7 @@ def substitute(insert_statement) -> str: be left untouched. Args: - insert_statement (named_dict): Required keys are "template", which is + insert_statement: Required keys are "template", which is a filename with parameters to be replaced, and "substitute" which is a named_dict with values parameter-value mappings to be used. @@ -494,7 +349,7 @@ def substitute(insert_statement) -> str: str: Filename on temporary location for immediate use """ - if len([key for key in list(insert_statement) if key is not None]) > 3: + if len([value for _, value in list(insert_statement) if value is not None]) > 3: # (there should be also 'days' or 'date' in the dict) logger.warning( "Too many (?) configuration elements in %s", str(insert_statement) @@ -506,11 +361,9 @@ def substitute(insert_statement) -> str: Path(insert_statement.template).read_text(encoding="utf8").splitlines() ) - # Parse substitution list: - substdict = insert_statement.substitute # Perform substitution and put into a tmp file for line in templatelines: - for key, value in substdict: + for key, value in insert_statement.substitute.items(): if "<" + key + ">" in line: line = line.replace("<" + key + ">", str(value)) resultfile.write(line + "\n") @@ -571,14 +424,6 @@ def dategrid( Return: list of datetime.date. Always includes start-date, might not include end-date """ - - if interval not in SUPPORTED_DATEGRIDS: - raise ValueError( - 'Unsupported dategrid interval "' - + interval - + '". Pick among ' - + ", ".join(SUPPORTED_DATEGRIDS) - ) dates = [startdate] date = startdate + datetime.timedelta(days=1) startdateweekday = startdate.weekday() @@ -707,13 +552,13 @@ def main(): args = parser.parse_args() # Application defaults configuration: - defaults_config = {"output": "-", "startdate": datetime.date(1900, 1, 1)} + defaults_config: dict = {"output": "-", "startdate": datetime.date(1900, 1, 1)} # Users YAML configuration: - yaml_config = yaml.safe_load(Path(args.config).read_text(encoding="utf8")) + yaml_config: dict = yaml.safe_load(Path(args.config).read_text(encoding="utf8")) # Command line configuration: - cli_config = {} + cli_config: dict = {} if args.output: cli_config["output"] = args.output if args.startdate: @@ -725,33 +570,26 @@ def main(): if args.dategrid: cli_config["dategrid"] = args.dategrid - # Merge defaults-, yaml- and command line options, and then validate: - config = configsuite.ConfigSuite( - {}, - get_schema(), - layers=(defaults_config, yaml_config, cli_config), - deduce_required=True, - ) - if not config.valid: - logger.error(config.errors) - logger.error("Your configuration is invalid. Exiting.") - sys.exit(1) + merged_config = defaults_config.copy() + merged_config.update(yaml_config) + merged_config.update(cli_config) + merged_config = ensure_start_refdate(merged_config) - if args.verbose and config.snapshot.output != __MAGIC_STDOUT__: + config = SunschConfig(**merged_config) + + if args.verbose and config.output != __MAGIC_STDOUT__: logger.setLevel(logging.INFO) - if args.debug and config.snapshot.output != __MAGIC_STDOUT__: + if args.debug and config.output != __MAGIC_STDOUT__: logger.setLevel(logging.DEBUG) # Generate the schedule section, as a string: - schedule = wrap_long_lines( - str(process_sch_config(config.snapshot)), maxchars=128, warn=True - ) + schedule = wrap_long_lines(str(process_sch_config(config)), maxchars=128, warn=True) - if config.snapshot.output == __MAGIC_STDOUT__: + if config.output == __MAGIC_STDOUT__: print(schedule) else: - logger.info("Writing Eclipse deck to %s", str(config.snapshot.output)) - dirname = Path(config.snapshot.output).parent + logger.info("Writing Eclipse deck to %s", str(config.output)) + dirname = Path(config.output).parent if dirname and not dirname.exists(): warnings.warn( f"Implicit mkdir of directory {str(dirname)} is deprecated and " @@ -760,7 +598,7 @@ def main(): FutureWarning, ) dirname.mkdir() - Path(config.snapshot.output).write_text(schedule, encoding="utf8") + Path(config.output).write_text(schedule, encoding="utf8") if __name__ == "__main__": diff --git a/src/subscript/sunsch/time_vector.py b/src/subscript/sunsch/time_vector.py index f63f62ed8..105a55c37 100644 --- a/src/subscript/sunsch/time_vector.py +++ b/src/subscript/sunsch/time_vector.py @@ -3,12 +3,13 @@ import datetime from operator import attrgetter +from opm.io.parser import Parser + try: from StringIO import StringIO except ImportError: from io import StringIO -from opm.io.parser import Parser # This is from the TimeMap.cpp implementation in opm ecl_month = { diff --git a/tests/test_sunsch.py b/tests/test_sunsch.py index 5f3b9e090..15edb6ef0 100644 --- a/tests/test_sunsch.py +++ b/tests/test_sunsch.py @@ -4,9 +4,9 @@ import subprocess from pathlib import Path -import configsuite import pytest # noqa: F401 import yaml +from pydantic import ValidationError from subscript.sunsch import sunsch @@ -159,29 +159,6 @@ def test_dump_stdout(testdata, mocker): assert "DEBUG:subscript" not in result.stdout.decode() -def test_config_schema(tmp_path): - """Test the implementation of configsuite""" - os.chdir(tmp_path) - cfg = {"init": "existingfile.sch", "output": "newfile.sch"} - cfg_suite = configsuite.ConfigSuite(cfg, sunsch.CONFIG_SCHEMA, deduce_required=True) - assert not cfg_suite.valid # file missing - - Path("existingfile.sch").write_text("foo", encoding="utf8") - cfg_suite = configsuite.ConfigSuite(cfg, sunsch.CONFIG_SCHEMA, deduce_required=True) - print(cfg_suite.errors) - assert not cfg_suite.valid # "foo" is not valid configuration. - - cfg = { - "files": ["existingfile.sch"], - "output": "newfile.sch", - "startdate": datetime.date(2018, 2, 2), - "insert": [], - } - cfg_suite = configsuite.ConfigSuite(cfg, sunsch.CONFIG_SCHEMA, deduce_required=True) - print(cfg_suite.errors) - assert cfg_suite.valid - - def test_templating(tmp_path): """Test templating""" os.chdir(tmp_path) @@ -204,10 +181,6 @@ def test_templating(tmp_path): assert "A-007" in str(sch) assert "200.3" in str(sch) assert "1400000" in str(sch) - cfg_suite = configsuite.ConfigSuite( - sunschconf, sunsch.CONFIG_SCHEMA, deduce_required=True - ) - assert cfg_suite.valid # Let some of the valued be undefined: sunschconf = { @@ -427,16 +400,11 @@ def test_nonisodate(readonly_datadir): sunsch.process_sch_config(sunschconf) sunschconf = { - # Beware that using a ISO-string for a date in this config - # will give a wrong error message, since the code assumes - # all string dates are already parsed into datimes by the - # yaml loader. - # "startdate": "2020-01-01", "startdate": datetime.date(2020, 1, 1), "enddate": "01-01-2020", "insert": [{"filename": "foo1.sch", "date": datetime.date(2030, 1, 1)}], } - with pytest.raises(TypeError): + with pytest.raises(ValidationError): sunsch.process_sch_config(sunschconf) @@ -691,7 +659,7 @@ def test_dategrid(): assert min(sch.dates) == datetime.datetime(2020, 1, 1, 0, 0) # Unknown dategrid - with pytest.raises(ValueError, match="Unsupported dategrid interval"): + with pytest.raises(ValidationError): sch = sunsch.process_sch_config( { "startdate": datetime.date(2020, 1, 1),