From a98f4ba0cddf04165743a21c1cbf205ca70b6b6e Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Fri, 25 Jun 2021 16:59:04 -0400 Subject: [PATCH 01/33] Add epic --- {sql => asset}/mssql/extant.sql | 0 asset/postgres/epic/errors/insert.sql | 10 ++ asset/postgres/epic/notifications/channel.sql | 1 + asset/postgres/epic/notifications/insert.sql | 6 + {sql => asset}/postgres/extant.sql | 0 {sql => asset}/postgres/predictions/gold.sql | 0 .../postgres/predictions/insert.sql | 0 asset/postgres/predictions/listen.sql | 1 + asset/postgres/predictions/recover.sql | 22 +++ {sql => asset}/postgres/runs/close.sql | 0 {sql => asset}/postgres/runs/open.sql | 0 asset/postgres/schema.sql | 1 + postgres/sql/patchdb.d/004.patch.sql | 18 ++- postgres/sql/patchdb.d/005.public.sql | 19 ++- postgres/sql/patchdb.d/006.private.sql | 10 +- postgres/sql/patchdb.d/007.epic.sql | 20 ++- setup.py | 14 +- sql/postgres/schema.sql | 1 - src/dsdk/epic.py | 141 ++++++++++++++++++ src/dsdk/postgres.py | 22 +++ test/test_dsdk.py | 8 +- test/test_postgres.py | 2 +- 22 files changed, 264 insertions(+), 32 deletions(-) rename {sql => asset}/mssql/extant.sql (100%) create mode 100644 asset/postgres/epic/errors/insert.sql create mode 100644 asset/postgres/epic/notifications/channel.sql create mode 100644 asset/postgres/epic/notifications/insert.sql rename {sql => asset}/postgres/extant.sql (100%) rename {sql => asset}/postgres/predictions/gold.sql (100%) rename {sql => asset}/postgres/predictions/insert.sql (100%) create mode 100644 asset/postgres/predictions/listen.sql create mode 100644 asset/postgres/predictions/recover.sql rename {sql => asset}/postgres/runs/close.sql (100%) rename {sql => asset}/postgres/runs/open.sql (100%) create mode 100644 asset/postgres/schema.sql delete mode 100644 sql/postgres/schema.sql create mode 100644 src/dsdk/epic.py diff --git a/sql/mssql/extant.sql b/asset/mssql/extant.sql similarity index 100% rename from sql/mssql/extant.sql rename to asset/mssql/extant.sql diff --git a/asset/postgres/epic/errors/insert.sql b/asset/postgres/epic/errors/insert.sql new file mode 100644 index 0000000..3ab0ec9 --- /dev/null +++ b/asset/postgres/epic/errors/insert.sql @@ -0,0 +1,10 @@ +insert into epic_errors ( + prediction_id, + error_name, + error_description +) +select + %(prediction_id)s, + %(error_name)s, + %(error_description)s +returning * diff --git a/asset/postgres/epic/notifications/channel.sql b/asset/postgres/epic/notifications/channel.sql new file mode 100644 index 0000000..ad482c9 --- /dev/null +++ b/asset/postgres/epic/notifications/channel.sql @@ -0,0 +1 @@ +example.epic_notifications diff --git a/asset/postgres/epic/notifications/insert.sql b/asset/postgres/epic/notifications/insert.sql new file mode 100644 index 0000000..fad6279 --- /dev/null +++ b/asset/postgres/epic/notifications/insert.sql @@ -0,0 +1,6 @@ +insert into epic_notifications ( + prediction_id +) +select + %s +returning * diff --git a/sql/postgres/extant.sql b/asset/postgres/extant.sql similarity index 100% rename from sql/postgres/extant.sql rename to asset/postgres/extant.sql diff --git a/sql/postgres/predictions/gold.sql b/asset/postgres/predictions/gold.sql similarity index 100% rename from sql/postgres/predictions/gold.sql rename to asset/postgres/predictions/gold.sql diff --git a/sql/postgres/predictions/insert.sql b/asset/postgres/predictions/insert.sql similarity index 100% rename from sql/postgres/predictions/insert.sql rename to asset/postgres/predictions/insert.sql diff --git a/asset/postgres/predictions/listen.sql b/asset/postgres/predictions/listen.sql new file mode 100644 index 0000000..790e7bf --- /dev/null +++ b/asset/postgres/predictions/listen.sql @@ -0,0 +1 @@ +listen example.predictions diff --git a/asset/postgres/predictions/recover.sql b/asset/postgres/predictions/recover.sql new file mode 100644 index 0000000..a52e5d3 --- /dev/null +++ b/asset/postgres/predictions/recover.sql @@ -0,0 +1,22 @@ +select + p.id, + p.run_id, + p.csn, + p.empi, + p.score, + r.as_of +from + runs as r on + join predictions as p on + p.run_id = r.id + and upper(r.interval) != 'infinity' + left join epic_notifications as n on + n.prediction_id = p.prediction_id + left join epic_errors as e on + e.prediction_id = e.prediction_id + and e.acknowledged_on is null +where + n.id is null + and e.id is null +order by + p.id diff --git a/sql/postgres/runs/close.sql b/asset/postgres/runs/close.sql similarity index 100% rename from sql/postgres/runs/close.sql rename to asset/postgres/runs/close.sql diff --git a/sql/postgres/runs/open.sql b/asset/postgres/runs/open.sql similarity index 100% rename from sql/postgres/runs/open.sql rename to asset/postgres/runs/open.sql diff --git a/asset/postgres/schema.sql b/asset/postgres/schema.sql new file mode 100644 index 0000000..d2650f7 --- /dev/null +++ b/asset/postgres/schema.sql @@ -0,0 +1 @@ +set search_path = example; diff --git a/postgres/sql/patchdb.d/004.patch.sql b/postgres/sql/patchdb.d/004.patch.sql index 8ad9686..d302761 100644 --- a/postgres/sql/patchdb.d/004.patch.sql +++ b/postgres/sql/patchdb.d/004.patch.sql @@ -1,4 +1,4 @@ -set search_path = dsdk; +set search_path = example; create or replace function up_patch() @@ -47,7 +47,9 @@ begin in_id, unnest(in_requires); return true; end; - $function$ language plpgsql; + $function$ + language plpgsql + search_path example; exception when duplicate_function then end; @@ -67,7 +69,9 @@ begin delete from patches where id = in_id; return true; end; - $function$ language plpgsql; + $function$ + language plpgsql + search_path example; exception when duplicate_function then end; @@ -75,7 +79,9 @@ begin return; end if; end; -$$ language plpgsql; +$$ + language plpgsql + search_path example; create or replace function down_patch() @@ -89,7 +95,9 @@ begin drop table patch_requires; drop table patches; end; -$$ language plpgsql; +$$ + language plpgsql + search_path example; select up_patch(); diff --git a/postgres/sql/patchdb.d/005.public.sql b/postgres/sql/patchdb.d/005.public.sql index eac214a..c07db50 100644 --- a/postgres/sql/patchdb.d/005.public.sql +++ b/postgres/sql/patchdb.d/005.public.sql @@ -1,4 +1,4 @@ -set search_path = dsdk; +set search_path = example; create or replace function up_public() @@ -17,7 +17,10 @@ begin exception when invalid_parameter_value or others then return false; end; - $function$ language plpgsql stable; + $function$ + language plpgsql + search_path example + stable; create domain timezone as varchar check ( is_timezone(value) ); @@ -32,7 +35,9 @@ begin perform pg_notify(TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, last_id); return null; end; - $function$ language plpgsql; + $function$ + language plpgsql + search_path example; create table models ( id int primary key generated always as identity, @@ -103,7 +108,9 @@ begin for each statement execute procedure call_notify(); end; -$$ language plpgsql; +$$ + language plpgsql + search_path example; create or replace function down_public() @@ -122,7 +129,9 @@ begin drop domain timezone; drop function is_timezone; end; -$$ language plpgsql; +$$ + language plpgsql + search_path example; select up_public(); diff --git a/postgres/sql/patchdb.d/006.private.sql b/postgres/sql/patchdb.d/006.private.sql index 0847a61..2dab672 100644 --- a/postgres/sql/patchdb.d/006.private.sql +++ b/postgres/sql/patchdb.d/006.private.sql @@ -1,4 +1,4 @@ -set search_path = dsdk; +set search_path = example; create or replace function up_private() @@ -29,7 +29,9 @@ begin ) ); end; -$$ language plpgsql; +$$ + language plpgsql + search_path example; create or replace function down_private() @@ -41,7 +43,9 @@ begin drop table features; end; -$$ language plpgsql; +$$ + language plpgsql + search_path example; select up_private(); diff --git a/postgres/sql/patchdb.d/007.epic.sql b/postgres/sql/patchdb.d/007.epic.sql index 311f46b..c9c1e4d 100644 --- a/postgres/sql/patchdb.d/007.epic.sql +++ b/postgres/sql/patchdb.d/007.epic.sql @@ -1,4 +1,4 @@ -set search_path = dsdk; +set search_path = example; create or replace function up_epic() @@ -14,26 +14,32 @@ begin notified_on timestamptz default statement_timestamp(), constraint only_one_epic_notification_per_prediction unique (prediction_id), - constraint prediction_epic_notifications_required_a_prediction + constraint epic_notifications_require_a_prediction foreign key (prediction_id) references predictions (id) on delete cascade on update cascade ); + create trigger epic_notifications_inserted after insert on epic_notifications + referencing new table as inserted + for each statement + execute procedure call_notify(); create table epic_errors ( id int primary key generated always as identity, prediction_id int not null, recorded_on timestamptz default statement_timestamp(), - acknowledged_on timestamptz default statement_timestamp(), + acknowledged_on timestamptz default null, error_name varchar, error_description varchar, - constraint prediction_epic_error_required_a_prediction + constraint epic_errors_require_a_prediction foreign key (prediction_id) references predictions (id) on delete cascade on update cascade ); end; -$$ language plpgsql; +$$ + language plpgsql + search_path example; create or replace function down_epic() @@ -47,7 +53,9 @@ begin drop table epic_errors; drop table epic_notifications; end; -$$ language plpgsql; +$$ + language plpgsql + search_path example; select up_epic(); diff --git a/setup.py b/setup.py index 7e04e84..30dc81d 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,6 @@ "wheel>=0.35.1", ) -PYMONGO_REQUIRES = ("pymongo>=3.11.0",) - PYMSSQL_REQUIRES = ("cython>=0.29.21", "pymssql>=2.1.4") PSYCOPG2_REQUIRES = ("psycopg2-binary>=2.8.6",) @@ -49,13 +47,15 @@ ) setup( + entry_points={ + "console_scripts": [ + "epic-notify = dsdk.epic:Notify.main", + "epic-validate = dsdk.epic:Validate.main", + ] + }, extras_require={ - "all": PSYCOPG2_REQUIRES - + PYMONGO_REQUIRES - + PYMSSQL_REQUIRES - + TEST_REQUIRES, + "all": PSYCOPG2_REQUIRES + PYMSSQL_REQUIRES + TEST_REQUIRES, "psycopg2": PSYCOPG2_REQUIRES, - "pymongo": PYMONGO_REQUIRES, "pymssql": PYMSSQL_REQUIRES, "test": TEST_REQUIRES, }, diff --git a/sql/postgres/schema.sql b/sql/postgres/schema.sql deleted file mode 100644 index 4b84623..0000000 --- a/sql/postgres/schema.sql +++ /dev/null @@ -1 +0,0 @@ -set search_path = dsdk,public diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py new file mode 100644 index 0000000..dbbe0fb --- /dev/null +++ b/src/dsdk/epic.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- +"""Epic microservices.""" + +from logging import getLogger +from select import select +from urllib.request import urlopen + +logger = getLogger(__name__) + + +class Abstract: + """Abstract Service.""" + + def __init__(self, postgres, uri, poll_timeout=60, uri_timeout=5) -> None: + """__init__.""" + self.postgres = postgres + self.uri = uri + self.poll_timeout = poll_timeout + self.uri_timeout = uri_timeout + + def listen(self, listen): + """Listen.""" + while True: + readers, _, exceptions = select( + [listen], [], [], self.poll_timeout + ) + if listen.poll(): + while listen.notifies: + self.on_notify(listen.notifies.pop()) + + +class Notification: + """Notification Service.""" + + URI = "?".join( + ( + "api/epic/2011/Clinical/Patient/AddFlowsheetValue/FlowsheetValue", + "&".join( + ( + "PatientID=%(empi)s", + "PatientIDType={patient_id_type}", + "ContactID=%(csn)s", + "ContactIDType={contact_id_type}", + "UserID=PENNSIGNALS", + "UserIDType=EXTERNAL", + "FlowsheetID={flowsheet_id}", + "FlowsheetIDType={flowsheet_id_type}", + "Value=%(score)s", + "Comment={comment}", + "InstantValueTaken={instant_value_taken}", + "FlowsheetTemplateID={flowsheet_template_id}", + "FlowsheetTemplateIDType={flowsheet_template_id_type}", + ) + ), + ) + ) + + def __init__( + self, postgres, uri=URI, poll_timeout=60, uri_timeout=5 + ) -> None: + """__init__.""" + super().__init__(postgres, uri, poll_timeout, uri_timeout) + + def __call__(self): + """__call__.""" + postgres = self.postgres + sql = postgres.sql + with postgres.listen(sql.prediction.listen) as listen: + with postgres.commit() as cur: + self.recover(cur) + self.listen(listen) + + def recover(self, cur): + """Recover.""" + sql = self.postgres.sql + cur.execute(sql.epic.notification.recover) + for each in cur.fetchall(): + self.call_uri(each, cur) + + def on_notify(self, notify): + """On notify.""" + logger.debug(f"NOTIFY: {notify.pid}.{notify.channel}.{notify.payload}") + + def call_uri(self, prediction, cur): + """Call uri.""" + sql = self.postgres.sql + uri = self.uri % { + "csn": prediction["csn"], + "empi": prediction["empi"], + "score": prediction["score"], + } + response = urlopen(uri, self.timeout) + if response.status_code in [200, 201]: + cur.execute( + sql.epic.notification.insert, + {"prediction_id": prediction["id"],}, + ) + + +class Verification: + """Verification Service.""" + + URI = "api/epic/2014/Clinical/Patient/GetFlowsheetRows/FlowsheetRows" + + @classmethod + def main(cls): + """__main__.""" + pass + + def __init__( + self, postgres, uri=URI, poll_timeout=60, uri_timeout=5 + ) -> None: + """__init__.""" + super().__init__(postgres, uri, poll_timeout, uri_timeout) + + def __call__(self): + """__call__.""" + postgres = self.postgres + sql = postgres.sql + with postgres.listen(sql.notification.listen) as listen: + with postgres.commit() as cur: + self.recover(cur) + self.listen(listen) + + def recover(self, cur): + """Recover.""" + sql = self.postgres.sql + cur.execute(sql.epic.verification.recover) + for each in cur.fetchall(): + self.call_uri(each, cur) + + def on_notify(self, notify): + """On notify.""" + logger.debug(f"NOTIFY: {notify.pid}.{notify.channel}.{notify.payload}") + + def call_uri(self, prediction, cur): + """Call uri.""" + sql = self.postgres.sql + response = urlopen(self.uri, self.timeout) + if response.status_code in [200, 201]: + cur.execute(sql.epic.notification.insert, prediction.id) diff --git a/src/dsdk/postgres.py b/src/dsdk/postgres.py index 112a781..88c58f7 100644 --- a/src/dsdk/postgres.py +++ b/src/dsdk/postgres.py @@ -4,6 +4,7 @@ from __future__ import annotations from abc import ABC +from collections import deque from contextlib import contextmanager from json import dumps from logging import getLogger @@ -32,6 +33,7 @@ from psycopg2.extras import execute_batch from psycopg2.extensions import ( register_adapter, + ISOLATION_LEVEL_AUTOCOMMIT, ISQLQuote, Float, AsIs, @@ -93,6 +95,7 @@ class Messages: # pylint: disable=too-few-public-methods ERROR = dumps({"key": f"{KEY}.table.error", "table": "%s"}) ERRORS = dumps({"key": f"{KEY}.tables.error", "tables": "%s"}) EXTANT = dumps({"key": f"{KEY}.sql.extant", "value": "%s"}) + LISTEN = dumps({"key": f"{KEY}.listen", "value": "%s"}) ON = dumps({"key": f"{KEY}.on"}) OPEN = dumps({"key": f"{KEY}.open"}) ROLLBACK = dumps({"key": f"{KEY}.rollback"}) @@ -126,6 +129,25 @@ def check(self, cur, exceptions=(DatabaseError, InterfaceError)): """Check.""" super().check(cur, exceptions) + @contextmanager + def listen(self, *listens: str) -> Generator[Any, None, None]: + """Listen.""" + con = self.retry_connect() + con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + logger.info(self.OPEN) + try: + # replace list with a deque to allow + # users to pop the last notify + con.notifies = deque(con.notifies) + with con.cursor() as cur: + for each in listens: + logger.debug(self.LISTEN, each) + cur.execute(each) + yield con + finally: + con.close() + logger.info(self.CLOSE) + @contextmanager def connect(self) -> Generator[Any, None, None]: """Connect.""" diff --git a/test/test_dsdk.py b/test/test_dsdk.py index ca5e340..55ca3e0 100644 --- a/test/test_dsdk.py +++ b/test/test_dsdk.py @@ -66,7 +66,7 @@ def mixin_kwargs(): host="host", port=1433, database="database", - sql=namespace_directory("./sql/mssql"), + sql=namespace_directory("./asset/mssql"), tables=("foo", "bar", "baz"), ) postgres = PostgresPersistor( @@ -75,7 +75,7 @@ def mixin_kwargs(): host="host", port=5432, database="database", - sql=namespace_directory("./sql/postgres"), + sql=namespace_directory("./asset/postgres"), tables=("foo", "bar", "baz"), ) return { @@ -95,7 +95,7 @@ def mixin_parser_kwargs(): mssql.host = "host" mssql.password = "password" mssql.port = 1433 - mssql.sql = "./sql/mssql" + mssql.sql = "./asset/mssql" mssql.tables = ("foo", "bar", "baz") mssql.username = "username" @@ -104,7 +104,7 @@ def mixin_parser_kwargs(): postgres.host = "host" postgres.password = "password" postgres.port = 5432 - postgres.sql = "./sql/postgres" + postgres.sql = "./asset/postgres" postgres.tables = ("foo", "bar", "baz") postgres.username = "username" diff --git a/test/test_postgres.py b/test/test_postgres.py index 23f7cbc..6774486 100644 --- a/test/test_postgres.py +++ b/test/test_postgres.py @@ -37,7 +37,7 @@ def __init__( "database", env.get("POSTGRES_DATABASE", "test") ), sql=namespace_directory( - kwargs.get("sql", env.get("POSTGRES_SQL", "./sql/postgres")) + kwargs.get("sql", env.get("POSTGRES_SQL", "./asset/postgres")) ), tables=kwargs.get( "tables", From 82ab0e8407194a1a4879b14ed24da138d479d574 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 12 Jul 2021 17:39:48 -0400 Subject: [PATCH 02/33] Add explicit config file arguments --- {asset => assets}/mssql/extant.sql | 0 .../postgres/epic/errors/insert.sql | 0 .../postgres/epic/notifications/channel.sql | 0 .../postgres/epic/notifications/insert.sql | 0 {asset => assets}/postgres/extant.sql | 0 .../postgres/predictions/gold.sql | 0 .../postgres/predictions/insert.sql | 0 .../postgres/predictions/listen.sql | 0 .../postgres/predictions/recover.sql | 0 {asset => assets}/postgres/runs/close.sql | 0 {asset => assets}/postgres/runs/open.sql | 0 {asset => assets}/postgres/schema.sql | 0 docker-compose.yml | 2 +- postgres/dockerfile | 2 +- postgres/sql/initdb.d/003.create.sql | 6 +- postgres/sql/patchdb.d/004.patch.sql | 10 +- postgres/sql/patchdb.d/005.public.sql | 8 +- postgres/sql/patchdb.d/006.private.sql | 4 +- postgres/sql/patchdb.d/007.epic.sql | 4 +- setup.py | 2 +- src/dsdk/dependency.py | 140 ++++++++++++++---- src/dsdk/model.py | 4 +- src/dsdk/mongo.py | 10 +- src/dsdk/persistor.py | 53 ++++--- src/dsdk/service.py | 54 +++---- test/test_dsdk.py | 10 +- test/test_mixin.py | 36 +++-- test/test_postgres.py | 10 +- 28 files changed, 239 insertions(+), 116 deletions(-) rename {asset => assets}/mssql/extant.sql (100%) rename {asset => assets}/postgres/epic/errors/insert.sql (100%) rename {asset => assets}/postgres/epic/notifications/channel.sql (100%) rename {asset => assets}/postgres/epic/notifications/insert.sql (100%) rename {asset => assets}/postgres/extant.sql (100%) rename {asset => assets}/postgres/predictions/gold.sql (100%) rename {asset => assets}/postgres/predictions/insert.sql (100%) rename {asset => assets}/postgres/predictions/listen.sql (100%) rename {asset => assets}/postgres/predictions/recover.sql (100%) rename {asset => assets}/postgres/runs/close.sql (100%) rename {asset => assets}/postgres/runs/open.sql (100%) rename {asset => assets}/postgres/schema.sql (100%) diff --git a/asset/mssql/extant.sql b/assets/mssql/extant.sql similarity index 100% rename from asset/mssql/extant.sql rename to assets/mssql/extant.sql diff --git a/asset/postgres/epic/errors/insert.sql b/assets/postgres/epic/errors/insert.sql similarity index 100% rename from asset/postgres/epic/errors/insert.sql rename to assets/postgres/epic/errors/insert.sql diff --git a/asset/postgres/epic/notifications/channel.sql b/assets/postgres/epic/notifications/channel.sql similarity index 100% rename from asset/postgres/epic/notifications/channel.sql rename to assets/postgres/epic/notifications/channel.sql diff --git a/asset/postgres/epic/notifications/insert.sql b/assets/postgres/epic/notifications/insert.sql similarity index 100% rename from asset/postgres/epic/notifications/insert.sql rename to assets/postgres/epic/notifications/insert.sql diff --git a/asset/postgres/extant.sql b/assets/postgres/extant.sql similarity index 100% rename from asset/postgres/extant.sql rename to assets/postgres/extant.sql diff --git a/asset/postgres/predictions/gold.sql b/assets/postgres/predictions/gold.sql similarity index 100% rename from asset/postgres/predictions/gold.sql rename to assets/postgres/predictions/gold.sql diff --git a/asset/postgres/predictions/insert.sql b/assets/postgres/predictions/insert.sql similarity index 100% rename from asset/postgres/predictions/insert.sql rename to assets/postgres/predictions/insert.sql diff --git a/asset/postgres/predictions/listen.sql b/assets/postgres/predictions/listen.sql similarity index 100% rename from asset/postgres/predictions/listen.sql rename to assets/postgres/predictions/listen.sql diff --git a/asset/postgres/predictions/recover.sql b/assets/postgres/predictions/recover.sql similarity index 100% rename from asset/postgres/predictions/recover.sql rename to assets/postgres/predictions/recover.sql diff --git a/asset/postgres/runs/close.sql b/assets/postgres/runs/close.sql similarity index 100% rename from asset/postgres/runs/close.sql rename to assets/postgres/runs/close.sql diff --git a/asset/postgres/runs/open.sql b/assets/postgres/runs/open.sql similarity index 100% rename from asset/postgres/runs/open.sql rename to assets/postgres/runs/open.sql diff --git a/asset/postgres/schema.sql b/assets/postgres/schema.sql similarity index 100% rename from asset/postgres/schema.sql rename to assets/postgres/schema.sql diff --git a/docker-compose.yml b/docker-compose.yml index ede3192..3a2290b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,4 +36,4 @@ services: - POSTGRES_DATABASE=test - POSTGRES_PORT=5432 volumes: - - ./sql:/sql + - ./assets/sql:/sql diff --git a/postgres/dockerfile b/postgres/dockerfile index 247c05c..c8523f6 100644 --- a/postgres/dockerfile +++ b/postgres/dockerfile @@ -1,4 +1,4 @@ -FROM timescale/timescaledb-postgis:latest-pg11 as postgres +FROM timescale/timescaledb-postgis:latest-pg12 as postgres LABEL name=postgres COPY ./docker-entrypoint.sh /usr/local/bin/ COPY ./sql/initdb.d/*.sql /docker-entrypoint-initdb.d/ diff --git a/postgres/sql/initdb.d/003.create.sql b/postgres/sql/initdb.d/003.create.sql index 4067873..e78e4de 100644 --- a/postgres/sql/initdb.d/003.create.sql +++ b/postgres/sql/initdb.d/003.create.sql @@ -1,5 +1,5 @@ create extension if not exists btree_gist; -create schema if not exists dsdk; -grant usage on schema dsdk to public; -grant create on schema dsdk to public; +create schema if not exists example; +grant usage on schema example to public; +grant create on schema example to public; diff --git a/postgres/sql/patchdb.d/004.patch.sql b/postgres/sql/patchdb.d/004.patch.sql index d302761..9ac1a48 100644 --- a/postgres/sql/patchdb.d/004.patch.sql +++ b/postgres/sql/patchdb.d/004.patch.sql @@ -1,5 +1,7 @@ set search_path = example; +\dn + create or replace function up_patch() returns void as $$ @@ -49,7 +51,7 @@ begin end; $function$ language plpgsql - search_path example; + set search_path = example; exception when duplicate_function then end; @@ -71,7 +73,7 @@ begin end; $function$ language plpgsql - search_path example; + set search_path = example; exception when duplicate_function then end; @@ -81,7 +83,7 @@ begin end; $$ language plpgsql - search_path example; + set search_path = example; create or replace function down_patch() @@ -97,7 +99,7 @@ begin end; $$ language plpgsql - search_path example; + set search_path = example; select up_patch(); diff --git a/postgres/sql/patchdb.d/005.public.sql b/postgres/sql/patchdb.d/005.public.sql index c07db50..fb11b9f 100644 --- a/postgres/sql/patchdb.d/005.public.sql +++ b/postgres/sql/patchdb.d/005.public.sql @@ -19,7 +19,7 @@ begin end; $function$ language plpgsql - search_path example + set search_path = example stable; create domain timezone as varchar @@ -37,7 +37,7 @@ begin end; $function$ language plpgsql - search_path example; + set search_path = example; create table models ( id int primary key generated always as identity, @@ -110,7 +110,7 @@ begin end; $$ language plpgsql - search_path example; + set search_path = example; create or replace function down_public() @@ -131,7 +131,7 @@ begin end; $$ language plpgsql - search_path example; + set search_path = example; select up_public(); diff --git a/postgres/sql/patchdb.d/006.private.sql b/postgres/sql/patchdb.d/006.private.sql index 2dab672..fef2a7e 100644 --- a/postgres/sql/patchdb.d/006.private.sql +++ b/postgres/sql/patchdb.d/006.private.sql @@ -31,7 +31,7 @@ begin end; $$ language plpgsql - search_path example; + set search_path = example; create or replace function down_private() @@ -45,7 +45,7 @@ begin end; $$ language plpgsql - search_path example; + set search_path = example; select up_private(); diff --git a/postgres/sql/patchdb.d/007.epic.sql b/postgres/sql/patchdb.d/007.epic.sql index c9c1e4d..8cc93db 100644 --- a/postgres/sql/patchdb.d/007.epic.sql +++ b/postgres/sql/patchdb.d/007.epic.sql @@ -39,7 +39,7 @@ begin end; $$ language plpgsql - search_path example; + set search_path = example; create or replace function down_epic() @@ -55,7 +55,7 @@ begin end; $$ language plpgsql - search_path example; + set search_path = example; select up_epic(); diff --git a/setup.py b/setup.py index 30dc81d..ed4b5c8 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages, setup INSTALL_REQUIRES = ( - "configargparse>=1.2.3", + "configargparse>=1.5.1", "numpy>=1.15.4", "pandas>=0.23.4", "pip>=20.2.4", diff --git a/src/dsdk/dependency.py b/src/dsdk/dependency.py index 12194c3..e8bb446 100644 --- a/src/dsdk/dependency.py +++ b/src/dsdk/dependency.py @@ -1,13 +1,27 @@ # -*- coding: utf-8 -*- -"""Dependency injection.""" +"""Dependency injection as_type. + +The type parameter for parser.add_arugment(...) is a function/ctor. + +The xxx_as_type functions return the function/ctor used as the type parameter, +creating a closure for the other parameters passed to +xxx_as_type(parameters...). + +This is an alternative to the typical temporal coupling in code where +resolution of what to do with an command line or env argument is separate +code from the declaration of that parameter. +""" from argparse import Namespace from datetime import datetime, timezone, tzinfo +from json import load as json_load from os import listdir from os.path import isdir, join, splitext -from typing import Any, Callable, Dict, Optional, Tuple +from pickle import load as pickle_load +from typing import Any, Callable, Dict, Optional, Tuple, cast from dateutil import parser, tz +from yaml import safe_load as yaml_load class StubException(Exception): @@ -67,54 +81,128 @@ def namespace_directory(root: str = "./", ext: str = ".sql") -> Namespace: return result -def inject_float(key: str, kwargs: Dict[str, Any]) -> Callable: - """Inject float.""" +def as_type_pickle_file(key: str, kwargs: Dict[str, Any]) -> Callable: + """As type pickle.""" + + def _as_type(path: str) -> object: + with open(path) as fin: + result = pickle_load(fin) + if key is None: + kwargs.update(cast(Dict[str:Any], result)) + return result + kwargs[key] = result + return result + - def _inject(value) -> float: +def as_type_yaml_file(key: str, kwargs: Dict[str, Any]) -> Callable: + """As type yaml.""" + + def _as_type(path: str) -> Dict[str, Any]: + with open(path) as fin: + result = yaml_load(fin) + if key is None: + kwargs.update(result) + return result + kwargs[key] = result + return result + + return _as_type + + +def as_type_json_file(key: str, kwargs: Dict[str, Any]) -> Callable: + """As type json.""" + + def _as_type(path: str) -> Dict[str, Any]: + with open(path) as fin: + result = json_load(fin) + if key is None: + kwargs.update(result) + return result + kwargs[key] = result + return result + + return _as_type + + +def as_type_float(key: str, kwargs: Dict[str, Any]) -> Callable: + """As type float.""" + + def _as_type(value) -> float: kwargs[key] = result = float(value) return result - return _inject + return _as_type -def inject_int(key: str, kwargs: Dict[str, Any]) -> Callable: - """Inject int.""" +def as_type_int(key: str, kwargs: Dict[str, Any]) -> Callable: + """As type int.""" - def _inject(value) -> int: + def _as_type(value) -> int: kwargs[key] = result = int(value) return result - return _inject + return _as_type -def inject_str(key: str, kwargs: Dict[str, Any]) -> Callable: - """Inject str.""" +def as_type_str(key: str, kwargs: Dict[str, Any]) -> Callable: + """As type str.""" - def _inject(value: str) -> str: + def _as_type(value: str) -> str: assert value.__class__ is str kwargs[key] = result = value return result - return _inject + return _as_type -def inject_str_tuple(key: str, kwargs: Dict[str, Any]) -> Callable: - """Inject str tuple.""" +def as_type_str_tuple(key: str, kwargs: Dict[str, Any]) -> Callable: + """As type str tuple.""" - def _inject(value: str) -> Tuple[str, ...]: + def _as_type(value: str) -> Tuple[str, ...]: assert value.__class__ is str kwargs[key] = result = tuple(value.split(",")) return result - return _inject + return _as_type + + +def as_type_yaml_tuple(key: str, kwargs: Dict[str, Any]) -> Callable: + """As type yaml tuple.""" + + def _as_type(value: str) -> Tuple[Any, ...]: + kwargs[key] = result = tuple(yaml_load(value)) + return result + + return _as_type + + +def as_type_yaml_list(key: str, kwargs: Dict[str, Any]) -> Callable: + """As type yaml list.""" + + def _as_type(value: str) -> Dict[str, Any]: + kwargs[key] = result = yaml_load(value) + assert result.__class__ is list + + return _as_type + + +def as_type_yaml_dict(key: str, kwargs: Dict[str, Any]) -> Callable: + """As type yaml dict.""" + + def _as_type(value: str) -> Dict[str, Any]: + kwargs[key] = result = yaml_load(value) + assert result.__class__ is dict + return result + + return _as_type -def inject_utc_non_naive_datetime( +def as_type_utc_non_naive_datetime( key: str, kwargs: Dict[str, Any] ) -> Callable: - """Inject utc non-naive datetime.""" + """As type utc non-naive datetime.""" - def _inject(value: str) -> datetime: + def _as_type(value: str) -> datetime: assert value.__class__ is str # dateutil.parser can handle timestamptz output copied # from psql directly @@ -124,15 +212,15 @@ def _inject(value: str) -> datetime: kwargs[key] = result return result - return _inject + return _as_type -def inject_namespace(key: str, kwargs: Dict[str, Any]) -> Callable: - """Inject namespace.""" +def as_type_namespace(key: str, kwargs: Dict[str, Any]) -> Callable: + """As type Namespace.""" - def _inject(value: str) -> Namespace: + def _as_type(value: str) -> Namespace: result = namespace_directory(value) kwargs[key] = result return result - return _inject + return _as_type diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 1865a6a..f92e800 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -10,7 +10,7 @@ from configargparse import ArgParser as ArgumentParser -from .dependency import inject_str +from .dependency import as_type_str from .service import Delegate, Service from .utils import load_pickle_file @@ -51,7 +51,7 @@ def configure( env_var=f"{cls.KEY.upper()}", help="Path to pickled model.", required=True, - type=inject_str("path", kwargs), + type=as_type_str("path", kwargs), ) yield diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py index c0f231f..4da6877 100644 --- a/src/dsdk/mongo.py +++ b/src/dsdk/mongo.py @@ -9,9 +9,9 @@ from logging import getLogger from typing import TYPE_CHECKING, Any, Dict, Generator, Sequence, Tuple, cast -from configargparse import ArgParser as ArgumentParser +from configargparse import ArgumentParser -from .dependency import inject_str +from .dependency import as_type_str from .service import Delegate, Service from .utils import retry @@ -75,7 +75,7 @@ def configure( """Dependencies.""" kwargs: Dict[str, Any] = {} - for key, help_, inject in ( + for key, help_, as_type in ( ( "uri", " ".join( @@ -92,7 +92,7 @@ def configure( "Specifically, check url encoding of PASSWORD.", ) ), - inject_str, + as_type_str, ), ): parser.add( @@ -100,7 +100,7 @@ def configure( env_var=f"{cls.KEY.upper()}_{key.upper()}", help=help_, required=True, - type=inject(key, kwargs), + type=as_type(key, kwargs), ) yield diff --git a/src/dsdk/persistor.py b/src/dsdk/persistor.py index 3c98f26..1f61830 100644 --- a/src/dsdk/persistor.py +++ b/src/dsdk/persistor.py @@ -14,10 +14,10 @@ from pandas import DataFrame, concat from .dependency import ( - inject_int, - inject_namespace, - inject_str, - inject_str_tuple, + as_type_int, + as_type_namespace, + as_type_str, + as_type_yaml_tuple, ) from .service import Service from .utils import chunks @@ -82,10 +82,11 @@ def df_from_query_by_ids( # pylint: disable=too-many-arguments columns = None chunk = None # The sql 'in ()' used for ids is problematic - # - limit on the number of items - # - hard query plan - # - renders as multiple 'or' after query planning - # - poor performance + # - different implementation than set-based join + # - arbitrary limit on the number of items + # - terrible performance + # - renders as multiple 'or' after query planning + # - cpu branch prediction failure? for chunk in chunks(ids, size): cur.execute(query, {"ids": chunk, **parameters}) rows = cur.fetchall() @@ -111,13 +112,13 @@ def df_from_query_by_keys( Query is expected to use {name} for key sequences and %(name)s for parameters. The mogrified fragments produced by union_all are mogrified again. - There is a chance that python placeholders could be injected py the + There is a chance that python placeholders could be injected by the first pass from sequence data. However, it seems that percent in `'...%s...'` or `'...'%(name)s...'` inside string literals produced from the first mogrification pass are not interpreted as parameter placeholders in the second pass by the pymssql driver. - Actual placeholders to by interpolacted by the driver are not + Actual placeholders to by interpolated by the driver are not inside quotes. """ if keys is None: @@ -236,25 +237,37 @@ def configure( # Replace return type with ContextManager[None] when mypy is fixed. kwargs: Dict[str, Any] = {} - for key, help_, inject in ( - ("database", "The database name", inject_str), - ("host", "The database host name or ip address", inject_str), - ("password", "The database password", inject_str), - ("port", "The database port", inject_int), - ("sql", "A nested directory of sql fragments.", inject_namespace), + for key, help_, nargs, as_type in ( + ("database", "The database name", None, as_type_str), + ( + "host", + "The database host name or ip address", + None, + as_type_str, + ), + ("password", "The database password", None, as_type_str), + ("port", "The database port", None, as_type_int), + ( + "sql", + "A nested directory of sql fragments.", + None, + as_type_namespace, + ), ( "tables", - "A comma delimited list of tables to check", - inject_str_tuple, + "A yaml list of tables to check", + "+", + as_type_yaml_tuple, ), - ("username", "The database username", inject_str), + ("username", "The database username", None, as_type_str), ): parser.add( f"--{cls.KEY}-{key}", env_var=f"{cls.KEY.upper()}_{key.upper()}", help=help_, required=True, - type=inject(key, kwargs), + nargs=nargs, + type=as_type(key, kwargs), ) yield diff --git a/src/dsdk/service.py b/src/dsdk/service.py index 2bb15fc..acfd8eb 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -4,11 +4,13 @@ from __future__ import annotations import pickle +from argparse import Namespace from collections import OrderedDict from contextlib import contextmanager from datetime import date, datetime, tzinfo from json import dumps from logging import getLogger +from os import environ from sys import argv as sys_argv from typing import ( Any, @@ -21,16 +23,16 @@ cast, ) -from configargparse import ArgParser as ArgumentParser -from configargparse import Namespace +from configargparse import ArgumentParser from pandas import DataFrame from pkg_resources import DistributionNotFound, get_distribution from .dependency import ( Interval, + as_type_str, + as_type_utc_non_naive_datetime, + as_type_yaml_dict, get_tzinfo, - inject_str, - inject_utc_non_naive_datetime, now_utc_datetime, ) from .utils import configure_logger @@ -317,46 +319,48 @@ def tz_info(self) -> tzinfo: @contextmanager def inject_arguments( # pylint: disable=no-self-use,protected-access - self, parser: ArgumentParser + self, parser: ArgumentParser, env: Dict[str, Any] = environ, ) -> Generator[None, None, None]: """Inject arguments.""" kwargs: Dict[str, Any] = {} - parser._default_config_files = [ - "/local/config.yaml", - "/local/config.yml", - "/local/.yml", - "/secrets/config.yaml", - "/secrets/config.yml", - "/secrets/.yml", - ] - parser._ignore_unknown_config_file_keys = True - parser.add( + # TODO: force config file to be .yaml file + parser.add_argument( "-c", "--config", is_config_file=True, help="config file path", - env_var="CONFIG", # make ENV match default metavar + default="/local/config.yaml", + env_var="CONFIG", ) - parser.add( + # TODO: force secrets file to be .env file + parser.add_argument( + "-s", + "--secrets", + is_config_file=True, + default="/secrets/secrets.env", + env_var="SECRETS", + help="secrets env file path", + ) + parser.add_argument( "-d", "--as-of", - help="as of utc non-naive datetime", env_var="AS_OF", - type=inject_utc_non_naive_datetime("as_of", kwargs), + help="as of utc non-naive datetime", + type=as_type_utc_non_naive_datetime("as_of", kwargs), ) - parser.add( + parser.add_argument( "-g", "--gold", - help="gold validation file", env_var="GOLD", - type=inject_str("gold", kwargs), + help="gold validation file", + type=as_type_str("gold", kwargs), ) - parser.add( + parser.add_argument( "-t", "--time-zone", - help="time_zone", env_var="TIME_ZONE", - type=inject_str("time_zone", kwargs), + help="time_zone", + type=as_type_str("time_zone", kwargs), ) yield diff --git a/test/test_dsdk.py b/test/test_dsdk.py index 55ca3e0..faa6f75 100644 --- a/test/test_dsdk.py +++ b/test/test_dsdk.py @@ -66,7 +66,7 @@ def mixin_kwargs(): host="host", port=1433, database="database", - sql=namespace_directory("./asset/mssql"), + sql=namespace_directory("./assets/mssql"), tables=("foo", "bar", "baz"), ) postgres = PostgresPersistor( @@ -75,7 +75,7 @@ def mixin_kwargs(): host="host", port=5432, database="database", - sql=namespace_directory("./asset/postgres"), + sql=namespace_directory("./assets/postgres"), tables=("foo", "bar", "baz"), ) return { @@ -87,7 +87,7 @@ def mixin_kwargs(): def mixin_parser_kwargs(): """Return mixin parser kwargs.""" - model = "./model.pkl" + model = "./model/model.pkl" dump_pickle_file(Model(name="test", version="0.0.1"), model) mssql = Namespace() @@ -95,7 +95,7 @@ def mixin_parser_kwargs(): mssql.host = "host" mssql.password = "password" mssql.port = 1433 - mssql.sql = "./asset/mssql" + mssql.sql = "./assets/mssql" mssql.tables = ("foo", "bar", "baz") mssql.username = "username" @@ -104,7 +104,7 @@ def mixin_parser_kwargs(): postgres.host = "host" postgres.password = "password" postgres.port = 5432 - postgres.sql = "./asset/postgres" + postgres.sql = "./assets/postgres" postgres.tables = ("foo", "bar", "baz") postgres.username = "username" diff --git a/test/test_mixin.py b/test/test_mixin.py index 38b1a58..93b32b0 100644 --- a/test/test_mixin.py +++ b/test/test_mixin.py @@ -28,6 +28,9 @@ cast, ) +from yaml import safe_dump as yaml_dump +from yaml import safe_load as yaml_load + def dump_pickle_file(obj, path: str) -> None: """Dump pickle to file.""" @@ -53,6 +56,18 @@ def load_json_file(path: str) -> object: return json_load(fin) +def dump_yaml_file(obj, path: str) -> None: + """Dump yaml to file.""" + with open(path, "w") as fout: + yaml_dump(obj, fout) + + +def load_yaml_file(path: str) -> object: + """Load yaml from file.""" + with open(path, "r") as fin: + return yaml_load(fin) + + class Service: # pylint: disable=too-few-public-methods """Service.""" @@ -85,15 +100,16 @@ def inject_args( """Inject args.""" # In this example, the injected parameters here are simple: - # because no post parser.parse configuration is needed, - # but this is not necesarily the case. - # Using only one configuration file is not desirable if - # if it is more difficult to validate. - # Concider using separate files for tabular configuration. - # This is particularly true for the mixins. + # because no configuration is needed after the parse.parse + # call, but this is not necesarily the case. + # Notice that the configuration is not validated before it is + # used, but it should be. + # Also using only one configuration file is not desirable if + # if it is more difficult to edit and validate. + # Consider using separate files for tabular configuration. def _inject_cfg(path: str) -> Dict[str, Any]: - cfg = cast(Dict[str, Any], load_json_file(path)) + cfg = cast(Dict[str, Any], load_yaml_file(path)) self.cfg = cfg return cfg @@ -103,7 +119,7 @@ def _inject_cfg(path: str) -> Dict[str, Any]: yield # after parser.parse call - # load_json_file or a more complex constructor with multiple parameters + # load_*_file or a more complex constructor with multiple parameters # could be here instead of in the single parameter parse callback # _inject_cfg. @@ -221,8 +237,8 @@ def __init__(self, *, i: int = 0, **kwargs): def test_mixin_with_parser(): """Test mixin with parser.""" - model_path = "./model.pkl" - cfg_path = "./cfg.json" + model_path = "./model/model.pkl" + cfg_path = "./local/test.yaml" obj = {} dump_pickle_file(obj, model_path) dump_json_file(obj, cfg_path) diff --git a/test/test_postgres.py b/test/test_postgres.py index 6774486..06f2c4a 100644 --- a/test/test_postgres.py +++ b/test/test_postgres.py @@ -37,7 +37,7 @@ def __init__( "database", env.get("POSTGRES_DATABASE", "test") ), sql=namespace_directory( - kwargs.get("sql", env.get("POSTGRES_SQL", "./asset/postgres")) + kwargs.get("sql", env.get("POSTGRES_SQL", "./assets/postgres")) ), tables=kwargs.get( "tables", @@ -45,10 +45,10 @@ def __init__( "POSTGRES_TABLES", ",".join( ( - "dsdk.models", - "dsdk.microservices", - "dsdk.runs", - "dsdk.predictions", + "example.models", + "example.microservices", + "example.runs", + "example.predictions", ) ), ).split(","), From 703a1fd1c17a768e4e47f37b9687098298ac7b53 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Fri, 16 Jul 2021 23:57:08 -0400 Subject: [PATCH 03/33] Add yaml inits and reprs. --- postgres/sql/patchdb.d/005.public.sql | 2 +- src/dsdk/__init__.py | 26 ++- src/dsdk/asset.py | 71 ++++++ src/dsdk/dependency.py | 226 ------------------ src/dsdk/env.py | 63 ++++++ src/dsdk/epic.py | 2 +- src/dsdk/interval.py | 38 ++++ src/dsdk/model.py | 86 ++++--- src/dsdk/mongo.py | 277 ----------------------- src/dsdk/mssql.py | 36 +-- src/dsdk/persistor.py | 105 ++++----- src/dsdk/postgres.py | 48 ++-- src/dsdk/service.py | 253 ++++++++++++--------- src/dsdk/utils.py | 54 +++++ test/test_dsdk.py | 171 +++++++------- test/test_mixin.py | 314 -------------------------- test/test_postgres.py | 16 +- test/test_time_conversions.py | 2 +- 18 files changed, 596 insertions(+), 1194 deletions(-) create mode 100644 src/dsdk/asset.py delete mode 100644 src/dsdk/dependency.py create mode 100644 src/dsdk/env.py create mode 100644 src/dsdk/interval.py delete mode 100644 src/dsdk/mongo.py delete mode 100644 test/test_mixin.py diff --git a/postgres/sql/patchdb.d/005.public.sql b/postgres/sql/patchdb.d/005.public.sql index fb11b9f..f98b040 100644 --- a/postgres/sql/patchdb.d/005.public.sql +++ b/postgres/sql/patchdb.d/005.public.sql @@ -8,7 +8,7 @@ begin return; end if; - create or replace function is_timezone(time_zone varchar) + create function is_timezone(time_zone varchar) returns boolean as $function$ declare valid timestamptz; begin diff --git a/src/dsdk/__init__.py b/src/dsdk/__init__.py index 944eb5d..5fc60e4 100644 --- a/src/dsdk/__init__.py +++ b/src/dsdk/__init__.py @@ -1,18 +1,17 @@ # -*- coding: utf-8 -*- """Data Science Deployment Kit.""" -from .dependency import Interval, namespace_directory, now_utc_datetime +from .asset import Asset +from .env import Env +from .interval import Interval from .model import Mixin as ModelMixin from .model import Model -from .mongo import EvidenceMixin as MongoEvidenceMixin -from .mongo import Mixin as MongoMixin -from .mongo import Persistor as MongoPersistor from .mssql import CheckTablePrivileges as CheckMssqlTablePrivileges from .mssql import Mixin as MssqlMixin -from .mssql import Persistor as MssqlPersistor +from .mssql import Persistor as Mssql from .postgres import CheckTablePrivileges as CheckPostgresTablePrivileges from .postgres import Mixin as PostgresMixin -from .postgres import Persistor as PostgresPersistor +from .postgres import Persistor as Postgres from .postgres import PredictionMixin as PostgresPredictionMixin from .service import Batch, Delegate, Service, Task from .utils import ( @@ -20,37 +19,40 @@ configure_logger, dump_json_file, dump_pickle_file, + dump_yaml_file, load_json_file, load_pickle_file, + load_yaml_file, + now_utc_datetime, profile, retry, ) __all__ = ( + "Asset", "Batch", "Delegate", + "Env", "Interval", "Model", "ModelMixin", - "MongoMixin", - "MongoPersistor", - "MongoEvidenceMixin", "MssqlMixin", - "MssqlPersistor", + "Mssql", "CheckMssqlTablePrivileges", "CheckPostgresTablePrivileges", "PostgresPredictionMixin", "PostgresMixin", - "PostgresPersistor", + "Postgres", "Service", "Task", "chunks", "configure_logger", "dump_json_file", "dump_pickle_file", + "dump_yaml_file", "load_json_file", "load_pickle_file", - "namespace_directory", + "load_yaml_file", "profile", "now_utc_datetime", "retry", diff --git a/src/dsdk/asset.py b/src/dsdk/asset.py new file mode 100644 index 0000000..b2892f9 --- /dev/null +++ b/src/dsdk/asset.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +"""Asset.""" + +from __future__ import annotations + +from argparse import Namespace +from logging import getLogger +from os import listdir +from os.path import isdir +from os.path import join as joinpath +from os.path import splitext +from typing import Any, Dict + +from yaml import SafeDumper, SafeLoader, add_constructor, add_representer + +logger = getLogger(__name__) + + +class Asset(Namespace): + """Asset.""" + + YAML = "!asset" + + @classmethod + def as_yaml_type(cls): + """As yaml type.""" + add_constructor(cls.YAML, cls._yaml_init, Loader=SafeLoader) + add_representer(cls, cls._yaml_repr, Dumper=SafeDumper) + + @classmethod + def build(cls, *, path: str, ext: str): + """Build.""" + kwargs = {} + for name in listdir(path): + if name[0] == ".": + continue + child = joinpath(path, name) + if isdir(child): + kwargs[name] = cls.build(path=child, ext=ext) + continue + s_name, s_ext = splitext(name) + if s_ext != ext: + continue + with open(child) as fin: + kwargs[s_name] = fin.read() + return cls(path=path, ext=ext, **kwargs) + + @classmethod + def _yaml_init(cls, loader, node): + """Yaml init.""" + return cls.build(**loader.construct_mapping(node, deep=True)) + + @classmethod + def _yaml_repr(cls, dumper, self): + """Yaml repr.""" + return dumper.represent_mapping(cls.YAML, self.as_yaml()) + + def __init__( + self, *, path: str, ext: str, **kwargs: Asset, + ): + """__init__.""" + self.path = path + self.ext = ext + super().__init__(**kwargs) + + def as_yaml(self) -> Dict[str, Any]: + """As yaml.""" + return { + "ext": self.ext, + "path": self.path, + } diff --git a/src/dsdk/dependency.py b/src/dsdk/dependency.py deleted file mode 100644 index e8bb446..0000000 --- a/src/dsdk/dependency.py +++ /dev/null @@ -1,226 +0,0 @@ -# -*- coding: utf-8 -*- -"""Dependency injection as_type. - -The type parameter for parser.add_arugment(...) is a function/ctor. - -The xxx_as_type functions return the function/ctor used as the type parameter, -creating a closure for the other parameters passed to -xxx_as_type(parameters...). - -This is an alternative to the typical temporal coupling in code where -resolution of what to do with an command line or env argument is separate -code from the declaration of that parameter. -""" - -from argparse import Namespace -from datetime import datetime, timezone, tzinfo -from json import load as json_load -from os import listdir -from os.path import isdir, join, splitext -from pickle import load as pickle_load -from typing import Any, Callable, Dict, Optional, Tuple, cast - -from dateutil import parser, tz -from yaml import safe_load as yaml_load - - -class StubException(Exception): - """StubException.""" - - -class Interval: # pylint: disable=too-few-public-methods - """Interval.""" - - def __init__(self, on: datetime, end: Optional[datetime] = None): - """__init__.""" - self.on = on - self.end = end - - def as_doc(self) -> Dict[str, Any]: - """As doc.""" - return {"end": self.end, "on": self.on} - - -def epoch_ms_from_utc_datetime(utc: datetime) -> float: - """Epoch ms from non-naive UTC datetime.""" - return utc.timestamp() * 1000 - - -def utc_datetime_from_epoch_ms(epoch_ms: float) -> datetime: - """Non-naive UTC datetime from UTC epoch ms.""" - return datetime.fromtimestamp(epoch_ms / 1000, tz=timezone.utc) - - -def now_utc_datetime() -> datetime: - """Non-naive now UTC datetime.""" - return datetime.now(tz=timezone.utc) - - -def get_tzinfo(key: str) -> tzinfo: - """Get tzinfo.""" - result = tz.gettz(key) - assert result is not None - return result - - -def namespace_directory(root: str = "./", ext: str = ".sql") -> Namespace: - """Return namespace from code directory.""" - result = Namespace() - for name in listdir(root): - if name[0] == ".": - continue - path = join(root, name) - if isdir(path): - setattr(result, name, namespace_directory(path, ext)) - continue - s_name, s_ext = splitext(name) - if s_ext != ext: - continue - with open(path) as fin: - setattr(result, s_name, fin.read()) - return result - - -def as_type_pickle_file(key: str, kwargs: Dict[str, Any]) -> Callable: - """As type pickle.""" - - def _as_type(path: str) -> object: - with open(path) as fin: - result = pickle_load(fin) - if key is None: - kwargs.update(cast(Dict[str:Any], result)) - return result - kwargs[key] = result - return result - - -def as_type_yaml_file(key: str, kwargs: Dict[str, Any]) -> Callable: - """As type yaml.""" - - def _as_type(path: str) -> Dict[str, Any]: - with open(path) as fin: - result = yaml_load(fin) - if key is None: - kwargs.update(result) - return result - kwargs[key] = result - return result - - return _as_type - - -def as_type_json_file(key: str, kwargs: Dict[str, Any]) -> Callable: - """As type json.""" - - def _as_type(path: str) -> Dict[str, Any]: - with open(path) as fin: - result = json_load(fin) - if key is None: - kwargs.update(result) - return result - kwargs[key] = result - return result - - return _as_type - - -def as_type_float(key: str, kwargs: Dict[str, Any]) -> Callable: - """As type float.""" - - def _as_type(value) -> float: - kwargs[key] = result = float(value) - return result - - return _as_type - - -def as_type_int(key: str, kwargs: Dict[str, Any]) -> Callable: - """As type int.""" - - def _as_type(value) -> int: - kwargs[key] = result = int(value) - return result - - return _as_type - - -def as_type_str(key: str, kwargs: Dict[str, Any]) -> Callable: - """As type str.""" - - def _as_type(value: str) -> str: - assert value.__class__ is str - kwargs[key] = result = value - return result - - return _as_type - - -def as_type_str_tuple(key: str, kwargs: Dict[str, Any]) -> Callable: - """As type str tuple.""" - - def _as_type(value: str) -> Tuple[str, ...]: - assert value.__class__ is str - kwargs[key] = result = tuple(value.split(",")) - return result - - return _as_type - - -def as_type_yaml_tuple(key: str, kwargs: Dict[str, Any]) -> Callable: - """As type yaml tuple.""" - - def _as_type(value: str) -> Tuple[Any, ...]: - kwargs[key] = result = tuple(yaml_load(value)) - return result - - return _as_type - - -def as_type_yaml_list(key: str, kwargs: Dict[str, Any]) -> Callable: - """As type yaml list.""" - - def _as_type(value: str) -> Dict[str, Any]: - kwargs[key] = result = yaml_load(value) - assert result.__class__ is list - - return _as_type - - -def as_type_yaml_dict(key: str, kwargs: Dict[str, Any]) -> Callable: - """As type yaml dict.""" - - def _as_type(value: str) -> Dict[str, Any]: - kwargs[key] = result = yaml_load(value) - assert result.__class__ is dict - return result - - return _as_type - - -def as_type_utc_non_naive_datetime( - key: str, kwargs: Dict[str, Any] -) -> Callable: - """As type utc non-naive datetime.""" - - def _as_type(value: str) -> datetime: - assert value.__class__ is str - # dateutil.parser can handle timestamptz output copied - # from psql directly - result = parser.parse(value) - assert result.tzinfo == tz.tzutc() - result.replace(tzinfo=timezone.utc) - kwargs[key] = result - return result - - return _as_type - - -def as_type_namespace(key: str, kwargs: Dict[str, Any]) -> Callable: - """As type Namespace.""" - - def _as_type(value: str) -> Namespace: - result = namespace_directory(value) - kwargs[key] = result - return result - - return _as_type diff --git a/src/dsdk/env.py b/src/dsdk/env.py new file mode 100644 index 0000000..e59ed9e --- /dev/null +++ b/src/dsdk/env.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +"""Env.""" + +from __future__ import annotations + +from os import environ as os_env +from re import compile as re_compile +from typing import Mapping, Optional + +from yaml import SafeLoader, add_constructor, add_implicit_resolver + + +class Env: + """Env.""" + + YAML = "!env" + PATTERN = re_compile(r".*?\$\{(\w+)\}.*?") + + @classmethod + def as_yaml_type(cls, *, env: Optional[Mapping[str, str]] = None): + """As yaml type.""" + _env = env or os_env + + def _yaml_init(loader, node) -> str: + """This closure passed env.""" + return cls._yaml_init(loader, node, _env) + + add_implicit_resolver(cls.YAML, cls.PATTERN, None, Loader=SafeLoader) + add_constructor(cls.YAML, _yaml_init, Loader=SafeLoader) + + @classmethod + def _yaml_init(cls, loader, node, env: Mapping[str, str]): + """From yaml.""" + value = loader.construct_scalar(node) + match = cls.PATTERN.findall(value) + if not match: + return value + for group in match: + variable = env.get(group, None) + if not variable: + raise ValueError(f"No value for ${{{group}}}.") + value = value.replace(f"${{{group}}}", variable) + return value + + @classmethod + def load(cls, path: str) -> Mapping[str, str]: + """Env load.""" + with open(path) as fin: + return cls.loads(fin.read()) + + @classmethod + def loads(cls, envs: str) -> Mapping[str, str]: + """Env loads.""" + result = {} + for line in envs.split("\n"): + line = line.strip() + if not line: + continue + if line.startswith("#"): + continue + key, value = line.split("=", 1) + result[key] = value + return result diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index dbbe0fb..815d981 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -93,7 +93,7 @@ def call_uri(self, prediction, cur): if response.status_code in [200, 201]: cur.execute( sql.epic.notification.insert, - {"prediction_id": prediction["id"],}, + {"prediction_id": prediction["id"]}, ) diff --git a/src/dsdk/interval.py b/src/dsdk/interval.py new file mode 100644 index 0000000..9aac3ba --- /dev/null +++ b/src/dsdk/interval.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +"""Interval.""" + +from datetime import datetime +from typing import Any, Dict, Optional + +from yaml import SafeDumper, SafeLoader, add_constructor, add_representer + + +class Interval: + """Interval.""" + + YAML = "!interval" + + @classmethod + def as_yaml_type(cls): + """As yaml type.""" + add_constructor(cls.YAML, cls._yaml_init, Loader=SafeLoader) + add_representer(cls, cls._yaml_repr, Dumper=SafeDumper) + + @classmethod + def _yaml_init(cls, loader, node): + """Yaml init.""" + return cls(**loader.construct_mapping(node, deep=True)) + + @classmethod + def _yaml_repr(cls, dumper, self): + """Yaml repr.""" + return dumper.represent_mapping(cls.YAML, self.as_yaml()) + + def __init__(self, on: datetime, end: Optional[datetime] = None): + """__init__.""" + self.on = on + self.end = end + + def as_yaml(self) -> Dict[str, Any]: + """As yaml.""" + return {"end": self.end, "on": self.on} diff --git a/src/dsdk/model.py b/src/dsdk/model.py index f92e800..3c09a39 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -6,11 +6,10 @@ from abc import ABC from contextlib import contextmanager from logging import getLogger -from typing import TYPE_CHECKING, Any, Dict, Generator, Type, cast +from typing import TYPE_CHECKING, Any, Dict, Generator, Optional -from configargparse import ArgParser as ArgumentParser +from yaml import SafeDumper, SafeLoader, add_constructor, add_representer -from .dependency import as_type_str from .service import Delegate, Service from .utils import load_pickle_file @@ -26,42 +25,43 @@ class Model: # pylint: disable=too-few-public-methods """Model.""" - KEY = "model" + YAML = "!model" @classmethod - def load(cls, path: str) -> Model: - """Load.""" + def as_yaml_type(cls) -> None: + """As yaml type.""" + add_constructor(cls.YAML, cls._yaml_init, Loader=SafeLoader) + add_representer(cls, cls._yaml_repr, Dumper=SafeDumper) + + @classmethod + def _yaml_init(cls, loader, node): + """Yaml init.""" + path = loader.construct_scalar(node) pkl = load_pickle_file(path) if pkl.__class__ is dict: - assert pkl.__class__ is dict - return cls(**pkl) # type: ignore + pkl = cls(path=path, **pkl) # type: ignore + else: + pkl.path = path assert isinstance(pkl, Model) return pkl @classmethod - @contextmanager - def configure( - cls, service: Service, parser - ) -> Generator[None, None, None]: - """Dependencies.""" - kwargs: Dict[str, Any] = {} - - parser.add( - f"--{cls.KEY}", - env_var=f"{cls.KEY.upper()}", - help="Path to pickled model.", - required=True, - type=as_type_str("path", kwargs), - ) - yield - - service.dependency(cls.KEY, cls.load, kwargs) - - def __init__(self, name: str, version: str) -> None: + def _yaml_repr(cls, dumper, self): + """Yaml_repr.""" + return dumper.represent_scalar(cls.YAML, self.as_yaml()) + + def __init__( + self, *, name: str, version: str, path: Optional[str] = None, + ) -> None: """__init__.""" self.name = name + self.path = path self.version = version + def as_yaml(self) -> Dict[str, Any]: + """As yaml.""" + return {"path": self.path} + class Batch(Delegate): """Batch.""" @@ -71,13 +71,6 @@ def __init__(self, model_version: str, parent: Any): super().__init__(parent) self.model_version = model_version - def as_insert_doc(self) -> Dict[str, Any]: - """As insert doc.""" - return { - "model_version": self.model_version, - **self.parent.as_insert_doc(), - } - def as_insert_sql(self) -> Dict[str, Any]: """As insert sql.""" return { @@ -89,20 +82,23 @@ def as_insert_sql(self) -> Dict[str, Any]: class Mixin(BaseMixin): """Mixin.""" - def __init__(self, *, model=None, model_cls: Type = Model, **kwargs): + @classmethod + def yaml_types(cls) -> None: + """As yaml types.""" + Model.as_yaml_type() + super().yaml_types() + + def __init__(self, *, model: Model, **kwargs): """__init__.""" - self.model = cast(Model, model) - self.model_cls = model_cls + self.model = model super().__init__(**kwargs) - @contextmanager - def inject_arguments( - self, parser: ArgumentParser - ) -> Generator[None, None, None]: - """Inject arguments.""" - with self.model_cls.configure(self, parser): - with super().inject_arguments(parser): - yield + def as_yaml(self) -> Dict[str, Any]: + """As yaml.""" + return { + "model": self.model, + **super().as_yaml(), + } @contextmanager def open_batch(self) -> Generator[Any, None, None]: diff --git a/src/dsdk/mongo.py b/src/dsdk/mongo.py deleted file mode 100644 index 4da6877..0000000 --- a/src/dsdk/mongo.py +++ /dev/null @@ -1,277 +0,0 @@ -# -*- coding: utf-8 -*- -"""Mongo.""" - -from __future__ import annotations - -from abc import ABC -from contextlib import contextmanager -from json import dumps -from logging import getLogger -from typing import TYPE_CHECKING, Any, Dict, Generator, Sequence, Tuple, cast - -from configargparse import ArgumentParser - -from .dependency import as_type_str -from .service import Delegate, Service -from .utils import retry - -try: - # Since not everyone will use mongo - from bson.objectid import ObjectId - from pymongo import MongoClient - from pymongo.collection import Collection - from pymongo.database import Database - from pymongo.errors import AutoReconnect -except ImportError: - MongoClient = None - ObjectId = None - Database = None - AutoReconnect = None - -logger = getLogger(__name__) - - -if TYPE_CHECKING: - BaseMixin = Service -else: - BaseMixin = ABC - - -class Messages: # pylint: disable=too-few-public-methods - """Messages.""" - - KEY = "mongo" - - CLOSE = dumps({"key": f"{KEY}.close", "database": "%s"}) - OPEN = dumps({"key": f"{KEY}.open", "database": "%s"}) - RESULTSET_ERROR = dumps( - { - "actual": "%s", - "collection": "%s.%s", - "expected": "%s", - "key": f"{KEY}.resultset.error", - } - ) - - INSERT_ONE = dumps( - {"collection": "%s.%s", "id": "%s", "key": f"{KEY}.insert_one"} - ) - - INSERT_MANY = dumps( - {"collection": "%s.%s", "key": f"{KEY}.insert_many", "value": "%s"} - ) - - UPDATE_ONE = dumps({"collection": "%s.%s", "key": f"{KEY}.update_one"}) - - -class Persistor(Messages): - """Persistor.""" - - @classmethod - @contextmanager - def configure( - cls, service: Service, parser - ) -> Generator[None, None, None]: - """Dependencies.""" - kwargs: Dict[str, Any] = {} - - for key, help_, as_type in ( - ( - "uri", - " ".join( - ( - "Mongo URI used to connect to a Mongo database:", - ( - "mongodb://USER:PASSWORD@HOST1,HOST2,.../DATABASE?" - "replicaset=REPLICASET&authsource=admin" - ), - "Use a valid uri." - "Url encode parts, do not encode the entire uri.", - "No unencoded colons, ampersands, slashes,", - "question-marks, etc. in parts.", - "Specifically, check url encoding of PASSWORD.", - ) - ), - as_type_str, - ), - ): - parser.add( - f"--{cls.KEY}-{key}", - env_var=f"{cls.KEY.upper()}_{key.upper()}", - help=help_, - required=True, - type=as_type(key, kwargs), - ) - yield - - service.dependency(cls.KEY, cls, kwargs) - - def __init__(self, uri: str): - """__init__.""" - self.uri = uri - self.document_class = dict - self.tz_aware = True - self.connect_ = True - - @contextmanager - def connect(self, **kwargs) -> Generator[Database, None, None]: - """Contextmanager for database. - - Ensures that the mongo connection is opened and closed. - MongoClient also has defaults for its own: - - reconnectTries (30) - - reconnectInterval (1000ms) - - no backoff - """ - with MongoClient( - self.uri, - document_class=self.document_class, - tz_aware=self.tz_aware, - connect=self.connect_, - **kwargs, - ) as client: - database = client.get_database() - # retry to allowdb to spin up - # force lazy connection open with actual command - client.admin.command("ismaster") - logger.info(self.OPEN, database.name) - try: - yield database - finally: - logger.info(self.CLOSE, database.name) - - @retry(AutoReconnect) - def insert_one(self, collection: Collection, doc: Dict[str, Any]): - """Insert one with retry.""" - result = collection.insert_one(doc) - logger.info( - self.INSERT_ONE, - collection.database.name, - collection.name, - result.inserted_id, - ) - return result - - @retry(AutoReconnect) - def insert_many( - self, collection: Collection, docs: Sequence[Dict[str, Any]] - ): - """Insert many with retry.""" - if not docs: - return None - result = collection.insert_many(docs) - logger.info( - self.INSERT_MANY, - collection.database.name, - collection.name, - len(result.inserted_ids), - ) - return result - - def store_evidence(self, batch: Any, *args, **kwargs) -> None: - """Store Evidence.""" - exclude = kwargs.get("exclude", ()) - evidence = batch.evidence - while args: - key, df, *args = args # type: ignore - evidence[key] = df - # TODO We need to check column types and convert as needed - # TODO Find a way to add batch_id without mutating df - df["batch_id"] = batch.key - columns = df[[c for c in df.columns if c not in exclude]] - docs = columns.to_dict(orient="records") - with self.connect() as database: - collection = database[key] - result = self.insert_many(collection, docs) - actual = len(result.inserted_ids) - expected = columns.shape[0] - assert actual == expected, self.RESULTSET_ERROR % ( - database.name, - collection.name, - actual, - expected, - ) - - # TODO: Better exception - df.drop(columns=["batch_id"], inplace=True) - - @retry(AutoReconnect) - def update_one( - self, collection: Collection, key: Dict[str, Any], doc: Dict[str, Any] - ): - """Update one with retry.""" - result = collection.update_one(key, doc) - logger.info( - self.UPDATE_ONE, collection.database.name, collection.name, - ) - return result - - -class Mixin(BaseMixin): - """Mixin.""" - - def __init__( - self, - *, - mongo=None, - mongo_uri=None, # pylint: disable=unused-argument - mongo_cls=Persistor, - **kwargs, - ): - """__init__.""" - self.mongo = cast(Persistor, mongo) - self.mongo_cls = mongo_cls - super().__init__(**kwargs) - - @contextmanager - def inject_arguments( - self, parser: ArgumentParser, - ) -> Generator[None, None, None]: - """Inject arguments.""" - # Replace return type with ContextManager[None] when mypy is fixed. - with self.mongo_cls.configure(self, parser): - with super().inject_arguments(parser): - yield - - -class Batch(Delegate): - """Batch.""" - - def __init__(self, parent: Any): - """__init__.""" - super().__init__(parent) - self.key = ObjectId() - - def as_insert_doc(self) -> Dict[str, Any]: - """As insert doc.""" - return { - "_id": self.key, - **self.parent.as_insert_doc(), - } - - def as_update_doc(self) -> Tuple[Dict[str, Any], Dict[str, Any]]: - """As update doc.""" - keys, values = self.parent.as_update_doc() - return ( - {"_id": self.key, **keys}, - values, - ) - - -class EvidenceMixin(Mixin): - """Evidence Mixin.""" - - @contextmanager - def open_batch(self) -> Generator[Batch, None, None]: - """Open batch.""" - mongo = self.mongo - with super().open_batch() as parent: - batch = Batch(parent) - doc = batch.as_insert_doc() - with mongo.connect() as database: - key = mongo.insert_one(database.batches, doc) - yield batch - - key, values = batch.as_update_doc() - with mongo.connect() as database: - mongo.update_one(database.batches, key, {"$set": values}) diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index c55fac5..699faed 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -7,13 +7,11 @@ from contextlib import contextmanager from json import dumps from logging import getLogger -from typing import TYPE_CHECKING, Any, Generator, Type, cast +from typing import TYPE_CHECKING, Any, Dict, Generator -from configargparse import ArgParser as ArgumentParser - -from .dependency import StubException from .persistor import Persistor as BasePersistor from .service import Service, Task +from .utils import StubError logger = getLogger(__name__) @@ -23,7 +21,7 @@ except ImportError as import_error: logger.warning(import_error) - DatabaseError = InterfaceError = StubException + DatabaseError = InterfaceError = StubError def connect(*args, **kwargs): """Connect stub.""" @@ -56,6 +54,8 @@ class Messages: # pylint: disable=too-few-public-methods class Persistor(Messages, BasePersistor): """Persistor.""" + YAML = "!mssql" + @classmethod def mogrify(cls, cur, query: str, parameters: Any,) -> bytes: """Safely mogrify parameters into query or fragment.""" @@ -111,21 +111,23 @@ def connect(self) -> Generator[Any, None, None]: class Mixin(BaseMixin): """Mixin.""" - def __init__(self, *, mssql=None, mssql_cls: Type = Persistor, **kwargs): + @classmethod + def yaml_types(cls) -> None: + """Yaml types.""" + Persistor.as_yaml_type() + super().yaml_types() + + def __init__(self, *, mssql: Persistor = None, **kwargs): """__init__.""" - self.mssql = cast(Persistor, mssql) - self.mssql_cls = mssql_cls + self.mssql = mssql super().__init__(**kwargs) - @contextmanager - def inject_arguments( - self, parser: ArgumentParser - ) -> Generator[None, None, None]: - """Inject arguments.""" - # Replace return type with ContextManager[Any] when mypy is fixed. - with self.mssql_cls.configure(self, parser): - with super().inject_arguments(parser): - yield + def as_yaml(self) -> Dict[str, Any]: + """As yaml.""" + return { + "mssql": self.mssql, + **super().as_yaml(), + } class CheckTablePrivileges(Task): # pylint: disable=too-few-public-methods diff --git a/src/dsdk/persistor.py b/src/dsdk/persistor.py index 1f61830..0d92b9f 100644 --- a/src/dsdk/persistor.py +++ b/src/dsdk/persistor.py @@ -3,7 +3,6 @@ from __future__ import annotations -from argparse import Namespace from contextlib import contextmanager from json import dumps from logging import getLogger @@ -12,14 +11,9 @@ from typing import Any, Dict, Generator, Optional, Sequence, Tuple from pandas import DataFrame, concat +from yaml import SafeDumper, SafeLoader, add_constructor, add_representer -from .dependency import ( - as_type_int, - as_type_namespace, - as_type_str, - as_type_yaml_tuple, -) -from .service import Service +from .asset import Asset from .utils import chunks logger = getLogger(__name__) @@ -43,12 +37,6 @@ class AbstractPersistor: OPEN = dumps({"key": f"{KEY}.open"}) ROLLBACK = dumps({"key": f"{KEY}.rollback"}) - @classmethod - @contextmanager - def configure(cls, service: Service, parser): - """Configure.""" - raise NotImplementedError() - @classmethod def df_from_query( cls, cur, query: str, parameters: Optional[Dict[str, Any]], @@ -152,7 +140,7 @@ def union_all(cls, cur, keys: Sequence[Any],) -> str: union = cls.mogrify(cur, union, parameters).decode("utf-8") return union - def __init__(self, sql: Namespace, tables: Tuple[str, ...]): + def __init__(self, sql: Asset, tables: Tuple[str, ...]): """__init__.""" self.sql = sql self.tables = tables @@ -228,70 +216,55 @@ def rollback(self) -> Generator[Any, None, None]: class Persistor(AbstractPersistor): """Persistor.""" + YAML = "" + @classmethod - @contextmanager - def configure( - cls, service: Service, parser - ) -> Generator[None, None, None]: - """Configure.""" - # Replace return type with ContextManager[None] when mypy is fixed. - kwargs: Dict[str, Any] = {} - - for key, help_, nargs, as_type in ( - ("database", "The database name", None, as_type_str), - ( - "host", - "The database host name or ip address", - None, - as_type_str, - ), - ("password", "The database password", None, as_type_str), - ("port", "The database port", None, as_type_int), - ( - "sql", - "A nested directory of sql fragments.", - None, - as_type_namespace, - ), - ( - "tables", - "A yaml list of tables to check", - "+", - as_type_yaml_tuple, - ), - ("username", "The database username", None, as_type_str), - ): - parser.add( - f"--{cls.KEY}-{key}", - env_var=f"{cls.KEY.upper()}_{key.upper()}", - help=help_, - required=True, - nargs=nargs, - type=as_type(key, kwargs), - ) - - yield - - service.dependency(cls.KEY, cls, kwargs) + def as_yaml_type(cls) -> None: + """As yaml type.""" + add_constructor(cls.YAML, cls._yaml_init, Loader=SafeLoader) + add_representer(cls, cls._yaml_repr, Dumper=SafeDumper) + + @classmethod + def _yaml_init(cls, loader, node): + """Yaml init.""" + return cls(**loader.construct_mapping(node, deep=True)) + + @classmethod + def _yaml_repr(cls, dumper, self): + """Yaml repr.""" + return dumper.represent_mapping(cls.YAML, self.as_yaml()) def __init__( # pylint: disable=too-many-arguments self, - username: str, - password: str, + *, + database: str, host: str, + password: str, port: int, - database: str, - sql: Namespace, + sql: Asset, tables: Tuple[str, ...], + username: str, ): """__init__.""" - self.username = username - self.password = password + self.database = database self.host = host + self.password = password self.port = port - self.database = database + self.username = username super().__init__(sql, tables) + def as_yaml(self) -> Dict[str, Any]: + """As yaml.""" + return { + "database": self.database, + "host": self.host, + "password": self.password, + "port": self.port, + "sql": self.sql, + "tables": self.tables, + "username": self.username, + } + @contextmanager def connect(self) -> Generator[Any, None, None]: """Connect.""" diff --git a/src/dsdk/postgres.py b/src/dsdk/postgres.py index 88c58f7..8ca281a 100644 --- a/src/dsdk/postgres.py +++ b/src/dsdk/postgres.py @@ -8,16 +8,15 @@ from contextlib import contextmanager from json import dumps from logging import getLogger -from typing import TYPE_CHECKING, Any, Dict, Generator, Type, cast +from typing import TYPE_CHECKING, Any, Dict, Generator -from configargparse import ArgParser as ArgumentParser from numpy import integer from pandas import DataFrame, NaT, Series, isna -from .dependency import Interval, StubException +from .interval import Interval from .persistor import Persistor as BasePersistor from .service import Delegate, Service, Task -from .utils import retry +from .utils import StubError, retry logger = getLogger(__name__) @@ -71,7 +70,7 @@ def getquoted(self): except ImportError as import_error: logger.warning(import_error) - DatabaseError = InterfaceError = OperationalError = StubException + DatabaseError = InterfaceError = OperationalError = StubError def connect(*args, **kwargs): """Connect stub.""" @@ -120,6 +119,8 @@ class Messages: # pylint: disable=too-few-public-methods class Persistor(Messages, BasePersistor): """Persistor.""" + YAML = "!postgres" + @classmethod def mogrify(cls, cur, query: str, parameters: Any,) -> bytes: """Safely mogrify parameters into query or fragment.""" @@ -284,23 +285,23 @@ def _store_df( class Mixin(BaseMixin): """Mixin.""" - def __init__( - self, *, postgres=None, postgres_cls: Type = Persistor, **kwargs, - ): + @classmethod + def yaml_types(cls) -> None: + """Yaml types.""" + Persistor.as_yaml_type() + super().yaml_types() + + def __init__(self, *, postgres: Persistor, **kwargs): """__init__.""" - self.postgres = cast(Persistor, postgres) - self.postgres_cls = postgres_cls + self.postgres = postgres super().__init__(**kwargs) - @contextmanager - def inject_arguments( - self, parser: ArgumentParser - ) -> Generator[None, None, None]: - """Inject arguments.""" - # Replace return type with ContextManager[None] when mypy is fixed. - with self.postgres_cls.configure(self, parser): - with super().inject_arguments(parser): - yield + def as_yaml(self) -> Dict[str, Any]: + """As yaml.""" + return { + "postgres": self.postgres, + **super().as_yaml(), + } def scores(self, run_id) -> Series: """Get scores.""" @@ -319,15 +320,6 @@ def __init__( # pylint: disable=too-many-arguments self.microservice_id = microservice_id self.model_id = model_id - def as_insert_doc(self) -> Dict[str, Any]: - """As insert doc.""" - return { - "run_id": self.id, - "microservice_id": self.microservice_id, - "model_id": self.model_id, - **self.parent.as_insert_doc(), - } - class PredictionMixin(Mixin): # pylint: disable=too-few-public-methods. """Prediction Mixin.""" diff --git a/src/dsdk/service.py b/src/dsdk/service.py index acfd8eb..142a289 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -4,38 +4,34 @@ from __future__ import annotations import pickle -from argparse import Namespace +from argparse import ArgumentParser from collections import OrderedDict from contextlib import contextmanager from datetime import date, datetime, tzinfo from json import dumps from logging import getLogger -from os import environ +from os import environ as os_env from sys import argv as sys_argv from typing import ( Any, Callable, Dict, Generator, + List, + Mapping, Optional, Sequence, - Tuple, - cast, ) -from configargparse import ArgumentParser from pandas import DataFrame from pkg_resources import DistributionNotFound, get_distribution +from yaml import SafeDumper, SafeLoader, add_constructor, add_representer +from yaml import safe_load as yaml_loads -from .dependency import ( - Interval, - as_type_str, - as_type_utc_non_naive_datetime, - as_type_yaml_dict, - get_tzinfo, - now_utc_datetime, -) -from .utils import configure_logger +from .asset import Asset +from .env import Env +from .interval import Interval +from .utils import configure_logger, get_tzinfo, now_utc_datetime try: __version__ = get_distribution("dsdk").version @@ -123,18 +119,10 @@ def tz_info(self) -> tzinfo: """Return tzinfo.""" return self.parent.tz_info - def as_insert_doc(self) -> Dict[str, Any]: - """As insert doc.""" - return self.parent.as_insert_doc() - def as_insert_sql(self) -> Dict[str, Any]: """As insert sql.""" return self.parent.as_insert_sql() - def as_update_doc(self) -> Tuple[Dict[str, Any], Dict[str, Any]]: - """As update doc.""" - return self.parent.as_update_doc() - class Batch: """Batch.""" @@ -178,14 +166,6 @@ def parent(self) -> Any: """Return parent.""" raise ValueError() - def as_insert_doc(self) -> Dict[str, Any]: - """As insert doc.""" - return { - "as_of": self.as_of, - "microservice_version": self.microservice_version, - "time_zone": self.time_zone, - } - def as_insert_sql(self) -> Dict[str, Any]: """As insert sql.""" # duration comes from the database clock. @@ -195,11 +175,6 @@ def as_insert_sql(self) -> Dict[str, Any]: "time_zone": self.time_zone, } - def as_update_doc(self) -> Tuple[Dict[str, Any], Dict[str, Any]]: - """As update doc.""" - assert self.duration is not None - return {}, {"duration": self.duration.as_doc()} - class Evidence(OrderedDict): """Evidence.""" @@ -226,18 +201,36 @@ class Service: # pylint: disable=too-many-instance-attributes {"key": "validate.count", "scores": "%s", "test": "%s", "status": "%s"} ) MATCH = dumps({"key": "validate.match", "status": "%s"}) + YAML = "" VERSION = __version__ + @classmethod + def as_yaml_type(cls) -> None: + """As yaml type.""" + add_constructor(cls.YAML, cls._yaml_init, Loader=SafeLoader) + add_representer(cls, cls._yaml_repr, Dumper=SafeDumper) + @classmethod @contextmanager - def context(cls, key: str): + def context( + cls, + key: str, + argv: Optional[List[str]] = None, + env: Optional[Mapping[str, str]] = None, + ): """Context.""" configure_logger("dsdk") logger.info(cls.ON, key) - yield cls(parser=ArgumentParser()) + yield cls.parse(argv=argv, env=env) logger.info(cls.END, key) + @classmethod + def create_gold(cls): + """Create gold.""" + with cls.context("create_gold") as service: + service.on_create_gold() + @classmethod def main(cls): """Main.""" @@ -245,10 +238,91 @@ def main(cls): service() @classmethod - def create_gold(cls): - """Create gold.""" - with cls.context("create_gold") as service: - service.on_create_gold() + def parse( + cls, + *, + argv: Optional[List[str]] = None, + env: Optional[Mapping[str, str]] = None, + ) -> Service: + """Parse.""" + if argv is None: + argv = sys_argv[1:] + assert argv is not None + if env is None: + env = os_env + assert env is not None + + parser = ArgumentParser() + parser.add_argument( + "-c", + "--config", + dest="config_file", + type=str, + help="configuration yaml file", + default=env.get("CONFIG", None), + ) + parser.add_argument( + "-e", + "--env", + dest="env_file", + type=str, + help="env file", + default=env.get("env", None), + ) + args = parser.parse_args(argv) + if args.config_file is None: + parser.error( + " ".join( + ( + "the following arguments are required:", + "-c/--config or CONFIG from env variable", + ) + ) + ) + return cls.load( + config_file=args.config_file, env=env, env_file=args.env_file, + ) + + @classmethod + def load( + cls, + *, + config_file: str, + env: Optional[Mapping[str, str]] = None, + env_file: Optional[str] = None, + ): + """Load.""" + if env is None: + env = os_env + assert env is not None + + if env_file: + with open(env_file) as fin: + envs = fin.read() + logger.debug("Environment variables are only from %s", env_file) + else: + logger.debug("Environment variables are being used.") + + with open(config_file) as fin: + configs = fin.read() + return cls.loads(configs=configs, env=env, envs=envs) + + @classmethod + def loads( + cls, + *, + configs: str, + env: Mapping[str, str], + envs: Optional[str] = None, + ) -> Service: + """Loads from strings.""" + if envs is not None: + env = Env.loads(envs) + + Env.as_yaml_type(env=env) + + cls.yaml_types() + return yaml_loads(configs) @classmethod def validate_gold(cls): @@ -256,35 +330,37 @@ def validate_gold(cls): with cls.context("validate_gold") as service: service.on_validate_gold() + @classmethod + def yaml_types(cls) -> None: + """Yaml types.""" + Asset.as_yaml_type() + Interval.as_yaml_type() + + @classmethod + def _yaml_init(cls, loader, node): + """Yaml init.""" + return cls(**loader.construct_mapping(node, deep=True)) + + @classmethod + def _yaml_repr(cls, dumper, self): + """Yaml repr.""" + return dumper.represent_mapping(cls.YAML, self.as_yaml()) + def __init__( # pylint: disable=too-many-arguments self, - argv: Optional[Sequence[str]] = None, - parser: Optional[ArgumentParser] = None, - pipeline: Optional[Sequence[Task]] = None, + pipeline: Sequence[Task] = None, as_of: Optional[datetime] = None, gold: Optional[str] = None, - time_zone: Optional[str] = None, + time_zone: Optional[datetime] = None, batch_cls: Callable = Batch, ) -> None: """__init__.""" - self.args: Optional[Namespace] = None - self.parser = parser self.gold = gold - - # inferred type of self.pipeline must not be optional... - self.pipeline = cast(Sequence[Task], pipeline) + self.pipeline = pipeline self.duration: Optional[Interval] = None self.as_of = as_of self.time_zone = time_zone self.batch_cls = batch_cls - if parser: - with self.inject_arguments(parser): - if not argv: - argv = sys_argv[1:] - self.args = parser.parse_args(argv) - - # ... because self.pipeline is not optional - assert self.pipeline is not None def __call__(self) -> Batch: """Run.""" @@ -317,58 +393,6 @@ def tz_info(self) -> tzinfo: assert self.time_zone is not None return get_tzinfo(self.time_zone) - @contextmanager - def inject_arguments( # pylint: disable=no-self-use,protected-access - self, parser: ArgumentParser, env: Dict[str, Any] = environ, - ) -> Generator[None, None, None]: - """Inject arguments.""" - kwargs: Dict[str, Any] = {} - # TODO: force config file to be .yaml file - parser.add_argument( - "-c", - "--config", - is_config_file=True, - help="config file path", - default="/local/config.yaml", - env_var="CONFIG", - ) - # TODO: force secrets file to be .env file - parser.add_argument( - "-s", - "--secrets", - is_config_file=True, - default="/secrets/secrets.env", - env_var="SECRETS", - help="secrets env file path", - ) - parser.add_argument( - "-d", - "--as-of", - env_var="AS_OF", - help="as of utc non-naive datetime", - type=as_type_utc_non_naive_datetime("as_of", kwargs), - ) - parser.add_argument( - "-g", - "--gold", - env_var="GOLD", - help="gold validation file", - type=as_type_str("gold", kwargs), - ) - parser.add_argument( - "-t", - "--time-zone", - env_var="TIME_ZONE", - help="time_zone", - type=as_type_str("time_zone", kwargs), - ) - - yield - - self.gold = kwargs.get("gold") - self.as_of = kwargs.get("as_of") - self.time_zone = kwargs.get("time_zone") - def dependency(self, key, cls, kwargs): """Dependency.""" dependency = getattr(self, key) @@ -383,6 +407,15 @@ def dependency(self, key, cls, kwargs): dependency = cls(**kwargs) setattr(self, key, dependency) + def as_yaml(self) -> Dict[str, Any]: + """As yaml.""" + return { + "as_of": self.as_of, + "duration": self.duration, + "gold": self.gold, + "time_zone": self.time_zone, + } + def on_create_gold(self) -> Batch: """On create gold.""" path = self.gold diff --git a/src/dsdk/utils.py b/src/dsdk/utils.py index a7bdbe2..9e499fe 100644 --- a/src/dsdk/utils.py +++ b/src/dsdk/utils.py @@ -4,6 +4,7 @@ from __future__ import annotations from contextlib import contextmanager +from datetime import datetime, timezone, tzinfo from functools import wraps from json import dump as json_dump from json import load as json_load @@ -15,9 +16,24 @@ from time import sleep as default_sleep from typing import Any, Callable, Generator, Sequence +from dateutil import parser, tz +from yaml import safe_dump as yaml_dumps +from yaml import safe_load as yaml_loads + logger = getLogger(__name__) +def as_utc_non_naive_datetime(value: str) -> datetime: + """As utc non-naive datetime.""" + assert value.__class__ is str + # dateutil.parser can handle timestamptz output copied + # from psql directly + result = parser.parse(value) + assert result.tzinfo == tz.tzutc() + result.replace(tzinfo=timezone.utc) + return result + + def configure_logger(name, level=INFO): """Configure logger. @@ -59,6 +75,24 @@ def dump_pickle_file(obj: Any, path: str) -> None: pickle_dump(obj, fout) +def dump_yaml_file(obj: Any, path: str) -> None: + """Dump yaml file.""" + with open(path, "w") as fout: + yaml_dumps(obj, fout) + + +def epoch_ms_from_utc_datetime(utc: datetime) -> float: + """Epoch ms from non-naive UTC datetime.""" + return utc.timestamp() * 1000 + + +def get_tzinfo(key: str) -> tzinfo: + """Get tzinfo.""" + result = tz.gettz(key) + assert result is not None + return result + + def load_json_file(path: str) -> object: """Load json from file.""" with open(path, "r") as fin: @@ -71,6 +105,17 @@ def load_pickle_file(path: str) -> object: return pickle_load(fin) +def load_yaml_file(path: str): + """Load yaml file.""" + with open(path) as fin: + return yaml_loads(fin.read()) + + +def now_utc_datetime() -> datetime: + """Non-naive now UTC datetime.""" + return datetime.now(tz=timezone.utc) + + @contextmanager def profile(key: str) -> Generator[Any, None, None]: """Profile.""" @@ -127,3 +172,12 @@ def wrapped(*args, **kwargs): return wrapped return wrapper + + +def utc_datetime_from_epoch_ms(epoch_ms: float) -> datetime: + """Non-naive UTC datetime from UTC epoch ms.""" + return datetime.fromtimestamp(epoch_ms / 1000, tz=timezone.utc) + + +class StubError(Exception): + """StubError.""" diff --git a/test/test_dsdk.py b/test/test_dsdk.py index faa6f75..1b52ca1 100644 --- a/test/test_dsdk.py +++ b/test/test_dsdk.py @@ -1,25 +1,23 @@ # -*- coding: utf-8 -*- """Test dsdk.""" -from argparse import Namespace -from typing import Any, Dict +from typing import Any, Callable, Dict, Tuple -from configargparse import ArgParser as ArgumentParser from pandas import DataFrame from pytest import mark from dsdk import ( + Asset, Batch, Model, ModelMixin, + Mssql, MssqlMixin, - MssqlPersistor, + Postgres, PostgresMixin, - PostgresPersistor, Service, Task, dump_pickle_file, - namespace_directory, retry, ) @@ -54,109 +52,104 @@ def __call__(self, batch: Batch, service: Service) -> None: assert batch.evidence["test"] is df -def mixin_kwargs(): - """Return mixin kwargs.""" +class TestService(ModelMixin, MssqlMixin, PostgresMixin, Service): + """TestService.""" - # TODO - # do not use filesystem for init of sql namespaces - model = Model("test", "0.0.1-rc.1") - mssql = MssqlPersistor( + YAML = "!test" + + @classmethod + def yaml_types(cls): + """Yaml types.""" + cls.as_yaml_type() + super().yaml_types() + + def __init__(self, **kwargs): + """__init__.""" + pipeline = (_Extract, _Transform, _Predict) + super().__init__(pipeline=pipeline, **kwargs) + + +def build_from_parameters() -> Tuple[Callable, Dict[str, Any]]: + """Build from parameters.""" + model = Model(name="test", path="./test/model.pkl", version="0.0.1-rc.1") + mssql = Mssql( username="username", password="password", host="host", port=1433, database="database", - sql=namespace_directory("./assets/mssql"), + sql=Asset.build(path="./assets/mssql", ext=".sql"), tables=("foo", "bar", "baz"), ) - postgres = PostgresPersistor( + postgres = Postgres( username="username", password="password", host="host", port=5432, database="database", - sql=namespace_directory("./assets/postgres"), + sql=Asset.build(path="./assets/postgres", ext=".sql"), tables=("foo", "bar", "baz"), ) - return { - "model": model, - "mssql": mssql, - "postgres": postgres, - } - - -def mixin_parser_kwargs(): - """Return mixin parser kwargs.""" - model = "./model/model.pkl" - dump_pickle_file(Model(name="test", version="0.0.1"), model) - - mssql = Namespace() - mssql.database = "test" - mssql.host = "host" - mssql.password = "password" - mssql.port = 1433 - mssql.sql = "./assets/mssql" - mssql.tables = ("foo", "bar", "baz") - mssql.username = "username" - - postgres = Namespace() - postgres.database = "test" - postgres.host = "host" - postgres.password = "password" - postgres.port = 5432 - postgres.sql = "./assets/postgres" - postgres.tables = ("foo", "bar", "baz") - postgres.username = "username" - - argv = [ - "--model", - model, - "--mssql-database", - mssql.database, - "--mssql-host", - mssql.host, - "--mssql-password", - mssql.password, - "--mssql-port", - str(mssql.port), - "--mssql-sql", - mssql.sql, - "--mssql-tables", - ",".join(mssql.tables), - "--mssql-username", - mssql.username, - "--postgres-database", - postgres.database, - "--postgres-host", - postgres.host, - "--postgres-password", - postgres.password, - "--postgres-port", - str(postgres.port), - "--postgres-sql", - postgres.sql, - "--postgres-tables", - ",".join(postgres.tables), - "--postgres-username", - postgres.username, - ] - parser = ArgumentParser(argv) - return {"argv": argv, "parser": parser} + return ( + TestService, + {"model": model, "mssql": mssql, "postgres": postgres}, + ) + +def build_from_yaml() -> Tuple[Callable, Dict[str, Any]]: + """Build from yaml.""" + pickle_file = "./test/model.pkl" + dump_pickle_file( + Model(name="test", path=pickle_file, version="0.0.1"), pickle_file + ) -@mark.parametrize("kwargs", (mixin_kwargs(), mixin_parser_kwargs())) -def test_mixin_service(kwargs: Dict[str, Any]): - """Test postgres, mssql mixin.""" + configs = """ +!test +mssql: !mssql + database: test + host: 0.0.0.0 + password: ${MSSQL_PASSWORD} + port: 1433 + sql: ./assets/mssql + tables: + - foo + - bar + - baz + username: mssql +model: !model ./test/model.pkl +postgres: !postgres + database: test + host: 0.0.0.0 + password: ${POSTGRES_PASSWORD} + port: 5432 + sql: ./asset/postgres + tables: + - foo + - bar + - baz + username: postgres +""" + env = {"POSTGRES_PASSWORD": "oops!", "MSSQL_PASSWORD": "oops!"} + envs = """ +MSSQL_PASSWORD=password +POSTGRES_PASSWORD=password +""" + return ( + TestService.loads, + {"configs": configs, "env": env, "envs": envs}, + ) - class _Service(MssqlMixin, PostgresMixin, ModelMixin, Service): - def __init__(self, **kwargs): - pipeline = (_Extract, _Transform, _Predict) - super().__init__(pipeline=pipeline, **kwargs) - service = _Service(**kwargs) - assert service.postgres.__class__ is PostgresPersistor - assert service.mssql.__class__ is MssqlPersistor +@mark.parametrize("cls,kwargs", (build_from_yaml(), build_from_parameters())) +def test_service(cls, kwargs: Dict[str, Any]): + """Test parameters, config, and env.""" + service = cls(**kwargs) + assert service.__class__ is TestService assert service.model.__class__ is Model + assert service.postgres.__class__ is Postgres + assert service.mssql.__class__ is Mssql + assert service.postgres.password == "password" + assert service.mssql.password == "password" def test_retry_other_exception(): diff --git a/test/test_mixin.py b/test/test_mixin.py deleted file mode 100644 index 93b32b0..0000000 --- a/test/test_mixin.py +++ /dev/null @@ -1,314 +0,0 @@ -# -*- coding: utf-8 -*- -"""Test mixin.""" -# Goals: -# - test mixin -# - self-use with inject_args -# - avoid additional method like set_config -# - make parser optional -# - demo dependency injection - -from __future__ import annotations - -from abc import ABC -from argparse import ArgumentParser -from contextlib import contextmanager -from json import dump as json_dump -from json import load as json_load -from pickle import dump as pickle_dump -from pickle import load as pickle_load -from sys import argv as sys_argv -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Generator, - Optional, - Sequence, - cast, -) - -from yaml import safe_dump as yaml_dump -from yaml import safe_load as yaml_load - - -def dump_pickle_file(obj, path: str) -> None: - """Dump pickle to file.""" - with open(path, "wb") as fout: - pickle_dump(obj, fout) - - -def load_pickle_file(path: str) -> object: - """Load pickle from file.""" - with open(path, "rb") as fin: - return pickle_load(fin) - - -def dump_json_file(obj, path: str) -> None: - """Dump json to file.""" - with open(path, "w") as fout: - json_dump(obj, fout) - - -def load_json_file(path: str) -> object: - """Load json from file.""" - with open(path, "r") as fin: - return json_load(fin) - - -def dump_yaml_file(obj, path: str) -> None: - """Dump yaml to file.""" - with open(path, "w") as fout: - yaml_dump(obj, fout) - - -def load_yaml_file(path: str) -> object: - """Load yaml from file.""" - with open(path, "r") as fin: - return yaml_load(fin) - - -class Service: # pylint: disable=too-few-public-methods - """Service.""" - - def __init__( - self, - *, - argv: Optional[Sequence[str]] = None, - cfg: Optional[Dict[str, Any]] = None, - i: int = 0, - parser: Optional[ArgumentParser] = None, - ) -> None: - """__init__.""" - self.service_i = i - self.cfg = cast(Dict[str, Any], cfg) - self.parser = parser - - # parsing arguments must be optional - if parser: - with self.inject_args(parser): - if not argv: - argv = sys_argv[1:] - parser.parse_args(argv) - - assert self.cfg is not None - - @contextmanager - def inject_args( - self, parser: ArgumentParser - ) -> Generator[None, None, None]: - """Inject args.""" - - # In this example, the injected parameters here are simple: - # because no configuration is needed after the parse.parse - # call, but this is not necesarily the case. - # Notice that the configuration is not validated before it is - # used, but it should be. - # Also using only one configuration file is not desirable if - # if it is more difficult to edit and validate. - # Consider using separate files for tabular configuration. - - def _inject_cfg(path: str) -> Dict[str, Any]: - cfg = cast(Dict[str, Any], load_yaml_file(path)) - self.cfg = cfg - return cfg - - parser.add_argument("--cfg", type=_inject_cfg) - - # before parser.parse call - yield - # after parser.parse call - - # load_*_file or a more complex constructor with multiple parameters - # could be here instead of in the single parameter parse callback - # _inject_cfg. - - -if TYPE_CHECKING: - Mixin = Service -else: - Mixin = ABC - - -class ModelMixin(Mixin): # pylint: disable=too-few-public-methods - """Model Mixin.""" - - def __init__( - self, *, i: int = 0, model: Optional[Dict[str, Any]] = None, **kwargs - ) -> None: - """__init__.""" - self.model_i = i - self.model = cast(Dict[str, Any], model) - super().__init__(i=i + 1, **kwargs) - - # self.model is not optional - assert self.model is not None - - @contextmanager - def inject_args( - self, parser: ArgumentParser - ) -> Generator[None, None, None]: - """Inject args.""" - - def _inject_model(path: str) -> Dict[str, Any]: - model = cast(Dict[str, Any], load_pickle_file(path)) - self.model = model - return model - - parser.add_argument("--model", type=_inject_model) - # before parser.parse call - with super().inject_args(parser): - yield - # after parser.parse call - - -class Postgres: # pylint: disable=too-few-public-methods - """Postgres.""" - - def __init__( - self, username: str, password: str, host: str, database: str, - ) -> None: - """__init__.""" - self.username = username - self.password = password - self.host = host - self.database = database - - -class PostgresMixin(Mixin): # pylint: disable=too-few-public-methods - """Postgres Mixin.""" - - def __init__( - self, *, i: int = 0, postgres: Optional[Postgres] = None, **kwargs, - ) -> None: - """__init__.""" - self.postgres_i = i - self.postgres = cast(Postgres, postgres) - super().__init__(i=i + 1, **kwargs) - - # self.model is not optional - assert self.postgres is not None - - @contextmanager - def inject_args( - self, parser: ArgumentParser - ) -> Generator[None, None, None]: - """Inject args.""" - kwargs: Dict[str, Any] = {} - - def _inject_str(key: str) -> Callable[[str], str]: - def _inject(value: str) -> str: - kwargs[key] = value - return value - - return _inject - - parser.add_argument("--username", type=_inject_str("username")) - parser.add_argument("--password", type=_inject_str("password")) - parser.add_argument("--host", type=_inject_str("host")) - parser.add_argument("--database", type=_inject_str("database")) - - # before parser.parse call - with super().inject_args(parser): - yield - # after parser.parse call - - self.postgres = Postgres(**kwargs) - - -# Service must be last in inheritence -# to ensure mixin methods are all called. -class App( - PostgresMixin, ModelMixin, Service -): # pylint: disable=too-few-public-methods - """App.""" - - def __init__(self, *, i: int = 0, **kwargs): - """__init__.""" - self.app_i = i - super().__init__(i=i + 1, **kwargs) - - # Assert correct order of initialization - assert self.app_i == 0 - assert self.postgres_i == 1 - assert self.model_i == 2 - assert self.service_i == 3 - - -def test_mixin_with_parser(): - """Test mixin with parser.""" - model_path = "./model/model.pkl" - cfg_path = "./local/test.yaml" - obj = {} - dump_pickle_file(obj, model_path) - dump_json_file(obj, cfg_path) - - argv = [ - "--cfg", - cfg_path, - "--model", - model_path, - "--username", - "username", - "--password", - "password", - "--host", - "host", - "--database", - "database", - ] - - parser = ArgumentParser() - App(parser=parser, argv=argv) - - -def test_mixin_without_parser(): - """Test mixin without parser.""" - model: Dict[str, Any] = {} - cfg: Dict[str, Any] = {} - username = "username" - password = "password" - host = "host" - database = "database" - postgres = Postgres( - username=username, password=password, host=host, database=database, - ) - - App( - cfg=cfg, model=model, postgres=postgres, - ) - - -def test_mixins_are_abstract(): - """Test mixins are abstract.""" - # pylint: disable=abstract-class-instantiated - username = "username" - password = "password" - host = "host" - database = "database" - - try: - PostgresMixin( - username=username, password=password, host=host, database=database, - ) - raise AssertionError("Ensure PostgresMixin is abstract.") - except TypeError: - pass - - model: Dict[str, Any] = {} - try: - ModelMixin(model=model) - raise AssertionError("Ensure ModelMixin is abstract.") - except TypeError: - pass - - -def main(): - """Main.""" - test_mixin_with_parser() - test_mixin_without_parser() - test_mixins_are_abstract() - - -if __name__ == "__main__": - main() diff --git a/test/test_postgres.py b/test/test_postgres.py index 06f2c4a..4c5dcd3 100644 --- a/test/test_postgres.py +++ b/test/test_postgres.py @@ -2,19 +2,18 @@ """Test postgres.""" from contextlib import contextmanager -from os import environ +from os import environ as os_env from typing import Any, Generator from pandas import DataFrame, read_sql_query -from dsdk import Batch, PostgresPersistor, configure_logger -from dsdk.dependency import namespace_directory +from dsdk import Asset, Batch, Postgres, configure_logger from dsdk.model import Batch as ModelBatch logger = configure_logger(__name__) -class Persistor(PostgresPersistor): +class Persistor(Postgres): """Persistor.""" def __init__( @@ -22,7 +21,7 @@ def __init__( ): """__init__.""" if env is None: - env = environ + env = os_env self.attempts = 0 super().__init__( username=kwargs.get( @@ -36,8 +35,11 @@ def __init__( database=kwargs.get( "database", env.get("POSTGRES_DATABASE", "test") ), - sql=namespace_directory( - kwargs.get("sql", env.get("POSTGRES_SQL", "./assets/postgres")) + sql=Asset.build( + path=kwargs.get( + "sql", env.get("POSTGRES_SQL", "./assets/postgres") + ), + ext=".sql", ), tables=kwargs.get( "tables", diff --git a/test/test_time_conversions.py b/test/test_time_conversions.py index fc8c39b..7df86b2 100644 --- a/test/test_time_conversions.py +++ b/test/test_time_conversions.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Test epoch_ms and utc datetime conversions.""" -from dsdk.dependency import ( +from dsdk.utils import ( epoch_ms_from_utc_datetime, now_utc_datetime, utc_datetime_from_epoch_ms, From cb04fae900e5249b03a19acca61d52e5ad3165b5 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 19 Jul 2021 14:17:16 -0400 Subject: [PATCH 04/33] Fix yaml loaders and dumpers --- .pre-commit-config.yaml | 12 ++++--- postgres/sql/patchdb.d/007.epic.sql | 40 +++++++++++++++++++--- pyproject.toml | 5 ++- setup.py | 1 + src/dsdk/asset.py | 17 +++++++--- src/dsdk/env.py | 9 +++-- src/dsdk/epic.py | 52 +++++++++++++++++++++-------- src/dsdk/interval.py | 11 ++++-- src/dsdk/model.py | 22 ++++++++---- src/dsdk/mssql.py | 11 ++++-- src/dsdk/persistor.py | 31 +++++++++++++---- src/dsdk/postgres.py | 49 ++++++++++++++++++++------- src/dsdk/service.py | 37 ++++++++++++++------ test/test_dsdk.py | 26 +++++++++------ test/test_postgres.py | 4 ++- 15 files changed, 243 insertions(+), 84 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 19f56a3..6dad9ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,19 +18,21 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: http://github.com/pre-commit/pygrep-hooks - rev: v1.4.2 + rev: v1.9.0 hooks: - id: python-check-blanket-noqa + - id: python-check-blanket-type-ignore + - id: python-no-eval + - id: python-no-log-warn - id: python-use-type-annotations - id: rst-backticks -# - repo: https://github.com/pre-commit/mirrors-isort -- repo: https://github.com/timothycrosley/isort - rev: 4.3.21-2 +- repo: https://github.com/pycqa/isort + rev: 5.8.0 hooks: - id: isort additional_dependencies: [toml] - repo: https://github.com/psf/black - rev: 19.10b0 + rev: 21.7b0 hooks: - id: black language_version: python3.9 diff --git a/postgres/sql/patchdb.d/007.epic.sql b/postgres/sql/patchdb.d/007.epic.sql index 8cc93db..e70717e 100644 --- a/postgres/sql/patchdb.d/007.epic.sql +++ b/postgres/sql/patchdb.d/007.epic.sql @@ -11,7 +11,7 @@ begin create table epic_notifications ( id int primary key generated always as identity, prediction_id int not null, - notified_on timestamptz default statement_timestamp(), + recorded_on timestamptz default statement_timestamp(), constraint only_one_epic_notification_per_prediction unique (prediction_id), constraint epic_notifications_require_a_prediction @@ -24,18 +24,48 @@ begin for each statement execute procedure call_notify(); - create table epic_errors ( + create table epic_notification_errors ( id int primary key generated always as identity, prediction_id int not null, recorded_on timestamptz default statement_timestamp(), acknowledged_on timestamptz default null, - error_name varchar, - error_description varchar, - constraint epic_errors_require_a_prediction + name varchar, + description varchar, + constraint epic_notification_errors_require_a_prediction foreign key (prediction_id) references predictions (id) on delete cascade on update cascade ); + + create table epic_verifications ( + id int primary key generated always as identity, + notification_id int not null, + recorded_on timestamptz default statement_timestamp(), + constraint only_one_epic_verification_per_notification + unique (notification_id), + constraint epic_verifications_require_a_notification + foreign key (notification_id) references epic_notifications (id) + on delete cascade + on update cascade + ); + create trigger epci_verifications_inserted after insert on epic_verifications + referencing new table as inserted + for each statement + execute procedure call_notify(); + + create table epic_verification_errors ( + id int primary key generated always as identity, + notification_id int not null, + recorded_on timestamptz default statement_timestamp(), + acknowledged_on timestamptz default null, + name varchar, + description varchar, + constraint epic_verification_errors_require_a_notification + foreign key (notification_id) references epic_notifications (id) + on delete cascade + on update cascade + ); + end; $$ language plpgsql diff --git a/pyproject.toml b/pyproject.toml index 6d47238..094852d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,9 @@ good-names = '''a,b,c,d,df,e,i,id,logger,n,on,tz''' [tool.pylint.message_control] disable = '''duplicate-code,C0330''' +extension-pkg-allow-list = [ + "pymssql._mssql", +] [tool.pylint.miscellaneous] notes = '''FIXME,XXX''' @@ -57,7 +60,7 @@ ignore-docstrings = "yes" ignore-imports = "yes" [tool.pytest.ini_options] -addopts = "-ra --cov=dsdk --cov-report=term-missing --strict --ignore=.eggs --tb=short" +addopts = "-ra --cov=dsdk --cov-report=term-missing --strict-markers --ignore=.eggs --tb=short" testpaths = ["test"] norecursedirs = ".env .git build dist" python_files = "test.py tests.py test_*.py *_test.py" diff --git a/setup.py b/setup.py index ed4b5c8..9189fec 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ "pytest-cov", "types-pkg-resources", "types-python-dateutil", + "types-pymssql", "types-pyyaml", ) diff --git a/src/dsdk/asset.py b/src/dsdk/asset.py index b2892f9..2995f9d 100644 --- a/src/dsdk/asset.py +++ b/src/dsdk/asset.py @@ -11,7 +11,12 @@ from os.path import splitext from typing import Any, Dict -from yaml import SafeDumper, SafeLoader, add_constructor, add_representer +try: + from yaml import CSafeDumper as Dumper # type: ignore[misc] + from yaml import CSafeLoader as Loader # type: ignore[misc] +except ImportError: + from yaml import SafeDumper as Dumper # type: ignore[misc] + from yaml import SafeLoader as Loader # type: ignore[misc] logger = getLogger(__name__) @@ -24,8 +29,8 @@ class Asset(Namespace): @classmethod def as_yaml_type(cls): """As yaml type.""" - add_constructor(cls.YAML, cls._yaml_init, Loader=SafeLoader) - add_representer(cls, cls._yaml_repr, Dumper=SafeDumper) + Loader.add_constructor(cls.YAML, cls._yaml_init) + Dumper.add_representer(cls, cls._yaml_repr) @classmethod def build(cls, *, path: str, ext: str): @@ -56,7 +61,11 @@ def _yaml_repr(cls, dumper, self): return dumper.represent_mapping(cls.YAML, self.as_yaml()) def __init__( - self, *, path: str, ext: str, **kwargs: Asset, + self, + *, + path: str, + ext: str, + **kwargs: Asset, ): """__init__.""" self.path = path diff --git a/src/dsdk/env.py b/src/dsdk/env.py index e59ed9e..2b4c305 100644 --- a/src/dsdk/env.py +++ b/src/dsdk/env.py @@ -7,7 +7,10 @@ from re import compile as re_compile from typing import Mapping, Optional -from yaml import SafeLoader, add_constructor, add_implicit_resolver +try: + from yaml import CSafeLoader as Loader # type: ignore[misc] +except ImportError: + from yaml import SafeLoader as Loader # type: ignore[misc] class Env: @@ -25,8 +28,8 @@ def _yaml_init(loader, node) -> str: """This closure passed env.""" return cls._yaml_init(loader, node, _env) - add_implicit_resolver(cls.YAML, cls.PATTERN, None, Loader=SafeLoader) - add_constructor(cls.YAML, _yaml_init, Loader=SafeLoader) + Loader.add_implicit_resolver(cls.YAML, cls.PATTERN, None) + Loader.add_constructor(cls.YAML, _yaml_init) @classmethod def _yaml_init(cls, loader, node, env: Mapping[str, str]): diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index 815d981..1553925 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -3,7 +3,7 @@ from logging import getLogger from select import select -from urllib.request import urlopen +from urllib.request import Request, urlopen logger = getLogger(__name__) @@ -29,12 +29,12 @@ def listen(self, listen): self.on_notify(listen.notifies.pop()) -class Notification: +class Notification(Abstract): """Notification Service.""" URI = "?".join( ( - "api/epic/2011/Clinical/Patient/AddFlowsheetValue/FlowsheetValue", + "api/epic/2011/Clinical/Patient/AddFlowsheetValue/FlowsheetValue?", "&".join( ( "PatientID=%(empi)s", @@ -89,15 +89,25 @@ def call_uri(self, prediction, cur): "empi": prediction["empi"], "score": prediction["score"], } - response = urlopen(uri, self.timeout) - if response.status_code in [200, 201]: - cur.execute( - sql.epic.notification.insert, - {"prediction_id": prediction["id"]}, - ) + request = Request(uri, data=None) + with urlopen(request, self.timeout) as response: + if response.ok: + cur.execute( + sql.epic.notification.insert, + {"prediction_id": prediction["id"]}, + ) + else: + cur.execute( + sql.epic.notification_error.insert, + { + "description": response.text, + "name": response.reason, + "prediction_id": prediction["id"], + }, + ) -class Verification: +class Verification(Abstract): """Verification Service.""" URI = "api/epic/2014/Clinical/Patient/GetFlowsheetRows/FlowsheetRows" @@ -133,9 +143,23 @@ def on_notify(self, notify): """On notify.""" logger.debug(f"NOTIFY: {notify.pid}.{notify.channel}.{notify.payload}") - def call_uri(self, prediction, cur): + def call_uri(self, notification, cur): """Call uri.""" sql = self.postgres.sql - response = urlopen(self.uri, self.timeout) - if response.status_code in [200, 201]: - cur.execute(sql.epic.notification.insert, prediction.id) + # TODO add notification flowsheet ids to data? + request = Request(self.URI, data=None) + with urlopen(request, self.timeout) as response: + if response.ok: + cur.execute( + sql.epic.verification.insert, + {"notification_id": notification.id}, + ) + else: + cur.execute( + sql.epic.verification_error.insert, + { + "description": response.text, + "name": response.reason, + "notification_id": notification.id, + }, + ) diff --git a/src/dsdk/interval.py b/src/dsdk/interval.py index 9aac3ba..9c3f068 100644 --- a/src/dsdk/interval.py +++ b/src/dsdk/interval.py @@ -4,7 +4,12 @@ from datetime import datetime from typing import Any, Dict, Optional -from yaml import SafeDumper, SafeLoader, add_constructor, add_representer +try: + from yaml import CSafeDumper as Dumper # type: ignore[misc] + from yaml import CSafeLoader as Loader # type: ignore[misc] +except ImportError: + from yaml import SafeDumper as Dumper # type: ignore[misc] + from yaml import SafeLoader as Loader # type: ignore[misc] class Interval: @@ -15,8 +20,8 @@ class Interval: @classmethod def as_yaml_type(cls): """As yaml type.""" - add_constructor(cls.YAML, cls._yaml_init, Loader=SafeLoader) - add_representer(cls, cls._yaml_repr, Dumper=SafeDumper) + Loader.add_constructor(cls.YAML, cls._yaml_init) + Dumper.add_representer(cls, cls._yaml_repr) @classmethod def _yaml_init(cls, loader, node): diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 3c09a39..25bf9bb 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -8,11 +8,17 @@ from logging import getLogger from typing import TYPE_CHECKING, Any, Dict, Generator, Optional -from yaml import SafeDumper, SafeLoader, add_constructor, add_representer - from .service import Delegate, Service from .utils import load_pickle_file +try: + from yaml import CSafeDumper as Dumper # type: ignore[misc] + from yaml import CSafeLoader as Loader # type: ignore[misc] +except ImportError: + from yaml import SafeDumper as Dumper # type: ignore[misc] + from yaml import SafeLoader as Loader # type: ignore[misc] + + logger = getLogger(__name__) @@ -30,8 +36,8 @@ class Model: # pylint: disable=too-few-public-methods @classmethod def as_yaml_type(cls) -> None: """As yaml type.""" - add_constructor(cls.YAML, cls._yaml_init, Loader=SafeLoader) - add_representer(cls, cls._yaml_repr, Dumper=SafeDumper) + Loader.add_constructor(cls.YAML, cls._yaml_init) + Dumper.add_representer(cls, cls._yaml_repr) @classmethod def _yaml_init(cls, loader, node): @@ -39,7 +45,7 @@ def _yaml_init(cls, loader, node): path = loader.construct_scalar(node) pkl = load_pickle_file(path) if pkl.__class__ is dict: - pkl = cls(path=path, **pkl) # type: ignore + pkl = cls(path=path, **pkl) else: pkl.path = path assert isinstance(pkl, Model) @@ -51,7 +57,11 @@ def _yaml_repr(cls, dumper, self): return dumper.represent_scalar(cls.YAML, self.as_yaml()) def __init__( - self, *, name: str, version: str, path: Optional[str] = None, + self, + *, + name: str, + version: str, + path: Optional[str] = None, ) -> None: """__init__.""" self.name = name diff --git a/src/dsdk/mssql.py b/src/dsdk/mssql.py index 699faed..4ba7907 100644 --- a/src/dsdk/mssql.py +++ b/src/dsdk/mssql.py @@ -17,7 +17,7 @@ try: - from pymssql import connect, _mssql, DatabaseError, InterfaceError + from pymssql import DatabaseError, InterfaceError, _mssql, connect except ImportError as import_error: logger.warning(import_error) @@ -57,9 +57,14 @@ class Persistor(Messages, BasePersistor): YAML = "!mssql" @classmethod - def mogrify(cls, cur, query: str, parameters: Any,) -> bytes: + def mogrify( + cls, + cur, + query: str, + parameters: Any, + ) -> bytes: """Safely mogrify parameters into query or fragment.""" - return _mssql.substitute_params(query, parameters) # type: ignore + return _mssql.substitute_params(query, parameters) def check(self, cur, exceptions=(DatabaseError, InterfaceError)): """check.""" diff --git a/src/dsdk/persistor.py b/src/dsdk/persistor.py index 0d92b9f..2e7db4c 100644 --- a/src/dsdk/persistor.py +++ b/src/dsdk/persistor.py @@ -11,11 +11,18 @@ from typing import Any, Dict, Generator, Optional, Sequence, Tuple from pandas import DataFrame, concat -from yaml import SafeDumper, SafeLoader, add_constructor, add_representer from .asset import Asset from .utils import chunks +try: + from yaml import CSafeDumper as Dumper # type: ignore[misc] + from yaml import CSafeLoader as Loader # type: ignore[misc] +except ImportError: + from yaml import SafeDumper as Dumper # type: ignore[misc] + from yaml import SafeLoader as Loader # type: ignore[misc] + + logger = getLogger(__name__) @@ -39,7 +46,10 @@ class AbstractPersistor: @classmethod def df_from_query( - cls, cur, query: str, parameters: Optional[Dict[str, Any]], + cls, + cur, + query: str, + parameters: Optional[Dict[str, Any]], ) -> DataFrame: """Return DataFrame from query.""" if parameters is None: @@ -128,12 +138,21 @@ def df_from_query_by_keys( return df @classmethod - def mogrify(cls, cur, query: str, parameters: Any,) -> bytes: + def mogrify( + cls, + cur, + query: str, + parameters: Any, + ) -> bytes: """Safely mogrify parameters into query or fragment.""" raise NotImplementedError() @classmethod - def union_all(cls, cur, keys: Sequence[Any],) -> str: + def union_all( + cls, + cur, + keys: Sequence[Any], + ) -> str: """Return 'union all select %s...' clause.""" parameters = tuple(keys) union = "\n ".join("union all select %s" for _ in parameters) @@ -221,8 +240,8 @@ class Persistor(AbstractPersistor): @classmethod def as_yaml_type(cls) -> None: """As yaml type.""" - add_constructor(cls.YAML, cls._yaml_init, Loader=SafeLoader) - add_representer(cls, cls._yaml_repr, Dumper=SafeDumper) + Loader.add_constructor(cls.YAML, cls._yaml_init) + Dumper.add_representer(cls, cls._yaml_repr) @classmethod def _yaml_init(cls, loader, node): diff --git a/src/dsdk/postgres.py b/src/dsdk/postgres.py index 8ca281a..2e56f39 100644 --- a/src/dsdk/postgres.py +++ b/src/dsdk/postgres.py @@ -29,15 +29,15 @@ OperationalError, connect, ) - from psycopg2.extras import execute_batch from psycopg2.extensions import ( - register_adapter, ISOLATION_LEVEL_AUTOCOMMIT, - ISQLQuote, - Float, AsIs, + Float, Int, + ISQLQuote, + register_adapter, ) + from psycopg2.extras import execute_batch def na_adapter(as_type): """Na adapter.""" @@ -122,7 +122,12 @@ class Persistor(Messages, BasePersistor): YAML = "!postgres" @classmethod - def mogrify(cls, cur, query: str, parameters: Any,) -> bytes: + def mogrify( + cls, + cur, + query: str, + parameters: Any, + ) -> bytes: """Safely mogrify parameters into query or fragment.""" return cur.mogrify(query, parameters) @@ -182,7 +187,12 @@ def open_run(self, parent: Any) -> Generator[Run, None, None]: time_zone, *_, ) = row - run = Run(id_, microservice_id, model_id, parent,) + run = Run( + id_, + microservice_id, + model_id, + parent, + ) parent.as_of = as_of parent.duration = Interval( on=duration.lower, end=duration.upper @@ -226,7 +236,9 @@ def scores(self, run_id) -> Series: with self.rollback() as cur: cur.execute(sql.schema) return self.df_from_query( - cur, sql.predictions.gold, {"run_id": run_id}, + cur, + sql.predictions.gold, + {"run_id": run_id}, ).score.values # pylint: disable=no-member def store_evidence(self, run: Any, *args, **kwargs) -> None: @@ -237,7 +249,7 @@ def store_evidence(self, run: Any, *args, **kwargs) -> None: evidence = run.evidence exclude = set(kwargs.get("exclude", ())) while args: - key, df, *args = args # type: ignore + key, df, *args = args # type: ignore[assignment] evidence[key] = df # setattr(batch.evidence, name, data) if df.empty: @@ -249,11 +261,18 @@ def store_evidence(self, run: Any, *args, **kwargs) -> None: f"Missing sql/postgres/{key}/insert.sql" ) from e self._store_df( - schema, insert, run_id, df[list(set(df.columns) - exclude)], + schema, + insert, + run_id, + df[list(set(df.columns) - exclude)], ) def _store_df( - self, schema: str, insert: str, run_id: int, df: DataFrame, + self, + schema: str, + insert: str, + run_id: int, + df: DataFrame, ): df["run_id"] = run_id out = df.to_dict("records") @@ -261,7 +280,9 @@ def _store_df( with self.commit() as cur: cur.execute(schema) execute_batch( - cur, insert, out, + cur, + insert, + out, ) except DatabaseError as e: enumeration = enumerate(out) @@ -312,7 +333,11 @@ class Run(Delegate): """Run.""" def __init__( # pylint: disable=too-many-arguments - self, id_: int, microservice_id: str, model_id: str, parent: Any, + self, + id_: int, + microservice_id: str, + model_id: str, + parent: Any, ): """__init__.""" super().__init__(parent) diff --git a/src/dsdk/service.py b/src/dsdk/service.py index 142a289..e53031d 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -25,7 +25,6 @@ from pandas import DataFrame from pkg_resources import DistributionNotFound, get_distribution -from yaml import SafeDumper, SafeLoader, add_constructor, add_representer from yaml import safe_load as yaml_loads from .asset import Asset @@ -33,6 +32,13 @@ from .interval import Interval from .utils import configure_logger, get_tzinfo, now_utc_datetime +try: + from yaml import CSafeDumper as Dumper # type: ignore[misc] + from yaml import CSafeLoader as Loader # type: ignore[misc] +except ImportError: + from yaml import SafeDumper as Dumper # type: ignore[misc] + from yaml import SafeLoader as Loader # type: ignore[misc] + try: __version__ = get_distribution("dsdk").version except DistributionNotFound: @@ -208,8 +214,8 @@ class Service: # pylint: disable=too-many-instance-attributes @classmethod def as_yaml_type(cls) -> None: """As yaml type.""" - add_constructor(cls.YAML, cls._yaml_init, Loader=SafeLoader) - add_representer(cls, cls._yaml_repr, Dumper=SafeDumper) + Loader.add_constructor(cls.YAML, cls._yaml_init) + Dumper.add_representer(cls, cls._yaml_repr) @classmethod @contextmanager @@ -280,7 +286,9 @@ def parse( ) ) return cls.load( - config_file=args.config_file, env=env, env_file=args.env_file, + config_file=args.config_file, + env=env, + env_file=args.env_file, ) @classmethod @@ -346,12 +354,13 @@ def _yaml_repr(cls, dumper, self): """Yaml repr.""" return dumper.represent_mapping(cls.YAML, self.as_yaml()) - def __init__( # pylint: disable=too-many-arguments + def __init__( self, - pipeline: Sequence[Task] = None, + *, + pipeline: Sequence[Task], as_of: Optional[datetime] = None, gold: Optional[str] = None, - time_zone: Optional[datetime] = None, + time_zone: Optional[str] = None, batch_cls: Callable = Batch, ) -> None: """__init__.""" @@ -372,7 +381,10 @@ def __call__(self) -> Batch: if batch.time_zone is None: batch.time_zone = "America/New_York" if batch.duration is None: - batch.duration = Interval(on=batch.as_of, end=None,) + batch.duration = Interval( + on=batch.as_of, + end=None, + ) logger.info(self.PIPELINE_ON, self.__class__.__name__) for task in self.pipeline: @@ -426,13 +438,14 @@ def on_create_gold(self) -> Batch: scores = self.scores(run.id) n_scores = scores.shape[0] logger.info("Write %s scores to %s", n_scores, path) + # pylint: disable=no-member + model_version = self.model.version # type: ignore[attr-defined] with open(path, "wb") as fout: pickle.dump( { "as_of": run.as_of, "microservice_version": self.VERSION, - # pylint: disable=no-member - "model_version": self.model.version, # type: ignore + "model_version": model_version, "scores": scores, "time_zone": run.time_zone, }, @@ -486,7 +499,9 @@ def on_validate_gold(self) -> Batch: def open_batch(self) -> Generator[Any, None, None]: """Open batch.""" logger.info( - self.BATCH_OPEN, self.as_of, self.time_zone, + self.BATCH_OPEN, + self.as_of, + self.time_zone, ) yield self.batch_cls( as_of=self.as_of, diff --git a/test/test_dsdk.py b/test/test_dsdk.py index 1b52ca1..8d40c9a 100644 --- a/test/test_dsdk.py +++ b/test/test_dsdk.py @@ -5,6 +5,7 @@ from pandas import DataFrame from pytest import mark +from yaml import safe_dump as yaml_dumps from dsdk import ( Asset, @@ -52,10 +53,10 @@ def __call__(self, batch: Batch, service: Service) -> None: assert batch.evidence["test"] is df -class TestService(ModelMixin, MssqlMixin, PostgresMixin, Service): - """TestService.""" +class MyService(ModelMixin, MssqlMixin, PostgresMixin, Service): + """MyService.""" - YAML = "!test" + YAML = "!myservice" @classmethod def yaml_types(cls): @@ -69,7 +70,7 @@ def __init__(self, **kwargs): super().__init__(pipeline=pipeline, **kwargs) -def build_from_parameters() -> Tuple[Callable, Dict[str, Any]]: +def build_from_parameters(cls) -> Tuple[Callable, Dict[str, Any]]: """Build from parameters.""" model = Model(name="test", path="./test/model.pkl", version="0.0.1-rc.1") mssql = Mssql( @@ -91,12 +92,12 @@ def build_from_parameters() -> Tuple[Callable, Dict[str, Any]]: tables=("foo", "bar", "baz"), ) return ( - TestService, + cls, {"model": model, "mssql": mssql, "postgres": postgres}, ) -def build_from_yaml() -> Tuple[Callable, Dict[str, Any]]: +def build_from_yaml(cls) -> Tuple[Callable, Dict[str, Any]]: """Build from yaml.""" pickle_file = "./test/model.pkl" dump_pickle_file( @@ -104,7 +105,7 @@ def build_from_yaml() -> Tuple[Callable, Dict[str, Any]]: ) configs = """ -!test +!myservice mssql: !mssql database: test host: 0.0.0.0 @@ -135,21 +136,26 @@ def build_from_yaml() -> Tuple[Callable, Dict[str, Any]]: POSTGRES_PASSWORD=password """ return ( - TestService.loads, + cls.loads, {"configs": configs, "env": env, "envs": envs}, ) -@mark.parametrize("cls,kwargs", (build_from_yaml(), build_from_parameters())) +@mark.parametrize( + "cls,kwargs", + (build_from_yaml(MyService), build_from_parameters(MyService)), +) def test_service(cls, kwargs: Dict[str, Any]): """Test parameters, config, and env.""" service = cls(**kwargs) - assert service.__class__ is TestService + assert service.__class__ is cls assert service.model.__class__ is Model assert service.postgres.__class__ is Postgres assert service.mssql.__class__ is Mssql assert service.postgres.password == "password" assert service.mssql.password == "password" + # yaml = yaml_dumps(service) + # print(yaml) def test_retry_other_exception(): diff --git a/test/test_postgres.py b/test/test_postgres.py index 4c5dcd3..4a20d24 100644 --- a/test/test_postgres.py +++ b/test/test_postgres.py @@ -17,7 +17,9 @@ class Persistor(Postgres): """Persistor.""" def __init__( - self, env=None, **kwargs, + self, + env=None, + **kwargs, ): """__init__.""" if env is None: From d97b0a4d528624970428abe0ac2381dd0fb8c805 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 19 Jul 2021 14:50:52 -0400 Subject: [PATCH 05/33] Fix pytest --- postgres/sql/patchdb.d/007.epic.sql | 8 +++---- src/dsdk/epic.py | 35 ++++++++++++++++------------- test/test_dsdk.py | 1 - 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/postgres/sql/patchdb.d/007.epic.sql b/postgres/sql/patchdb.d/007.epic.sql index e70717e..5fe215a 100644 --- a/postgres/sql/patchdb.d/007.epic.sql +++ b/postgres/sql/patchdb.d/007.epic.sql @@ -11,7 +11,7 @@ begin create table epic_notifications ( id int primary key generated always as identity, prediction_id int not null, - recorded_on timestamptz default statement_timestamp(), + assert_on timestamptz default statement_timestamp(), constraint only_one_epic_notification_per_prediction unique (prediction_id), constraint epic_notifications_require_a_prediction @@ -27,7 +27,7 @@ begin create table epic_notification_errors ( id int primary key generated always as identity, prediction_id int not null, - recorded_on timestamptz default statement_timestamp(), + assert_on timestamptz default statement_timestamp(), acknowledged_on timestamptz default null, name varchar, description varchar, @@ -40,7 +40,7 @@ begin create table epic_verifications ( id int primary key generated always as identity, notification_id int not null, - recorded_on timestamptz default statement_timestamp(), + assert_on timestamptz default statement_timestamp(), constraint only_one_epic_verification_per_notification unique (notification_id), constraint epic_verifications_require_a_notification @@ -56,7 +56,7 @@ begin create table epic_verification_errors ( id int primary key generated always as identity, notification_id int not null, - recorded_on timestamptz default statement_timestamp(), + assert_on timestamptz default statement_timestamp(), acknowledged_on timestamptz default null, name varchar, description varchar, diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index 1553925..b3ab1d4 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -22,12 +22,28 @@ def listen(self, listen): """Listen.""" while True: readers, _, exceptions = select( - [listen], [], [], self.poll_timeout + [listen], [], [listen], self.poll_timeout ) + if exceptions: + break + if not readers: + continue if listen.poll(): while listen.notifies: self.on_notify(listen.notifies.pop()) + def on_notify(self, event): # pylint: disable=no-self-use + """On postgres notify handler.""" + logger.debug( + "NOTIFY: %(pid)s.%(channel)s.%(payload)s", + { + "channel": event.chennel, + "payload": event.payload, + "pid": event.pid, + }, + ) + raise NotImplementedError() + class Notification(Abstract): """Notification Service.""" @@ -77,10 +93,6 @@ def recover(self, cur): for each in cur.fetchall(): self.call_uri(each, cur) - def on_notify(self, notify): - """On notify.""" - logger.debug(f"NOTIFY: {notify.pid}.{notify.channel}.{notify.payload}") - def call_uri(self, prediction, cur): """Call uri.""" sql = self.postgres.sql @@ -90,7 +102,7 @@ def call_uri(self, prediction, cur): "score": prediction["score"], } request = Request(uri, data=None) - with urlopen(request, self.timeout) as response: + with urlopen(request, self.uri_timeout) as response: if response.ok: cur.execute( sql.epic.notification.insert, @@ -112,11 +124,6 @@ class Verification(Abstract): URI = "api/epic/2014/Clinical/Patient/GetFlowsheetRows/FlowsheetRows" - @classmethod - def main(cls): - """__main__.""" - pass - def __init__( self, postgres, uri=URI, poll_timeout=60, uri_timeout=5 ) -> None: @@ -139,16 +146,12 @@ def recover(self, cur): for each in cur.fetchall(): self.call_uri(each, cur) - def on_notify(self, notify): - """On notify.""" - logger.debug(f"NOTIFY: {notify.pid}.{notify.channel}.{notify.payload}") - def call_uri(self, notification, cur): """Call uri.""" sql = self.postgres.sql # TODO add notification flowsheet ids to data? request = Request(self.URI, data=None) - with urlopen(request, self.timeout) as response: + with urlopen(request, self.uri_timeout) as response: if response.ok: cur.execute( sql.epic.verification.insert, diff --git a/test/test_dsdk.py b/test/test_dsdk.py index 8d40c9a..592e2de 100644 --- a/test/test_dsdk.py +++ b/test/test_dsdk.py @@ -5,7 +5,6 @@ from pandas import DataFrame from pytest import mark -from yaml import safe_dump as yaml_dumps from dsdk import ( Asset, From 67a9bf6e4ff8c2864cac9f669b661728c31a45c9 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 19 Jul 2021 18:08:01 -0400 Subject: [PATCH 06/33] Move yaml import checks into utils --- buster.dockerfile | 2 +- src/dsdk/__init__.py | 8 ++++++++ src/dsdk/asset.py | 11 +++-------- src/dsdk/env.py | 17 +++++++---------- src/dsdk/interval.py | 11 +++-------- src/dsdk/model.py | 14 +++----------- src/dsdk/persistor.py | 16 ++++------------ src/dsdk/service.py | 24 +++++++++++------------- src/dsdk/utils.py | 28 +++++++++++++++++++++++----- test/test_dsdk.py | 30 ++++++++++++++++++++++++------ 10 files changed, 87 insertions(+), 74 deletions(-) diff --git a/buster.dockerfile b/buster.dockerfile index e77ec5b..f42a8b7 100644 --- a/buster.dockerfile +++ b/buster.dockerfile @@ -19,7 +19,7 @@ COPY test ./test RUN \ chmod +x /usr/bin/tini && \ apt-get -qq update --fix-missing && \ - apt-get -qq install -y --no-install-recommends git > /dev/null && \ + apt-get -qq install -y --no-install-recommends git libyaml-dev > /dev/null && \ pip install ${IFLAGS} "." && \ apt-get -qq clean && \ apt-get -qq autoremove -y --purge && \ diff --git a/src/dsdk/__init__.py b/src/dsdk/__init__.py index 5fc60e4..8a36e0b 100644 --- a/src/dsdk/__init__.py +++ b/src/dsdk/__init__.py @@ -15,6 +15,8 @@ from .postgres import PredictionMixin as PostgresPredictionMixin from .service import Batch, Delegate, Service, Task from .utils import ( + YamlDumper, + YamlLoader, chunks, configure_logger, dump_json_file, @@ -26,6 +28,8 @@ now_utc_datetime, profile, retry, + yaml_dumps, + yaml_loads, ) __all__ = ( @@ -45,6 +49,8 @@ "Postgres", "Service", "Task", + "YamlLoader", + "YamlDumper", "chunks", "configure_logger", "dump_json_file", @@ -56,4 +62,6 @@ "profile", "now_utc_datetime", "retry", + "yaml_dumps", + "yaml_loads", ) diff --git a/src/dsdk/asset.py b/src/dsdk/asset.py index 2995f9d..81fcba0 100644 --- a/src/dsdk/asset.py +++ b/src/dsdk/asset.py @@ -11,12 +11,7 @@ from os.path import splitext from typing import Any, Dict -try: - from yaml import CSafeDumper as Dumper # type: ignore[misc] - from yaml import CSafeLoader as Loader # type: ignore[misc] -except ImportError: - from yaml import SafeDumper as Dumper # type: ignore[misc] - from yaml import SafeLoader as Loader # type: ignore[misc] +from .utils import YamlDumper, YamlLoader logger = getLogger(__name__) @@ -29,8 +24,8 @@ class Asset(Namespace): @classmethod def as_yaml_type(cls): """As yaml type.""" - Loader.add_constructor(cls.YAML, cls._yaml_init) - Dumper.add_representer(cls, cls._yaml_repr) + YamlLoader.add_constructor(cls.YAML, cls._yaml_init) + YamlDumper.add_representer(cls, cls._yaml_repr) @classmethod def build(cls, *, path: str, ext: str): diff --git a/src/dsdk/env.py b/src/dsdk/env.py index 2b4c305..ec7cb95 100644 --- a/src/dsdk/env.py +++ b/src/dsdk/env.py @@ -7,17 +7,14 @@ from re import compile as re_compile from typing import Mapping, Optional -try: - from yaml import CSafeLoader as Loader # type: ignore[misc] -except ImportError: - from yaml import SafeLoader as Loader # type: ignore[misc] +from .utils import YamlLoader class Env: """Env.""" YAML = "!env" - PATTERN = re_compile(r".*?\$\{(\w+)\}.*?") + PATTERN = re_compile(r".?\$\{([^\}^\{]+)\}.?") @classmethod def as_yaml_type(cls, *, env: Optional[Mapping[str, str]] = None): @@ -28,8 +25,8 @@ def _yaml_init(loader, node) -> str: """This closure passed env.""" return cls._yaml_init(loader, node, _env) - Loader.add_implicit_resolver(cls.YAML, cls.PATTERN, None) - Loader.add_constructor(cls.YAML, _yaml_init) + YamlLoader.add_implicit_resolver(cls.YAML, cls.PATTERN, None) + YamlLoader.add_constructor(cls.YAML, _yaml_init) @classmethod def _yaml_init(cls, loader, node, env: Mapping[str, str]): @@ -49,13 +46,13 @@ def _yaml_init(cls, loader, node, env: Mapping[str, str]): def load(cls, path: str) -> Mapping[str, str]: """Env load.""" with open(path) as fin: - return cls.loads(fin.read()) + return cls.loads(fin) @classmethod - def loads(cls, envs: str) -> Mapping[str, str]: + def loads(cls, stream) -> Mapping[str, str]: """Env loads.""" result = {} - for line in envs.split("\n"): + for line in stream: line = line.strip() if not line: continue diff --git a/src/dsdk/interval.py b/src/dsdk/interval.py index 9c3f068..77cccfa 100644 --- a/src/dsdk/interval.py +++ b/src/dsdk/interval.py @@ -4,12 +4,7 @@ from datetime import datetime from typing import Any, Dict, Optional -try: - from yaml import CSafeDumper as Dumper # type: ignore[misc] - from yaml import CSafeLoader as Loader # type: ignore[misc] -except ImportError: - from yaml import SafeDumper as Dumper # type: ignore[misc] - from yaml import SafeLoader as Loader # type: ignore[misc] +from .utils import YamlDumper, YamlLoader class Interval: @@ -20,8 +15,8 @@ class Interval: @classmethod def as_yaml_type(cls): """As yaml type.""" - Loader.add_constructor(cls.YAML, cls._yaml_init) - Dumper.add_representer(cls, cls._yaml_repr) + YamlLoader.add_constructor(cls.YAML, cls._yaml_init) + YamlDumper.add_representer(cls, cls._yaml_repr) @classmethod def _yaml_init(cls, loader, node): diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 25bf9bb..3006818 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -9,15 +9,7 @@ from typing import TYPE_CHECKING, Any, Dict, Generator, Optional from .service import Delegate, Service -from .utils import load_pickle_file - -try: - from yaml import CSafeDumper as Dumper # type: ignore[misc] - from yaml import CSafeLoader as Loader # type: ignore[misc] -except ImportError: - from yaml import SafeDumper as Dumper # type: ignore[misc] - from yaml import SafeLoader as Loader # type: ignore[misc] - +from .utils import YamlDumper, YamlLoader, load_pickle_file logger = getLogger(__name__) @@ -36,8 +28,8 @@ class Model: # pylint: disable=too-few-public-methods @classmethod def as_yaml_type(cls) -> None: """As yaml type.""" - Loader.add_constructor(cls.YAML, cls._yaml_init) - Dumper.add_representer(cls, cls._yaml_repr) + YamlLoader.add_constructor(cls.YAML, cls._yaml_init) + YamlDumper.add_representer(cls, cls._yaml_repr) @classmethod def _yaml_init(cls, loader, node): diff --git a/src/dsdk/persistor.py b/src/dsdk/persistor.py index 2e7db4c..ca3d222 100644 --- a/src/dsdk/persistor.py +++ b/src/dsdk/persistor.py @@ -13,15 +13,7 @@ from pandas import DataFrame, concat from .asset import Asset -from .utils import chunks - -try: - from yaml import CSafeDumper as Dumper # type: ignore[misc] - from yaml import CSafeLoader as Loader # type: ignore[misc] -except ImportError: - from yaml import SafeDumper as Dumper # type: ignore[misc] - from yaml import SafeLoader as Loader # type: ignore[misc] - +from .utils import YamlDumper, YamlLoader, chunks logger = getLogger(__name__) @@ -235,13 +227,13 @@ def rollback(self) -> Generator[Any, None, None]: class Persistor(AbstractPersistor): """Persistor.""" - YAML = "" + YAML = "!basepersistor" @classmethod def as_yaml_type(cls) -> None: """As yaml type.""" - Loader.add_constructor(cls.YAML, cls._yaml_init) - Dumper.add_representer(cls, cls._yaml_repr) + YamlLoader.add_constructor(cls.YAML, cls._yaml_init) + YamlDumper.add_representer(cls, cls._yaml_repr) @classmethod def _yaml_init(cls, loader, node): diff --git a/src/dsdk/service.py b/src/dsdk/service.py index e53031d..b46b5e2 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -25,19 +25,18 @@ from pandas import DataFrame from pkg_resources import DistributionNotFound, get_distribution -from yaml import safe_load as yaml_loads from .asset import Asset from .env import Env from .interval import Interval -from .utils import configure_logger, get_tzinfo, now_utc_datetime - -try: - from yaml import CSafeDumper as Dumper # type: ignore[misc] - from yaml import CSafeLoader as Loader # type: ignore[misc] -except ImportError: - from yaml import SafeDumper as Dumper # type: ignore[misc] - from yaml import SafeLoader as Loader # type: ignore[misc] +from .utils import ( + YamlDumper, + YamlLoader, + configure_logger, + get_tzinfo, + now_utc_datetime, + yaml_loads, +) try: __version__ = get_distribution("dsdk").version @@ -207,15 +206,15 @@ class Service: # pylint: disable=too-many-instance-attributes {"key": "validate.count", "scores": "%s", "test": "%s", "status": "%s"} ) MATCH = dumps({"key": "validate.match", "status": "%s"}) - YAML = "" + YAML = "!baseservice" VERSION = __version__ @classmethod def as_yaml_type(cls) -> None: """As yaml type.""" - Loader.add_constructor(cls.YAML, cls._yaml_init) - Dumper.add_representer(cls, cls._yaml_repr) + YamlLoader.add_constructor(cls.YAML, cls._yaml_init) + YamlDumper.add_representer(cls, cls._yaml_repr) @classmethod @contextmanager @@ -328,7 +327,6 @@ def loads( env = Env.loads(envs) Env.as_yaml_type(env=env) - cls.yaml_types() return yaml_loads(configs) diff --git a/src/dsdk/utils.py b/src/dsdk/utils.py index 9e499fe..fcdd97b 100644 --- a/src/dsdk/utils.py +++ b/src/dsdk/utils.py @@ -16,9 +16,17 @@ from time import sleep as default_sleep from typing import Any, Callable, Generator, Sequence +from yaml import dump as _yaml_dumps +from yaml import load as _yaml_loads + +try: + from yaml import CSafeDumper as YamlDumper # type: ignore[misc] + from yaml import CSafeLoader as YamlLoader # type: ignore[misc] +except ImportError: + from yaml import SafeDumper as YamlDumper # type: ignore[misc] + from yaml import SafeLoader as YamlLoader # type: ignore[misc] + from dateutil import parser, tz -from yaml import safe_dump as yaml_dumps -from yaml import safe_load as yaml_loads logger = getLogger(__name__) @@ -75,10 +83,10 @@ def dump_pickle_file(obj: Any, path: str) -> None: pickle_dump(obj, fout) -def dump_yaml_file(obj: Any, path: str) -> None: +def dump_yaml_file(data: Any, path: str, **kwargs) -> None: """Dump yaml file.""" with open(path, "w") as fout: - yaml_dumps(obj, fout) + yaml_dumps(data=data, stream=fout, **kwargs) def epoch_ms_from_utc_datetime(utc: datetime) -> float: @@ -108,7 +116,7 @@ def load_pickle_file(path: str) -> object: def load_yaml_file(path: str): """Load yaml file.""" with open(path) as fin: - return yaml_loads(fin.read()) + return yaml_loads(fin) def now_utc_datetime() -> datetime: @@ -179,5 +187,15 @@ def utc_datetime_from_epoch_ms(epoch_ms: float) -> datetime: return datetime.fromtimestamp(epoch_ms / 1000, tz=timezone.utc) +def yaml_dumps(data, **kwargs): + """Yaml dumps.""" + return _yaml_dumps(data=data, Dumper=YamlDumper, **kwargs) + + +def yaml_loads(stream, **kwargs): + """Yaml loads.""" + return _yaml_loads(stream=stream, Loader=YamlLoader, **kwargs) + + class StubError(Exception): """StubError.""" diff --git a/test/test_dsdk.py b/test/test_dsdk.py index 592e2de..9090a2d 100644 --- a/test/test_dsdk.py +++ b/test/test_dsdk.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Test dsdk.""" +from io import StringIO from typing import Any, Callable, Dict, Tuple from pandas import DataFrame @@ -19,6 +20,7 @@ Task, dump_pickle_file, retry, + yaml_dumps, ) @@ -103,7 +105,8 @@ def build_from_yaml(cls) -> Tuple[Callable, Dict[str, Any]]: Model(name="test", path=pickle_file, version="0.0.1"), pickle_file ) - configs = """ + configs = StringIO( + """ !myservice mssql: !mssql database: test @@ -129,11 +132,14 @@ def build_from_yaml(cls) -> Tuple[Callable, Dict[str, Any]]: - baz username: postgres """ + ) env = {"POSTGRES_PASSWORD": "oops!", "MSSQL_PASSWORD": "oops!"} - envs = """ + envs = StringIO( + """ MSSQL_PASSWORD=password POSTGRES_PASSWORD=password """ + ) return ( cls.loads, {"configs": configs, "env": env, "envs": envs}, @@ -144,17 +150,24 @@ def build_from_yaml(cls) -> Tuple[Callable, Dict[str, Any]]: "cls,kwargs", (build_from_yaml(MyService), build_from_parameters(MyService)), ) -def test_service(cls, kwargs: Dict[str, Any]): +def test_service( + cls, # pylint: disable=redefined-outer-name + kwargs: Dict[str, Any], +): """Test parameters, config, and env.""" service = cls(**kwargs) - assert service.__class__ is cls + assert service.__class__ is MyService assert service.model.__class__ is Model assert service.postgres.__class__ is Postgres assert service.mssql.__class__ is Mssql assert service.postgres.password == "password" assert service.mssql.password == "password" - # yaml = yaml_dumps(service) - # print(yaml) + buf = StringIO() + buf.write(yaml_dumps(service)) + expected = """ + + """ + assert buf.getvalue() == expected def test_retry_other_exception(): @@ -219,3 +232,8 @@ def explode(): except NotImplementedError as exception: assert actual == expected assert str(exception) == "when?" + + +if __name__ == "__main__": + cls, kwargs = build_from_yaml(MyService) + test_service(cls, kwargs) From 02303b0729d2839d3fc36661254434b10739f152 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Wed, 21 Jul 2021 15:04:15 -0400 Subject: [PATCH 07/33] Fix yaml deserialization --- src/dsdk/model.py | 8 ++-- test/test_dsdk.py | 116 +++++++++++++++++++++++++++++++--------------- 2 files changed, 83 insertions(+), 41 deletions(-) diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 3006818..4078b4f 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -6,7 +6,7 @@ from abc import ABC from contextlib import contextmanager from logging import getLogger -from typing import TYPE_CHECKING, Any, Dict, Generator, Optional +from typing import TYPE_CHECKING, Any, Dict, Generator from .service import Delegate, Service from .utils import YamlDumper, YamlLoader, load_pickle_file @@ -52,17 +52,17 @@ def __init__( self, *, name: str, + path: str, version: str, - path: Optional[str] = None, ) -> None: """__init__.""" self.name = name self.path = path self.version = version - def as_yaml(self) -> Dict[str, Any]: + def as_yaml(self) -> str: """As yaml.""" - return {"path": self.path} + return self.path class Batch(Delegate): diff --git a/test/test_dsdk.py b/test/test_dsdk.py index 9090a2d..6109fde 100644 --- a/test/test_dsdk.py +++ b/test/test_dsdk.py @@ -75,22 +75,22 @@ def build_from_parameters(cls) -> Tuple[Callable, Dict[str, Any]]: """Build from parameters.""" model = Model(name="test", path="./test/model.pkl", version="0.0.1-rc.1") mssql = Mssql( - username="username", + username="mssql", password="password", - host="host", + host="0.0.0.0", port=1433, - database="database", + database="test", sql=Asset.build(path="./assets/mssql", ext=".sql"), - tables=("foo", "bar", "baz"), + tables=("a", "b", "c"), ) postgres = Postgres( - username="username", + username="postgres", password="password", - host="host", + host="0.0.0.0", port=5432, - database="database", + database="test", sql=Asset.build(path="./assets/postgres", ext=".sql"), - tables=("foo", "bar", "baz"), + tables=("ichi", "ni", "san", "shi", "go"), ) return ( cls, @@ -98,26 +98,20 @@ def build_from_parameters(cls) -> Tuple[Callable, Dict[str, Any]]: ) -def build_from_yaml(cls) -> Tuple[Callable, Dict[str, Any]]: - """Build from yaml.""" - pickle_file = "./test/model.pkl" - dump_pickle_file( - Model(name="test", path=pickle_file, version="0.0.1"), pickle_file - ) - - configs = StringIO( - """ +CONFIGS = """ !myservice mssql: !mssql database: test host: 0.0.0.0 password: ${MSSQL_PASSWORD} port: 1433 - sql: ./assets/mssql + sql: !asset + ext: .sql + path: ./assets/mssql tables: - - foo - - bar - - baz + - a + - b + - c username: mssql model: !model ./test/model.pkl postgres: !postgres @@ -125,21 +119,71 @@ def build_from_yaml(cls) -> Tuple[Callable, Dict[str, Any]]: host: 0.0.0.0 password: ${POSTGRES_PASSWORD} port: 5432 - sql: ./asset/postgres + sql: !asset + ext: .sql + path: ./assets/postgres tables: - - foo - - bar - - baz + - ichi + - ni + - san + - shi + - go username: postgres -""" - ) - env = {"POSTGRES_PASSWORD": "oops!", "MSSQL_PASSWORD": "oops!"} - envs = StringIO( - """ +""".strip() + +ENVS = """ MSSQL_PASSWORD=password POSTGRES_PASSWORD=password -""" +""".strip() + +EXPECTED = """ +!myservice +as_of: null +duration: null +gold: null +model: !model ./test/model.pkl +mssql: !mssql + database: test + host: 0.0.0.0 + password: password + port: 1433 + sql: !asset + ext: .sql + path: ./assets/mssql + tables: + - a + - b + - c + username: mssql +postgres: !postgres + database: test + host: 0.0.0.0 + password: password + port: 5432 + sql: !asset + ext: .sql + path: ./assets/postgres + tables: + - ichi + - ni + - san + - shi + - go + username: postgres +time_zone: null +""".strip() + + +def build_from_yaml(cls) -> Tuple[Callable, Dict[str, Any]]: + """Build from yaml.""" + pickle_file = "./test/model.pkl" + dump_pickle_file( + Model(name="test", path=pickle_file, version="0.0.1"), pickle_file ) + + configs = StringIO(CONFIGS) + env = {"POSTGRES_PASSWORD": "oops!", "MSSQL_PASSWORD": "oops!"} + envs = StringIO(ENVS) return ( cls.loads, {"configs": configs, "env": env, "envs": envs}, @@ -155,6 +199,7 @@ def test_service( kwargs: Dict[str, Any], ): """Test parameters, config, and env.""" + expected = EXPECTED service = cls(**kwargs) assert service.__class__ is MyService assert service.model.__class__ is Model @@ -164,10 +209,8 @@ def test_service( assert service.mssql.password == "password" buf = StringIO() buf.write(yaml_dumps(service)) - expected = """ - - """ - assert buf.getvalue() == expected + actual = buf.getvalue().strip() + assert actual == expected def test_retry_other_exception(): @@ -235,5 +278,4 @@ def explode(): if __name__ == "__main__": - cls, kwargs = build_from_yaml(MyService) - test_service(cls, kwargs) + test_service(*build_from_yaml(MyService)) From 6688657b2213b131d76fb957c7902b3ec6294c9e Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Fri, 23 Jul 2021 11:44:50 -0400 Subject: [PATCH 08/33] Update epic --- src/dsdk/epic.py | 191 +++++++++++++++++++++++++++++++++++++------- src/dsdk/service.py | 2 +- test/model.pkl | Bin 0 -> 101 bytes 3 files changed, 163 insertions(+), 30 deletions(-) create mode 100644 test/model.pkl diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index b3ab1d4..f4f8025 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -3,26 +3,121 @@ from logging import getLogger from select import select +from typing import Any, Dict from urllib.request import Request, urlopen +from .postgres import Persistor as Postgres +from .utils import YamlDumper, YamlLoader + logger = getLogger(__name__) -class Abstract: - """Abstract Service.""" +class Epic: + """Epic.""" + + YAML = "!epic" + + @classmethod + def as_yaml_type(cls): + """As yaml type.""" + YamlLoader.add_constructor(cls.YAML, cls._yaml_init) + YamlDumper.add_representer(cls, cls._yaml_repr) + + @classmethod + def _yaml_init(cls, loader, node): + """Yaml init.""" + return cls(**loader.construct_mapping(node, deep=True)) - def __init__(self, postgres, uri, poll_timeout=60, uri_timeout=5) -> None: + @classmethod + def _yaml_repr(cls, dumper, self): + """Yaml repr.""" + return dumper.represent_mapper(cls.YAML, self.as_yaml()) + + def __init__( + self, + *, + authorization: str, + cookie: str, + postgres: Postgres, + poll_timeout: int = 60, + uri_timeout: int = 5, + user_id: str = "pennsignals", + user_id_type: str = "external", + ) -> None: """__init__.""" + self.authorization = authorization + self.cookie = cookie self.postgres = postgres - self.uri = uri self.poll_timeout = poll_timeout self.uri_timeout = uri_timeout + self.user_id = user_id + self.user_id_type = user_id_type + + def as_yaml(self) -> Dict[str, Any]: + """As yaml.""" + return { + "authorization": self.authorization, + "cookie": self.cookie, + "poll_timeout": self.poll_timeout, + "postgres": self.postgres, + "uri_timeout": self.uri_timeout, + "user_id": self.user_id, + "user_id_type": self.user_id_type, + } + + +class FlowsheetEgress: # pylint: disable=too-many-instance-attributes + """Flowsheet Egress.""" + + YAML = "!flowsheetegress" + + @classmethod + def as_yaml_type(cls): + """As yaml type.""" + YamlLoader.add_constructor(cls.YAML, cls._yaml_init) + YamlDumper.add_representer(cls, cls._yaml_repr) + + @classmethod + def _yaml_init(cls, loader, node): + """Yaml init.""" + return cls(**loader.construct_mapping(node, deep=True)) + + @classmethod + def _yaml_repr(cls, dumper, self): + """Yaml repr.""" + return dumper.represent_mapper(cls.YAML, self.as_yaml()) + + def __init__( + self, + *, + epic: Epic, + uri: str, + flowsheet_id: str = "3040015333", + flowsheet_template_id: str = "3040005300", + flowsheet_template_id_type: str = "internal", + ) -> None: + """__init__.""" + self.epic = epic + self.flowsheet_id = flowsheet_id + self.flowsheet_template_id = flowsheet_template_id + self.flowsheet_template_id_type = flowsheet_template_id_type + self.uri = uri + + def as_yaml(self) -> Dict[str, Any]: + """As yaml.""" + return { + "epic": self.epic, + "flowsheet_id": self.flowsheet_id, + "flowsheet_template_id": self.flowsheet_template_id, + "flowsheet_template_id_type": self.flowsheet_template_id_type, + "uri": self.uri, + } def listen(self, listen): """Listen.""" while True: readers, _, exceptions = select( - [listen], [], [listen], self.poll_timeout + [listen], [], [listen], self.epic.poll_timeout ) if exceptions: break @@ -45,64 +140,91 @@ def on_notify(self, event): # pylint: disable=no-self-use raise NotImplementedError() -class Notification(Abstract): +class Notification(FlowsheetEgress): """Notification Service.""" + YAML = "!notification" + URI = "?".join( ( "api/epic/2011/Clinical/Patient/AddFlowsheetValue/FlowsheetValue?", "&".join( ( "PatientID=%(empi)s", - "PatientIDType={patient_id_type}", + "PatientIDType=%(patient_id_type)s", "ContactID=%(csn)s", - "ContactIDType={contact_id_type}", - "UserID=PENNSIGNALS", - "UserIDType=EXTERNAL", - "FlowsheetID={flowsheet_id}", - "FlowsheetIDType={flowsheet_id_type}", + "ContactIDType=%(contact_id_type)s", + "UserID=%(user_id)s", + "UserIDType=%(user_type_id)s", + "FlowsheetID=%(flowsheet_id)s", + "FlowsheetIDType=%(flowsheet_id_type)s", "Value=%(score)s", - "Comment={comment}", - "InstantValueTaken={instant_value_taken}", - "FlowsheetTemplateID={flowsheet_template_id}", - "FlowsheetTemplateIDType={flowsheet_template_id_type}", + "Comment=%(comment)s", + "InstantValueTaken=%(instant_value_taken)s", + "FlowsheetTemplateID=%(flowsheet_template_id)s", + "FlowsheetTemplateIDType=%(flowsheet_template_id_type)s", ) ), ) ) def __init__( - self, postgres, uri=URI, poll_timeout=60, uri_timeout=5 + self, + *, + epic: Epic, + comment: str = "Not for clinical use.", + contact_type_id: str = "csn", + patient_id_type: str = "uid", + uri=URI, + **kwargs, ) -> None: """__init__.""" - super().__init__(postgres, uri, poll_timeout, uri_timeout) + + super().__init__( + epic=epic, + uri=uri, + **kwargs, + ) + self.comment = comment + self.contact_type_id = contact_type_id + self.patient_id_type = patient_id_type def __call__(self): """__call__.""" - postgres = self.postgres + postgres = self.epic.postgres sql = postgres.sql with postgres.listen(sql.prediction.listen) as listen: with postgres.commit() as cur: self.recover(cur) self.listen(listen) + def as_yaml(self) -> Dict[str, Any]: + """As yaml.""" + return { + "comment": self.comment, + "contact_type_id": self.contact_type_id, + "patient_id_type": self.patient_id_type, + **super().as_yaml(), + } + def recover(self, cur): """Recover.""" - sql = self.postgres.sql + sql = self.epic.postgres.sql cur.execute(sql.epic.notification.recover) for each in cur.fetchall(): self.call_uri(each, cur) def call_uri(self, prediction, cur): """Call uri.""" - sql = self.postgres.sql + epic = self.epic + sql = epic.postgres.sql uri = self.uri % { "csn": prediction["csn"], "empi": prediction["empi"], "score": prediction["score"], } request = Request(uri, data=None) - with urlopen(request, self.uri_timeout) as response: + with urlopen(request, epic.uri_timeout) as response: if response.ok: cur.execute( sql.epic.notification.insert, @@ -119,20 +241,30 @@ def call_uri(self, prediction, cur): ) -class Verification(Abstract): +class Verification(FlowsheetEgress): """Verification Service.""" URI = "api/epic/2014/Clinical/Patient/GetFlowsheetRows/FlowsheetRows" def __init__( - self, postgres, uri=URI, poll_timeout=60, uri_timeout=5 + self, + *, + epic: Epic, + uri=URI, + flowsheet_id: str = "3040015333", + **kwargs, ) -> None: """__init__.""" - super().__init__(postgres, uri, poll_timeout, uri_timeout) + super().__init__( + epic=epic, + flowsheet_id=flowsheet_id, + uri=uri, + **kwargs, + ) def __call__(self): """__call__.""" - postgres = self.postgres + postgres = self.epic.postgres sql = postgres.sql with postgres.listen(sql.notification.listen) as listen: with postgres.commit() as cur: @@ -141,17 +273,18 @@ def __call__(self): def recover(self, cur): """Recover.""" - sql = self.postgres.sql + sql = self.epic.postgres.sql cur.execute(sql.epic.verification.recover) for each in cur.fetchall(): self.call_uri(each, cur) def call_uri(self, notification, cur): """Call uri.""" - sql = self.postgres.sql + epic = self.epic + sql = epic.postgres.sql # TODO add notification flowsheet ids to data? request = Request(self.URI, data=None) - with urlopen(request, self.uri_timeout) as response: + with urlopen(request, epic.uri_timeout) as response: if response.ok: cur.execute( sql.epic.verification.insert, diff --git a/src/dsdk/service.py b/src/dsdk/service.py index b46b5e2..4a29cd6 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -272,7 +272,7 @@ def parse( dest="env_file", type=str, help="env file", - default=env.get("env", None), + default=env.get("ENV", None), ) args = parser.parse_args(argv) if args.config_file is None: diff --git a/test/model.pkl b/test/model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..c6ad97fb54026af99cca187ac90e0c255209df83 GIT binary patch literal 101 zcmZo*nHt3a0X9Z0a68t oB^gtC1oZSle0{JUy@G6@KK8QIqT Date: Fri, 23 Jul 2021 14:52:31 -0400 Subject: [PATCH 09/33] Encapsulate Yaml Loader, Dumper, and closures for tag, pattern and env. --- src/dsdk/__init__.py | 8 ++--- src/dsdk/asset.py | 18 ++++++----- src/dsdk/env.py | 38 +++++++++++++++-------- src/dsdk/epic.py | 32 +++++++++++-------- src/dsdk/interval.py | 16 ++++++---- src/dsdk/model.py | 18 ++++++----- src/dsdk/persistor.py | 16 ++++++---- src/dsdk/service.py | 17 ++++++----- src/dsdk/utils.py | 71 ++++++++++++++++++++++++++++++++++++++----- 9 files changed, 165 insertions(+), 69 deletions(-) diff --git a/src/dsdk/__init__.py b/src/dsdk/__init__.py index 8a36e0b..9e78637 100644 --- a/src/dsdk/__init__.py +++ b/src/dsdk/__init__.py @@ -15,8 +15,6 @@ from .postgres import PredictionMixin as PostgresPredictionMixin from .service import Batch, Delegate, Service, Task from .utils import ( - YamlDumper, - YamlLoader, chunks, configure_logger, dump_json_file, @@ -29,7 +27,9 @@ profile, retry, yaml_dumps, + yaml_implicit_type, yaml_loads, + yaml_type, ) __all__ = ( @@ -49,8 +49,6 @@ "Postgres", "Service", "Task", - "YamlLoader", - "YamlDumper", "chunks", "configure_logger", "dump_json_file", @@ -64,4 +62,6 @@ "retry", "yaml_dumps", "yaml_loads", + "yaml_type", + "yaml_implicit_type", ) diff --git a/src/dsdk/asset.py b/src/dsdk/asset.py index 81fcba0..5b13110 100644 --- a/src/dsdk/asset.py +++ b/src/dsdk/asset.py @@ -9,9 +9,9 @@ from os.path import isdir from os.path import join as joinpath from os.path import splitext -from typing import Any, Dict +from typing import Any, Dict, Optional -from .utils import YamlDumper, YamlLoader +from .utils import yaml_type logger = getLogger(__name__) @@ -22,10 +22,14 @@ class Asset(Namespace): YAML = "!asset" @classmethod - def as_yaml_type(cls): + def as_yaml_type(cls, tag: Optional[str] = None): """As yaml type.""" - YamlLoader.add_constructor(cls.YAML, cls._yaml_init) - YamlDumper.add_representer(cls, cls._yaml_repr) + yaml_type( + cls, + tag or cls.YAML, + init=cls._yaml_init, + repr=cls._yaml_repr, + ) @classmethod def build(cls, *, path: str, ext: str): @@ -51,9 +55,9 @@ def _yaml_init(cls, loader, node): return cls.build(**loader.construct_mapping(node, deep=True)) @classmethod - def _yaml_repr(cls, dumper, self): + def _yaml_repr(cls, dumper, self, *, tag: str): """Yaml repr.""" - return dumper.represent_mapping(cls.YAML, self.as_yaml()) + return dumper.represent_mapping(tag, self.as_yaml()) def __init__( self, diff --git a/src/dsdk/env.py b/src/dsdk/env.py index ec7cb95..80c2d32 100644 --- a/src/dsdk/env.py +++ b/src/dsdk/env.py @@ -5,9 +5,9 @@ from os import environ as os_env from re import compile as re_compile -from typing import Mapping, Optional +from typing import Mapping, Optional, Pattern -from .utils import YamlLoader +from .utils import yaml_implicit_type class Env: @@ -17,22 +17,34 @@ class Env: PATTERN = re_compile(r".?\$\{([^\}^\{]+)\}.?") @classmethod - def as_yaml_type(cls, *, env: Optional[Mapping[str, str]] = None): + def as_yaml_type( + cls, + tag: Optional[str] = None, + *, + env: Optional[Mapping[str, str]] = None, + pattern: Optional[Pattern] = None, + ): """As yaml type.""" - _env = env or os_env - - def _yaml_init(loader, node) -> str: - """This closure passed env.""" - return cls._yaml_init(loader, node, _env) - - YamlLoader.add_implicit_resolver(cls.YAML, cls.PATTERN, None) - YamlLoader.add_constructor(cls.YAML, _yaml_init) + yaml_implicit_type( + cls, + tag or cls.YAML, + pattern=pattern or cls.PATTERN, + init=cls._yaml_init, + env=env or os_env, + ) @classmethod - def _yaml_init(cls, loader, node, env: Mapping[str, str]): + def _yaml_init( + cls, + loader, + node, + *, + env: Mapping[str, str], + pattern: Pattern, + ): """From yaml.""" value = loader.construct_scalar(node) - match = cls.PATTERN.findall(value) + match = pattern.findall(value) if not match: return value for group in match: diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index f4f8025..60c5b4a 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -3,11 +3,11 @@ from logging import getLogger from select import select -from typing import Any, Dict +from typing import Any, Dict, Optional from urllib.request import Request, urlopen from .postgres import Persistor as Postgres -from .utils import YamlDumper, YamlLoader +from .utils import yaml_type logger = getLogger(__name__) @@ -18,10 +18,14 @@ class Epic: YAML = "!epic" @classmethod - def as_yaml_type(cls): + def as_yaml_type(cls, *, tag: Optional[str] = None): """As yaml type.""" - YamlLoader.add_constructor(cls.YAML, cls._yaml_init) - YamlDumper.add_representer(cls, cls._yaml_repr) + yaml_type( + cls, + tag or cls.YAML, + init=cls._yaml_init, + repr=cls._yaml_repr, + ) @classmethod def _yaml_init(cls, loader, node): @@ -29,9 +33,9 @@ def _yaml_init(cls, loader, node): return cls(**loader.construct_mapping(node, deep=True)) @classmethod - def _yaml_repr(cls, dumper, self): + def _yaml_repr(cls, dumper, self, *, tag: str): """Yaml repr.""" - return dumper.represent_mapper(cls.YAML, self.as_yaml()) + return dumper.represent_mapping(tag, self.as_yaml()) def __init__( self, @@ -72,10 +76,14 @@ class FlowsheetEgress: # pylint: disable=too-many-instance-attributes YAML = "!flowsheetegress" @classmethod - def as_yaml_type(cls): + def as_yaml_type(cls, tag: Optional[str] = None): """As yaml type.""" - YamlLoader.add_constructor(cls.YAML, cls._yaml_init) - YamlDumper.add_representer(cls, cls._yaml_repr) + yaml_type( + cls, + tag or cls.YAML, + init=cls._yaml_init, + repr=cls._yaml_repr, + ) @classmethod def _yaml_init(cls, loader, node): @@ -83,9 +91,9 @@ def _yaml_init(cls, loader, node): return cls(**loader.construct_mapping(node, deep=True)) @classmethod - def _yaml_repr(cls, dumper, self): + def _yaml_repr(cls, dumper, self, *, tag): """Yaml repr.""" - return dumper.represent_mapper(cls.YAML, self.as_yaml()) + return dumper.represent_mapper(tag, self.as_yaml()) def __init__( self, diff --git a/src/dsdk/interval.py b/src/dsdk/interval.py index 77cccfa..08f9570 100644 --- a/src/dsdk/interval.py +++ b/src/dsdk/interval.py @@ -4,7 +4,7 @@ from datetime import datetime from typing import Any, Dict, Optional -from .utils import YamlDumper, YamlLoader +from .utils import yaml_type class Interval: @@ -13,10 +13,14 @@ class Interval: YAML = "!interval" @classmethod - def as_yaml_type(cls): + def as_yaml_type(cls, tag: Optional[str] = None): """As yaml type.""" - YamlLoader.add_constructor(cls.YAML, cls._yaml_init) - YamlDumper.add_representer(cls, cls._yaml_repr) + yaml_type( + cls, + tag or cls.YAML, + init=cls._yaml_init, + repr=cls._yaml_repr, + ) @classmethod def _yaml_init(cls, loader, node): @@ -24,9 +28,9 @@ def _yaml_init(cls, loader, node): return cls(**loader.construct_mapping(node, deep=True)) @classmethod - def _yaml_repr(cls, dumper, self): + def _yaml_repr(cls, dumper, self, *, tag: str): """Yaml repr.""" - return dumper.represent_mapping(cls.YAML, self.as_yaml()) + return dumper.represent_mapping(tag, self.as_yaml()) def __init__(self, on: datetime, end: Optional[datetime] = None): """__init__.""" diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 4078b4f..6df57a4 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -6,10 +6,10 @@ from abc import ABC from contextlib import contextmanager from logging import getLogger -from typing import TYPE_CHECKING, Any, Dict, Generator +from typing import TYPE_CHECKING, Any, Dict, Generator, Optional from .service import Delegate, Service -from .utils import YamlDumper, YamlLoader, load_pickle_file +from .utils import load_pickle_file, yaml_type logger = getLogger(__name__) @@ -26,10 +26,14 @@ class Model: # pylint: disable=too-few-public-methods YAML = "!model" @classmethod - def as_yaml_type(cls) -> None: + def as_yaml_type(cls, tag: Optional[str] = None) -> None: """As yaml type.""" - YamlLoader.add_constructor(cls.YAML, cls._yaml_init) - YamlDumper.add_representer(cls, cls._yaml_repr) + yaml_type( + cls, + tag or cls.YAML, + init=cls._yaml_init, + repr=cls._yaml_repr, + ) @classmethod def _yaml_init(cls, loader, node): @@ -44,9 +48,9 @@ def _yaml_init(cls, loader, node): return pkl @classmethod - def _yaml_repr(cls, dumper, self): + def _yaml_repr(cls, dumper, self, *, tag: str): """Yaml_repr.""" - return dumper.represent_scalar(cls.YAML, self.as_yaml()) + return dumper.represent_scalar(tag, self.as_yaml()) def __init__( self, diff --git a/src/dsdk/persistor.py b/src/dsdk/persistor.py index ca3d222..5c370a8 100644 --- a/src/dsdk/persistor.py +++ b/src/dsdk/persistor.py @@ -13,7 +13,7 @@ from pandas import DataFrame, concat from .asset import Asset -from .utils import YamlDumper, YamlLoader, chunks +from .utils import chunks, yaml_type logger = getLogger(__name__) @@ -230,10 +230,14 @@ class Persistor(AbstractPersistor): YAML = "!basepersistor" @classmethod - def as_yaml_type(cls) -> None: + def as_yaml_type(cls, tag: Optional[str] = None) -> None: """As yaml type.""" - YamlLoader.add_constructor(cls.YAML, cls._yaml_init) - YamlDumper.add_representer(cls, cls._yaml_repr) + yaml_type( + cls, + tag or cls.YAML, + init=cls._yaml_init, + repr=cls._yaml_repr, + ) @classmethod def _yaml_init(cls, loader, node): @@ -241,9 +245,9 @@ def _yaml_init(cls, loader, node): return cls(**loader.construct_mapping(node, deep=True)) @classmethod - def _yaml_repr(cls, dumper, self): + def _yaml_repr(cls, dumper, self, *, tag: str): """Yaml repr.""" - return dumper.represent_mapping(cls.YAML, self.as_yaml()) + return dumper.represent_mapping(tag, self.as_yaml()) def __init__( # pylint: disable=too-many-arguments self, diff --git a/src/dsdk/service.py b/src/dsdk/service.py index 4a29cd6..c988994 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -30,12 +30,11 @@ from .env import Env from .interval import Interval from .utils import ( - YamlDumper, - YamlLoader, configure_logger, get_tzinfo, now_utc_datetime, yaml_loads, + yaml_type, ) try: @@ -211,10 +210,14 @@ class Service: # pylint: disable=too-many-instance-attributes VERSION = __version__ @classmethod - def as_yaml_type(cls) -> None: + def as_yaml_type(cls, tag: Optional[str] = None) -> None: """As yaml type.""" - YamlLoader.add_constructor(cls.YAML, cls._yaml_init) - YamlDumper.add_representer(cls, cls._yaml_repr) + yaml_type( + cls, + tag or cls.YAML, + init=cls._yaml_init, + repr=cls._yaml_repr, + ) @classmethod @contextmanager @@ -348,9 +351,9 @@ def _yaml_init(cls, loader, node): return cls(**loader.construct_mapping(node, deep=True)) @classmethod - def _yaml_repr(cls, dumper, self): + def _yaml_repr(cls, dumper, self, *, tag: str): """Yaml repr.""" - return dumper.represent_mapping(cls.YAML, self.as_yaml()) + return dumper.represent_mapping(tag, self.as_yaml()) def __init__( self, diff --git a/src/dsdk/utils.py b/src/dsdk/utils.py index fcdd97b..3436cca 100644 --- a/src/dsdk/utils.py +++ b/src/dsdk/utils.py @@ -14,17 +14,17 @@ from sys import stdout from time import perf_counter_ns from time import sleep as default_sleep -from typing import Any, Callable, Generator, Sequence +from typing import Any, Callable, Generator, Optional, Pattern, Sequence, Type from yaml import dump as _yaml_dumps from yaml import load as _yaml_loads try: - from yaml import CSafeDumper as YamlDumper # type: ignore[misc] - from yaml import CSafeLoader as YamlLoader # type: ignore[misc] + from yaml import CSafeDumper as Dumper # type: ignore[misc] + from yaml import CSafeLoader as Loader # type: ignore[misc] except ImportError: - from yaml import SafeDumper as YamlDumper # type: ignore[misc] - from yaml import SafeLoader as YamlLoader # type: ignore[misc] + from yaml import SafeDumper as Dumper # type: ignore[misc] + from yaml import SafeLoader as Loader # type: ignore[misc] from dateutil import parser, tz @@ -189,12 +189,69 @@ def utc_datetime_from_epoch_ms(epoch_ms: float) -> datetime: def yaml_dumps(data, **kwargs): """Yaml dumps.""" - return _yaml_dumps(data=data, Dumper=YamlDumper, **kwargs) + return _yaml_dumps(data=data, Dumper=Dumper, **kwargs) def yaml_loads(stream, **kwargs): """Yaml loads.""" - return _yaml_loads(stream=stream, Loader=YamlLoader, **kwargs) + return _yaml_loads(stream=stream, Loader=Loader, **kwargs) + + +def yaml_type( + cls: type, + tag: str, + *, + init: Optional[Callable] = None, + repr: Optional[Callable] = None, # pylint: disable=redefined-builtin + loader: Optional[Type[Loader]] = None, + dumper: Optional[Type[Dumper]] = None, + **kwargs, +): + """Yaml type.""" + _loader = loader or Loader + _dumper = dumper or Dumper + if init is not None: + + def _init_closure(loader, node): + return init(loader, node, **kwargs) + + _loader.add_constructor(tag, _init_closure) + + if repr is not None: + + def _repr_closure(dumper, self): + return repr(dumper, self, tag=tag, **kwargs) + + _dumper.add_representer(cls, _repr_closure) + + +def yaml_implicit_type( + cls: type, + tag: str, + *, + init: Callable, + pattern: Pattern, + repr: Optional[Callable] = None, # pylint: disable=redefined-builtin + loader: Optional[Type[Loader]] = None, + dumper: Optional[Type[Dumper]] = None, + **kwargs, +): + """Yaml implicit type.""" + _loader = loader or Loader + _dumper = dumper or Dumper + + def _init_closure(loader, node): + return init(loader, node, pattern=pattern, **kwargs) + + _loader.add_constructor(tag, _init_closure) + _loader.add_implicit_resolver(tag, pattern, None) + + if repr is not None: + + def _repr_closure(dumper, self): + return repr(dumper, self, tag=tag, pattern=pattern, **kwargs) + + _dumper.add_representer(cls, _repr_closure) class StubError(Exception): From 9c105a0b391abf0a0bb35bff950cd8bde4ee1a4e Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Tue, 27 Jul 2021 11:16:40 -0400 Subject: [PATCH 10/33] Integrate cfgenvy --- buster.dockerfile | 2 +- pyproject.toml | 6 +-- setup.py | 13 +++-- src/dsdk/__init__.py | 14 ------ src/dsdk/asset.py | 2 +- src/dsdk/env.py | 75 ----------------------------- src/dsdk/epic.py | 3 +- src/dsdk/interval.py | 2 +- src/dsdk/model.py | 4 +- src/dsdk/persistor.py | 3 +- src/dsdk/service.py | 104 ++-------------------------------------- src/dsdk/utils.py | 91 +---------------------------------- test/test_dsdk.py | 108 ++++++++++++++++++++++++------------------ test/test_import.py | 11 +++++ 14 files changed, 98 insertions(+), 340 deletions(-) delete mode 100644 src/dsdk/env.py create mode 100644 test/test_import.py diff --git a/buster.dockerfile b/buster.dockerfile index f42a8b7..1e034c9 100644 --- a/buster.dockerfile +++ b/buster.dockerfile @@ -1,6 +1,6 @@ ARG IFLAGS="--quiet --no-cache-dir --user" -FROM python:3.9.4-slim-buster as build +FROM python:3.9.6-slim-buster as build ARG IFLAGS WORKDIR /root ENV PATH /root/.local/bin:$PATH diff --git a/pyproject.toml b/pyproject.toml index 094852d..d24d4b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,15 @@ [build-system] requires = [ - "setuptools>=50.3.0", + "setuptools>=57.4.0", "setuptools_scm[toml]>=4.1.2", "wheel>=0.35.1", ] [project] authors = [ + "Jason Lubken", "Michael Becker", "Corey Chivers", - "Jason Lubken" ] license = "MIT" name = "dsdk" @@ -62,7 +62,7 @@ ignore-imports = "yes" [tool.pytest.ini_options] addopts = "-ra --cov=dsdk --cov-report=term-missing --strict-markers --ignore=.eggs --tb=short" testpaths = ["test"] -norecursedirs = ".env .git build dist" +norecursedirs = ".env .git .venv build dist" python_files = "test.py tests.py test_*.py *_test.py" [tool.setuptools_scm] diff --git a/setup.py b/setup.py index 9189fec..9f87a7f 100644 --- a/setup.py +++ b/setup.py @@ -4,12 +4,15 @@ from setuptools import find_packages, setup INSTALL_REQUIRES = ( - "configargparse>=1.5.1", + ( + "cfgenvy@" + "git+https://github.com/pennsignals/cfgenvy.git" + "@1.1.0#egg=cfgenvy" + ), "numpy>=1.15.4", "pandas>=0.23.4", - "pip>=20.2.4", - "pyyaml>=5.3.1", - "setuptools>=50.3.2", + "pip>=21.2.1", + "setuptools>=57.4.0", "wheel>=0.35.1", ) @@ -34,7 +37,7 @@ "flake8-logging-format", "flake8-mutable", "flake8-sorted-keys", - "isort<=4.2.5", + "isort", "mypy", "pep8-naming", "pre-commit", diff --git a/src/dsdk/__init__.py b/src/dsdk/__init__.py index 9e78637..ebdb72b 100644 --- a/src/dsdk/__init__.py +++ b/src/dsdk/__init__.py @@ -2,7 +2,6 @@ """Data Science Deployment Kit.""" from .asset import Asset -from .env import Env from .interval import Interval from .model import Mixin as ModelMixin from .model import Model @@ -19,24 +18,17 @@ configure_logger, dump_json_file, dump_pickle_file, - dump_yaml_file, load_json_file, load_pickle_file, - load_yaml_file, now_utc_datetime, profile, retry, - yaml_dumps, - yaml_implicit_type, - yaml_loads, - yaml_type, ) __all__ = ( "Asset", "Batch", "Delegate", - "Env", "Interval", "Model", "ModelMixin", @@ -53,15 +45,9 @@ "configure_logger", "dump_json_file", "dump_pickle_file", - "dump_yaml_file", "load_json_file", "load_pickle_file", - "load_yaml_file", "profile", "now_utc_datetime", "retry", - "yaml_dumps", - "yaml_loads", - "yaml_type", - "yaml_implicit_type", ) diff --git a/src/dsdk/asset.py b/src/dsdk/asset.py index 5b13110..b86b5eb 100644 --- a/src/dsdk/asset.py +++ b/src/dsdk/asset.py @@ -11,7 +11,7 @@ from os.path import splitext from typing import Any, Dict, Optional -from .utils import yaml_type +from cfgenvy import yaml_type logger = getLogger(__name__) diff --git a/src/dsdk/env.py b/src/dsdk/env.py deleted file mode 100644 index 80c2d32..0000000 --- a/src/dsdk/env.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- -"""Env.""" - -from __future__ import annotations - -from os import environ as os_env -from re import compile as re_compile -from typing import Mapping, Optional, Pattern - -from .utils import yaml_implicit_type - - -class Env: - """Env.""" - - YAML = "!env" - PATTERN = re_compile(r".?\$\{([^\}^\{]+)\}.?") - - @classmethod - def as_yaml_type( - cls, - tag: Optional[str] = None, - *, - env: Optional[Mapping[str, str]] = None, - pattern: Optional[Pattern] = None, - ): - """As yaml type.""" - yaml_implicit_type( - cls, - tag or cls.YAML, - pattern=pattern or cls.PATTERN, - init=cls._yaml_init, - env=env or os_env, - ) - - @classmethod - def _yaml_init( - cls, - loader, - node, - *, - env: Mapping[str, str], - pattern: Pattern, - ): - """From yaml.""" - value = loader.construct_scalar(node) - match = pattern.findall(value) - if not match: - return value - for group in match: - variable = env.get(group, None) - if not variable: - raise ValueError(f"No value for ${{{group}}}.") - value = value.replace(f"${{{group}}}", variable) - return value - - @classmethod - def load(cls, path: str) -> Mapping[str, str]: - """Env load.""" - with open(path) as fin: - return cls.loads(fin) - - @classmethod - def loads(cls, stream) -> Mapping[str, str]: - """Env loads.""" - result = {} - for line in stream: - line = line.strip() - if not line: - continue - if line.startswith("#"): - continue - key, value = line.split("=", 1) - result[key] = value - return result diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index 60c5b4a..d89e731 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -6,8 +6,9 @@ from typing import Any, Dict, Optional from urllib.request import Request, urlopen +from cfgenvy import yaml_type + from .postgres import Persistor as Postgres -from .utils import yaml_type logger = getLogger(__name__) diff --git a/src/dsdk/interval.py b/src/dsdk/interval.py index 08f9570..054a41f 100644 --- a/src/dsdk/interval.py +++ b/src/dsdk/interval.py @@ -4,7 +4,7 @@ from datetime import datetime from typing import Any, Dict, Optional -from .utils import yaml_type +from cfgenvy import yaml_type class Interval: diff --git a/src/dsdk/model.py b/src/dsdk/model.py index 6df57a4..46534a6 100644 --- a/src/dsdk/model.py +++ b/src/dsdk/model.py @@ -8,8 +8,10 @@ from logging import getLogger from typing import TYPE_CHECKING, Any, Dict, Generator, Optional +from cfgenvy import yaml_type + from .service import Delegate, Service -from .utils import load_pickle_file, yaml_type +from .utils import load_pickle_file logger = getLogger(__name__) diff --git a/src/dsdk/persistor.py b/src/dsdk/persistor.py index 5c370a8..ebf7f6f 100644 --- a/src/dsdk/persistor.py +++ b/src/dsdk/persistor.py @@ -10,10 +10,11 @@ from tempfile import NamedTemporaryFile from typing import Any, Dict, Generator, Optional, Sequence, Tuple +from cfgenvy import yaml_type from pandas import DataFrame, concat from .asset import Asset -from .utils import chunks, yaml_type +from .utils import chunks logger = getLogger(__name__) diff --git a/src/dsdk/service.py b/src/dsdk/service.py index c988994..ef233f8 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -4,14 +4,11 @@ from __future__ import annotations import pickle -from argparse import ArgumentParser from collections import OrderedDict from contextlib import contextmanager from datetime import date, datetime, tzinfo from json import dumps from logging import getLogger -from os import environ as os_env -from sys import argv as sys_argv from typing import ( Any, Callable, @@ -23,19 +20,13 @@ Sequence, ) +from cfgenvy import Parser, yaml_type from pandas import DataFrame from pkg_resources import DistributionNotFound, get_distribution from .asset import Asset -from .env import Env from .interval import Interval -from .utils import ( - configure_logger, - get_tzinfo, - now_utc_datetime, - yaml_loads, - yaml_type, -) +from .utils import configure_logger, get_tzinfo, now_utc_datetime try: __version__ = get_distribution("dsdk").version @@ -190,7 +181,7 @@ def __setitem__(self, key, value): super().__setitem__(key, value) -class Service: # pylint: disable=too-many-instance-attributes +class Service(Parser): # pylint: disable=too-many-instance-attributes """Service.""" ON = dumps({"key": "%s.on"}) @@ -245,94 +236,6 @@ def main(cls): with cls.context("main") as service: service() - @classmethod - def parse( - cls, - *, - argv: Optional[List[str]] = None, - env: Optional[Mapping[str, str]] = None, - ) -> Service: - """Parse.""" - if argv is None: - argv = sys_argv[1:] - assert argv is not None - if env is None: - env = os_env - assert env is not None - - parser = ArgumentParser() - parser.add_argument( - "-c", - "--config", - dest="config_file", - type=str, - help="configuration yaml file", - default=env.get("CONFIG", None), - ) - parser.add_argument( - "-e", - "--env", - dest="env_file", - type=str, - help="env file", - default=env.get("ENV", None), - ) - args = parser.parse_args(argv) - if args.config_file is None: - parser.error( - " ".join( - ( - "the following arguments are required:", - "-c/--config or CONFIG from env variable", - ) - ) - ) - return cls.load( - config_file=args.config_file, - env=env, - env_file=args.env_file, - ) - - @classmethod - def load( - cls, - *, - config_file: str, - env: Optional[Mapping[str, str]] = None, - env_file: Optional[str] = None, - ): - """Load.""" - if env is None: - env = os_env - assert env is not None - - if env_file: - with open(env_file) as fin: - envs = fin.read() - logger.debug("Environment variables are only from %s", env_file) - else: - logger.debug("Environment variables are being used.") - - with open(config_file) as fin: - configs = fin.read() - return cls.loads(configs=configs, env=env, envs=envs) - - @classmethod - def loads( - cls, - *, - configs: str, - env: Mapping[str, str], - envs: Optional[str] = None, - ) -> Service: - """Loads from strings.""" - if envs is not None: - env = Env.loads(envs) - - Env.as_yaml_type(env=env) - cls.yaml_types() - return yaml_loads(configs) - @classmethod def validate_gold(cls): """Validate gold.""" @@ -344,6 +247,7 @@ def yaml_types(cls) -> None: """Yaml types.""" Asset.as_yaml_type() Interval.as_yaml_type() + cls.as_yaml_type() @classmethod def _yaml_init(cls, loader, node): diff --git a/src/dsdk/utils.py b/src/dsdk/utils.py index 3436cca..3b9eb60 100644 --- a/src/dsdk/utils.py +++ b/src/dsdk/utils.py @@ -14,17 +14,7 @@ from sys import stdout from time import perf_counter_ns from time import sleep as default_sleep -from typing import Any, Callable, Generator, Optional, Pattern, Sequence, Type - -from yaml import dump as _yaml_dumps -from yaml import load as _yaml_loads - -try: - from yaml import CSafeDumper as Dumper # type: ignore[misc] - from yaml import CSafeLoader as Loader # type: ignore[misc] -except ImportError: - from yaml import SafeDumper as Dumper # type: ignore[misc] - from yaml import SafeLoader as Loader # type: ignore[misc] +from typing import Any, Callable, Generator, Sequence from dateutil import parser, tz @@ -83,12 +73,6 @@ def dump_pickle_file(obj: Any, path: str) -> None: pickle_dump(obj, fout) -def dump_yaml_file(data: Any, path: str, **kwargs) -> None: - """Dump yaml file.""" - with open(path, "w") as fout: - yaml_dumps(data=data, stream=fout, **kwargs) - - def epoch_ms_from_utc_datetime(utc: datetime) -> float: """Epoch ms from non-naive UTC datetime.""" return utc.timestamp() * 1000 @@ -113,12 +97,6 @@ def load_pickle_file(path: str) -> object: return pickle_load(fin) -def load_yaml_file(path: str): - """Load yaml file.""" - with open(path) as fin: - return yaml_loads(fin) - - def now_utc_datetime() -> datetime: """Non-naive now UTC datetime.""" return datetime.now(tz=timezone.utc) @@ -187,72 +165,5 @@ def utc_datetime_from_epoch_ms(epoch_ms: float) -> datetime: return datetime.fromtimestamp(epoch_ms / 1000, tz=timezone.utc) -def yaml_dumps(data, **kwargs): - """Yaml dumps.""" - return _yaml_dumps(data=data, Dumper=Dumper, **kwargs) - - -def yaml_loads(stream, **kwargs): - """Yaml loads.""" - return _yaml_loads(stream=stream, Loader=Loader, **kwargs) - - -def yaml_type( - cls: type, - tag: str, - *, - init: Optional[Callable] = None, - repr: Optional[Callable] = None, # pylint: disable=redefined-builtin - loader: Optional[Type[Loader]] = None, - dumper: Optional[Type[Dumper]] = None, - **kwargs, -): - """Yaml type.""" - _loader = loader or Loader - _dumper = dumper or Dumper - if init is not None: - - def _init_closure(loader, node): - return init(loader, node, **kwargs) - - _loader.add_constructor(tag, _init_closure) - - if repr is not None: - - def _repr_closure(dumper, self): - return repr(dumper, self, tag=tag, **kwargs) - - _dumper.add_representer(cls, _repr_closure) - - -def yaml_implicit_type( - cls: type, - tag: str, - *, - init: Callable, - pattern: Pattern, - repr: Optional[Callable] = None, # pylint: disable=redefined-builtin - loader: Optional[Type[Loader]] = None, - dumper: Optional[Type[Dumper]] = None, - **kwargs, -): - """Yaml implicit type.""" - _loader = loader or Loader - _dumper = dumper or Dumper - - def _init_closure(loader, node): - return init(loader, node, pattern=pattern, **kwargs) - - _loader.add_constructor(tag, _init_closure) - _loader.add_implicit_resolver(tag, pattern, None) - - if repr is not None: - - def _repr_closure(dumper, self): - return repr(dumper, self, tag=tag, pattern=pattern, **kwargs) - - _dumper.add_representer(cls, _repr_closure) - - class StubError(Exception): """StubError.""" diff --git a/test/test_dsdk.py b/test/test_dsdk.py index 6109fde..91081ed 100644 --- a/test/test_dsdk.py +++ b/test/test_dsdk.py @@ -2,8 +2,9 @@ """Test dsdk.""" from io import StringIO -from typing import Any, Callable, Dict, Tuple +from typing import Any, Callable, Dict, Tuple, Type +from cfgenvy import yaml_dumps from pandas import DataFrame from pytest import mark @@ -20,7 +21,6 @@ Task, dump_pickle_file, retry, - yaml_dumps, ) @@ -57,7 +57,7 @@ def __call__(self, batch: Batch, service: Service) -> None: class MyService(ModelMixin, MssqlMixin, PostgresMixin, Service): """MyService.""" - YAML = "!myservice" + YAML = "!test" @classmethod def yaml_types(cls): @@ -71,35 +71,8 @@ def __init__(self, **kwargs): super().__init__(pipeline=pipeline, **kwargs) -def build_from_parameters(cls) -> Tuple[Callable, Dict[str, Any]]: - """Build from parameters.""" - model = Model(name="test", path="./test/model.pkl", version="0.0.1-rc.1") - mssql = Mssql( - username="mssql", - password="password", - host="0.0.0.0", - port=1433, - database="test", - sql=Asset.build(path="./assets/mssql", ext=".sql"), - tables=("a", "b", "c"), - ) - postgres = Postgres( - username="postgres", - password="password", - host="0.0.0.0", - port=5432, - database="test", - sql=Asset.build(path="./assets/postgres", ext=".sql"), - tables=("ichi", "ni", "san", "shi", "go"), - ) - return ( - cls, - {"model": model, "mssql": mssql, "postgres": postgres}, - ) - - CONFIGS = """ -!myservice +!test mssql: !mssql database: test host: 0.0.0.0 @@ -137,7 +110,7 @@ def build_from_parameters(cls) -> Tuple[Callable, Dict[str, Any]]: """.strip() EXPECTED = """ -!myservice +!test as_of: null duration: null gold: null @@ -174,32 +147,79 @@ def build_from_parameters(cls) -> Tuple[Callable, Dict[str, Any]]: """.strip() -def build_from_yaml(cls) -> Tuple[Callable, Dict[str, Any]]: +def build( + cls: Type, + expected: str = EXPECTED, +) -> Tuple[Callable, Dict[str, Any], str]: + """Build from parameters.""" + cls.yaml_types() + model = Model(name="test", path="./test/model.pkl", version="0.0.1-rc.1") + mssql = Mssql( + username="mssql", + password="password", + host="0.0.0.0", + port=1433, + database="test", + sql=Asset.build(path="./assets/mssql", ext=".sql"), + tables=("a", "b", "c"), + ) + postgres = Postgres( + username="postgres", + password="password", + host="0.0.0.0", + port=5432, + database="test", + sql=Asset.build(path="./assets/postgres", ext=".sql"), + tables=("ichi", "ni", "san", "shi", "go"), + ) + return ( + cls, + { + "model": model, + "mssql": mssql, + "postgres": postgres, + }, + expected, + ) + + +def deserialize( + cls: Type, + configs: str = CONFIGS, + envs: str = ENVS, + expected: str = EXPECTED, +) -> Tuple[Callable, Dict[str, Any], str]: """Build from yaml.""" pickle_file = "./test/model.pkl" dump_pickle_file( Model(name="test", path=pickle_file, version="0.0.1"), pickle_file ) - configs = StringIO(CONFIGS) env = {"POSTGRES_PASSWORD": "oops!", "MSSQL_PASSWORD": "oops!"} - envs = StringIO(ENVS) return ( cls.loads, - {"configs": configs, "env": env, "envs": envs}, + { + "configs": StringIO(configs), + "env": env, + "envs": StringIO(envs), + }, + expected, ) @mark.parametrize( - "cls,kwargs", - (build_from_yaml(MyService), build_from_parameters(MyService)), + "cls,kwargs,expected", + ( + build(MyService), + deserialize(MyService), + ), ) def test_service( - cls, # pylint: disable=redefined-outer-name + cls: Callable, # pylint: disable=redefined-outer-name kwargs: Dict[str, Any], + expected: str, ): """Test parameters, config, and env.""" - expected = EXPECTED service = cls(**kwargs) assert service.__class__ is MyService assert service.model.__class__ is Model @@ -207,9 +227,7 @@ def test_service( assert service.mssql.__class__ is Mssql assert service.postgres.password == "password" assert service.mssql.password == "password" - buf = StringIO() - buf.write(yaml_dumps(service)) - actual = buf.getvalue().strip() + actual = yaml_dumps(service).strip() assert actual == expected @@ -275,7 +293,3 @@ def explode(): except NotImplementedError as exception: assert actual == expected assert str(exception) == "when?" - - -if __name__ == "__main__": - test_service(*build_from_yaml(MyService)) diff --git a/test/test_import.py b/test/test_import.py new file mode 100644 index 0000000..e55ba7b --- /dev/null +++ b/test/test_import.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +"""Test import.""" + + +def test_import(): + """Test import.""" + import dsdk # pylint: disable=import-outside-toplevel + + assert dsdk.Asset is not None + assert dsdk.Interval is not None + assert dsdk.Service is not None From d59b93010dd9df0adbc7a90a76ef0c9b19b5de3a Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Tue, 27 Jul 2021 11:22:52 -0400 Subject: [PATCH 11/33] Update dockerfile assets --- buster.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buster.dockerfile b/buster.dockerfile index 1e034c9..cadf9c8 100644 --- a/buster.dockerfile +++ b/buster.dockerfile @@ -14,7 +14,7 @@ COPY pyproject.toml . COPY .pre-commit-config.yaml . COPY .git ./.git COPY src ./src -COPY sql ./sql +COPY assets ./assets COPY test ./test RUN \ chmod +x /usr/bin/tini && \ From 653fc6911d936acdd3e34de41715e3a3cf5b82a6 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 2 Aug 2021 17:16:39 -0400 Subject: [PATCH 12/33] Update interconnect --- local/.gitignore | 1 + local/configuration.yaml | 23 +++++++++++++++++++ src/dsdk/{epic.py => interconnect.py} | 33 ++++++++++++++------------- 3 files changed, 41 insertions(+), 16 deletions(-) create mode 100644 local/configuration.yaml rename src/dsdk/{epic.py => interconnect.py} (94%) diff --git a/local/.gitignore b/local/.gitignore index d6b7ef3..b87b8a0 100644 --- a/local/.gitignore +++ b/local/.gitignore @@ -1,2 +1,3 @@ * !.gitignore +!configuration.yaml diff --git a/local/configuration.yaml b/local/configuration.yaml new file mode 100644 index 0000000..5e5b5e8 --- /dev/null +++ b/local/configuration.yaml @@ -0,0 +1,23 @@ +interconnect: !interconnect + authorization: ${INTERCONNECT_AUTHORIZATION} + comment: "Not for clinical use." + cookie: ${INTERCONNECT_COOKIE} + contact_type_id: csn + flowsheet_id: ${INTERCONNECT_FLOWSHEET_ID} + flowsheet_id_type: internal + flowsheet_template_id: 3040005300 + flowsheet_template_id_type: internal + lookback_hours: 72 + patient_id_type: uid + notification: !notification + uri: ${INTERCONNECT_HOST}${INTERCONNECT_NOTIFICATION_URI} + user_id: pennsignals + user_id_type: external + verification: !verification + uri: ${INTERCONNECT_HOST}${INTERCONNECT_VERIFICATION_URI} +postgres: !postgres + database: ${POSTGRES_DATABASE} + host: ${POSTRES_HOST} + password: ${POSTGRES_PASSWORD} + port: 5432 + username: ${POSTGRES_USERNAME} diff --git a/src/dsdk/epic.py b/src/dsdk/interconnect.py similarity index 94% rename from src/dsdk/epic.py rename to src/dsdk/interconnect.py index d89e731..c3584c5 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/interconnect.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Epic microservices.""" +"""Interconnect.""" from logging import getLogger from select import select @@ -13,10 +13,10 @@ logger = getLogger(__name__) -class Epic: - """Epic.""" +class Interconnect: + """Interconnect.""" - YAML = "!epic" + YAML = "!interconnect" @classmethod def as_yaml_type(cls, *, tag: Optional[str] = None): @@ -99,14 +99,14 @@ def _yaml_repr(cls, dumper, self, *, tag): def __init__( self, *, - epic: Epic, + interconnect: Interconnect, uri: str, flowsheet_id: str = "3040015333", flowsheet_template_id: str = "3040005300", flowsheet_template_id_type: str = "internal", ) -> None: """__init__.""" - self.epic = epic + self.interconnect = interconnect self.flowsheet_id = flowsheet_id self.flowsheet_template_id = flowsheet_template_id self.flowsheet_template_id_type = flowsheet_template_id_type @@ -154,9 +154,9 @@ class Notification(FlowsheetEgress): YAML = "!notification" - URI = "?".join( + QUERY = "".join( ( - "api/epic/2011/Clinical/Patient/AddFlowsheetValue/FlowsheetValue?", + "?", "&".join( ( "PatientID=%(empi)s", @@ -180,18 +180,19 @@ class Notification(FlowsheetEgress): def __init__( self, *, - epic: Epic, + uri: str, + interconnect: Interconnect, comment: str = "Not for clinical use.", contact_type_id: str = "csn", patient_id_type: str = "uid", - uri=URI, + query: str = QUERY, **kwargs, ) -> None: """__init__.""" super().__init__( - epic=epic, - uri=uri, + interconnect=interconnect, + uri=uri + query, **kwargs, ) self.comment = comment @@ -253,19 +254,19 @@ def call_uri(self, prediction, cur): class Verification(FlowsheetEgress): """Verification Service.""" - URI = "api/epic/2014/Clinical/Patient/GetFlowsheetRows/FlowsheetRows" + YAML = "!verification" def __init__( self, *, - epic: Epic, - uri=URI, + interconnect: Interconnect, + uri: str, flowsheet_id: str = "3040015333", **kwargs, ) -> None: """__init__.""" super().__init__( - epic=epic, + interconnect=interconnect, flowsheet_id=flowsheet_id, uri=uri, **kwargs, From 36c96fa893d632ab3d1ed42d77e75cfc24587f62 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 2 Aug 2021 18:42:00 -0400 Subject: [PATCH 13/33] Update config --- .pre-commit-config.yaml | 5 ---- local/configuration.yaml | 35 ++++++++++++++------------- src/dsdk/{interconnect.py => epic.py} | 22 ++++++++--------- 3 files changed, 29 insertions(+), 33 deletions(-) rename src/dsdk/{interconnect.py => epic.py} (95%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6dad9ff..a7dacb6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,16 +6,12 @@ repos: - id: check-case-conflict - id: check-docstring-first - id: check-executables-have-shebangs - - id: check-json - id: check-merge-conflict - id: check-toml - id: check-symlinks - - id: check-xml - - id: check-yaml - id: end-of-file-fixer - id: fix-encoding-pragma - id: mixed-line-ending - - id: requirements-txt-fixer - id: trailing-whitespace - repo: http://github.com/pre-commit/pygrep-hooks rev: v1.9.0 @@ -25,7 +21,6 @@ repos: - id: python-no-eval - id: python-no-log-warn - id: python-use-type-annotations - - id: rst-backticks - repo: https://github.com/pycqa/isort rev: 5.8.0 hooks: diff --git a/local/configuration.yaml b/local/configuration.yaml index 5e5b5e8..7610cdb 100644 --- a/local/configuration.yaml +++ b/local/configuration.yaml @@ -1,20 +1,21 @@ -interconnect: !interconnect - authorization: ${INTERCONNECT_AUTHORIZATION} - comment: "Not for clinical use." - cookie: ${INTERCONNECT_COOKIE} - contact_type_id: csn - flowsheet_id: ${INTERCONNECT_FLOWSHEET_ID} - flowsheet_id_type: internal - flowsheet_template_id: 3040005300 - flowsheet_template_id_type: internal - lookback_hours: 72 - patient_id_type: uid - notification: !notification - uri: ${INTERCONNECT_HOST}${INTERCONNECT_NOTIFICATION_URI} - user_id: pennsignals - user_id_type: external - verification: !verification - uri: ${INTERCONNECT_HOST}${INTERCONNECT_VERIFICATION_URI} +notification: + epic: !epic &epic + authorization: ${INTERCONNECT_AUTHORIZATION} + comment: "Not for clinical use." + cookie: ${INTERCONNECT_COOKIE} + contact_type_id: csn + flowsheet_id: ${INTERCONNECT_FLOWSHEET_ID} + flowsheet_id_type: internal + flowsheet_template_id: 3040005300 + flowsheet_template_id_type: internal + lookback_hours: 72 + patient_id_type: uid + user_id: pennsignals + user_id_type: external + uri: ${INTERCONNECT_HOST}${INTERCONNECT_NOTIFICATION_URI} +verification: !verification + epic: *epic + uri: ${INTERCONNECT_HOST}${INTERCONNECT_VERIFICATION_URI} postgres: !postgres database: ${POSTGRES_DATABASE} host: ${POSTRES_HOST} diff --git a/src/dsdk/interconnect.py b/src/dsdk/epic.py similarity index 95% rename from src/dsdk/interconnect.py rename to src/dsdk/epic.py index c3584c5..2a0eb09 100644 --- a/src/dsdk/interconnect.py +++ b/src/dsdk/epic.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Interconnect.""" +"""Epic.""" from logging import getLogger from select import select @@ -13,10 +13,10 @@ logger = getLogger(__name__) -class Interconnect: - """Interconnect.""" +class Epic: + """Epic.""" - YAML = "!interconnect" + YAML = "!epic" @classmethod def as_yaml_type(cls, *, tag: Optional[str] = None): @@ -99,14 +99,14 @@ def _yaml_repr(cls, dumper, self, *, tag): def __init__( self, *, - interconnect: Interconnect, + epic: Epic, uri: str, flowsheet_id: str = "3040015333", flowsheet_template_id: str = "3040005300", flowsheet_template_id_type: str = "internal", ) -> None: """__init__.""" - self.interconnect = interconnect + self.epic = epic self.flowsheet_id = flowsheet_id self.flowsheet_template_id = flowsheet_template_id self.flowsheet_template_id_type = flowsheet_template_id_type @@ -181,7 +181,7 @@ def __init__( self, *, uri: str, - interconnect: Interconnect, + epic: Epic, comment: str = "Not for clinical use.", contact_type_id: str = "csn", patient_id_type: str = "uid", @@ -191,7 +191,7 @@ def __init__( """__init__.""" super().__init__( - interconnect=interconnect, + epic=epic, uri=uri + query, **kwargs, ) @@ -259,14 +259,14 @@ class Verification(FlowsheetEgress): def __init__( self, *, - interconnect: Interconnect, + epic: Epic, uri: str, flowsheet_id: str = "3040015333", **kwargs, ) -> None: """__init__.""" super().__init__( - interconnect=interconnect, + epic=epic, flowsheet_id=flowsheet_id, uri=uri, **kwargs, @@ -293,7 +293,7 @@ def call_uri(self, notification, cur): epic = self.epic sql = epic.postgres.sql # TODO add notification flowsheet ids to data? - request = Request(self.URI, data=None) + request = Request(self.uri, data=None) with urlopen(request, epic.uri_timeout) as response: if response.ok: cur.execute( From c7a4f3c0167ad270e96f982eb1f6945b943b0814 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Tue, 3 Aug 2021 09:16:47 -0400 Subject: [PATCH 14/33] Add zeep for wsdl --- setup.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9f87a7f..a6d4fec 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,8 @@ "wheel>=0.35.1", ) +EPIC_REQUIRES = ("zeep",) + PYMSSQL_REQUIRES = ("cython>=0.29.21", "pymssql>=2.1.4") PSYCOPG2_REQUIRES = ("psycopg2-binary>=2.8.6",) @@ -58,7 +60,12 @@ ] }, extras_require={ - "all": PSYCOPG2_REQUIRES + PYMSSQL_REQUIRES + TEST_REQUIRES, + "all": ( + EPIC_REQUIRES + + PSYCOPG2_REQUIRES + + PYMSSQL_REQUIRES + + TEST_REQUIRES + ), "psycopg2": PSYCOPG2_REQUIRES, "pymssql": PYMSSQL_REQUIRES, "test": TEST_REQUIRES, From e2c7243abce22f3dcd432d2dcb34357718121f1f Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 16 Aug 2021 16:41:03 -0400 Subject: [PATCH 15/33] Auth works, service not found --- setup.py | 2 +- src/dsdk/epic.py | 437 +++++++++++++++++++++++++---------------------- 2 files changed, 231 insertions(+), 208 deletions(-) diff --git a/setup.py b/setup.py index a6d4fec..eff56d5 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ ), "numpy>=1.15.4", "pandas>=0.23.4", - "pip>=21.2.1", + "pip>=21.2.2", "setuptools>=57.4.0", "wheel>=0.35.1", ) diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index 2a0eb09..3184a38 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -1,25 +1,34 @@ # -*- coding: utf-8 -*- """Epic.""" +from contextlib import contextmanager from logging import getLogger from select import select from typing import Any, Dict, Optional -from urllib.request import Request, urlopen from cfgenvy import yaml_type from .postgres import Persistor as Postgres +try: + from requests import Session + from zeep import Client + from zeep.transports import Transport +except ImportError: + Client = None + Session = None + Transport = None + logger = getLogger(__name__) -class Epic: - """Epic.""" +class Interconnect: # pylint: disable=too-many-instance-attributes + """Interconnect.""" - YAML = "!epic" + YAML = "!interconnect" @classmethod - def as_yaml_type(cls, *, tag: Optional[str] = None): + def as_yaml_type(cls, tag: Optional[str] = None): """As yaml type.""" yaml_type( cls, @@ -34,9 +43,9 @@ def _yaml_init(cls, loader, node): return cls(**loader.construct_mapping(node, deep=True)) @classmethod - def _yaml_repr(cls, dumper, self, *, tag: str): + def _yaml_repr(cls, dumper, self, *, tag): """Yaml repr.""" - return dumper.represent_mapping(tag, self.as_yaml()) + return dumper.represent_mapper(tag, self.as_yaml()) def __init__( self, @@ -44,8 +53,18 @@ def __init__( authorization: str, cookie: str, postgres: Postgres, - poll_timeout: int = 60, - uri_timeout: int = 5, + service_name: str, + urn: str, + wsdl: str, + comment: str = "Not for clinical use.", + connect_timeout: int = 5, + contact_id_type: str = "csn", + flowsheet_id: str = "3040015333", + flowsheet_template_id: str = "3040005300", + flowsheet_template_id_type: str = "internal", + lookback_hours: int = 72, + operation_timeout: int = 3, + patient_id_type: str = "uid", user_id: str = "pennsignals", user_id_type: str = "external", ) -> None: @@ -53,76 +72,63 @@ def __init__( self.authorization = authorization self.cookie = cookie self.postgres = postgres - self.poll_timeout = poll_timeout - self.uri_timeout = uri_timeout + self.wsdl = wsdl + self.comment = comment + self.contact_id_type = contact_id_type + self.connect_timeout = connect_timeout + self.flowsheet_id = flowsheet_id + self.flowsheet_template_id = flowsheet_template_id + self.flowsheet_template_id_type = flowsheet_template_id_type + self.lookback_hours = lookback_hours + self.operation_timeout = operation_timeout + self.patient_id_type = patient_id_type self.user_id = user_id self.user_id_type = user_id_type + self.urn = urn def as_yaml(self) -> Dict[str, Any]: """As yaml.""" return { "authorization": self.authorization, + "comment": self.comment, + "contact_id_type": self.contact_id_type, "cookie": self.cookie, - "poll_timeout": self.poll_timeout, + "flowsheet_id": self.flowsheet_id, + "flowsheet_template_id": self.flowsheet_template_id, + "flowsheet_template_id_type": self.flowsheet_template_id_type, + "lookback_hours": self.lookback_hours, + "patient_id_type": self.patient_id_type, "postgres": self.postgres, - "uri_timeout": self.uri_timeout, + "urn": self.urn, "user_id": self.user_id, "user_id_type": self.user_id_type, + "wsdl": self.wsdl, } - -class FlowsheetEgress: # pylint: disable=too-many-instance-attributes - """Flowsheet Egress.""" - - YAML = "!flowsheetegress" - - @classmethod - def as_yaml_type(cls, tag: Optional[str] = None): - """As yaml type.""" - yaml_type( - cls, - tag or cls.YAML, - init=cls._yaml_init, - repr=cls._yaml_repr, + @contextmanager + def interconnect(self): + """Interconnect soap client.""" + session = Session() + session.headers.update( + { + "authorization": self.authorization, + "cookie": self.cookie, + } ) + session.verify = False + transport = Transport( + session=session, + timeout=self.connect_timeout, + opration_timeout=self.operation_timeout, + ) + client = Client( + wsdl=self.wsdl, + service_name=self.service_name, + transport=transport, + ) + yield client - @classmethod - def _yaml_init(cls, loader, node): - """Yaml init.""" - return cls(**loader.construct_mapping(node, deep=True)) - - @classmethod - def _yaml_repr(cls, dumper, self, *, tag): - """Yaml repr.""" - return dumper.represent_mapper(tag, self.as_yaml()) - - def __init__( - self, - *, - epic: Epic, - uri: str, - flowsheet_id: str = "3040015333", - flowsheet_template_id: str = "3040005300", - flowsheet_template_id_type: str = "internal", - ) -> None: - """__init__.""" - self.epic = epic - self.flowsheet_id = flowsheet_id - self.flowsheet_template_id = flowsheet_template_id - self.flowsheet_template_id_type = flowsheet_template_id_type - self.uri = uri - - def as_yaml(self) -> Dict[str, Any]: - """As yaml.""" - return { - "epic": self.epic, - "flowsheet_id": self.flowsheet_id, - "flowsheet_template_id": self.flowsheet_template_id, - "flowsheet_template_id_type": self.flowsheet_template_id_type, - "uri": self.uri, - } - - def listen(self, listen): + def listen(self, listen, cur, interconnect): """Listen.""" while True: readers, _, exceptions = select( @@ -134,178 +140,195 @@ def listen(self, listen): continue if listen.poll(): while listen.notifies: - self.on_notify(listen.notifies.pop()) + yield listen.notified.pop() - def on_notify(self, event): # pylint: disable=no-self-use + def on_notify(self, event, cur, intervconnect): """On postgres notify handler.""" logger.debug( - "NOTIFY: %(pid)s.%(channel)s.%(payload)s", + "NOTIFY: %(id)s.%(channel)s.%(payload)s", { - "channel": event.chennel, + "channel": event.channel, + "id": event.id, "payload": event.payload, - "pid": event.pid, }, ) + + def soap(self, event, cur, interconnect): + """Soap call.""" raise NotImplementedError() -class Notification(FlowsheetEgress): - """Notification Service.""" - - YAML = "!notification" - - QUERY = "".join( - ( - "?", - "&".join( - ( - "PatientID=%(empi)s", - "PatientIDType=%(patient_id_type)s", - "ContactID=%(csn)s", - "ContactIDType=%(contact_id_type)s", - "UserID=%(user_id)s", - "UserIDType=%(user_type_id)s", - "FlowsheetID=%(flowsheet_id)s", - "FlowsheetIDType=%(flowsheet_id_type)s", - "Value=%(score)s", - "Comment=%(comment)s", - "InstantValueTaken=%(instant_value_taken)s", - "FlowsheetTemplateID=%(flowsheet_template_id)s", - "FlowsheetTemplateIDType=%(flowsheet_template_id_type)s", - ) - ), - ) - ) +class Notifier(Interconnect): + """Notifier.""" - def __init__( - self, - *, - uri: str, - epic: Epic, - comment: str = "Not for clinical use.", - contact_type_id: str = "csn", - patient_id_type: str = "uid", - query: str = QUERY, - **kwargs, - ) -> None: - """__init__.""" - - super().__init__( - epic=epic, - uri=uri + query, - **kwargs, - ) - self.comment = comment - self.contact_type_id = contact_type_id - self.patient_id_type = patient_id_type + YAML = "!interconnectnotifier" def __call__(self): """__call__.""" postgres = self.epic.postgres sql = postgres.sql - with postgres.listen(sql.prediction.listen) as listen: - with postgres.commit() as cur: - self.recover(cur) - self.listen(listen) - - def as_yaml(self) -> Dict[str, Any]: - """As yaml.""" - return { - "comment": self.comment, - "contact_type_id": self.contact_type_id, - "patient_id_type": self.patient_id_type, - **super().as_yaml(), - } + with self.interconnect() as interconnect: + with postgres.listen(sql.prediction.listen) as listener: + with postgres.commit() as cur: + self.recover(cur, interconnect) + for each in listener: + self.on_notify(each, cur, interconnect) + + def on_notify(self, event, cur, interconnect): + """On notify.""" + super.on_notify(event, cur, interconnect) + sql = self.postgres.sql + cur.execute(sql.prediction.recent, event.id) + for each in cur.fetchall(): + self.soap(each, cur, interconnect) - def recover(self, cur): + def recover(self, cur, interconnect): """Recover.""" sql = self.epic.postgres.sql cur.execute(sql.epic.notification.recover) for each in cur.fetchall(): - self.call_uri(each, cur) - - def call_uri(self, prediction, cur): - """Call uri.""" - epic = self.epic - sql = epic.postgres.sql - uri = self.uri % { - "csn": prediction["csn"], - "empi": prediction["empi"], - "score": prediction["score"], - } - request = Request(uri, data=None) - with urlopen(request, epic.uri_timeout) as response: - if response.ok: - cur.execute( - sql.epic.notification.insert, - {"prediction_id": prediction["id"]}, - ) - else: - cur.execute( - sql.epic.notification_error.insert, - { - "description": response.text, - "name": response.reason, - "prediction_id": prediction["id"], - }, - ) - - -class Verification(FlowsheetEgress): - """Verification Service.""" - - YAML = "!verification" - - def __init__( - self, - *, - epic: Epic, - uri: str, - flowsheet_id: str = "3040015333", - **kwargs, - ) -> None: - """__init__.""" - super().__init__( - epic=epic, - flowsheet_id=flowsheet_id, - uri=uri, - **kwargs, + self.soap(each, cur, interconnect) + + def soap(self, prediction, cur, interconnect): + """Soap.""" + sql = self.postgres.sql + print(dir(interconnect)) + response = interconnect.service.AddFlowsheetValue( + Comment=self.comment, + ContactID=prediction["csn"], + ContactIDType=self.contact_id_type, + FlowsheetID=self.flowsheet_id, + FlowsheetTemplateID=self.flowsheet_template_id, + FlowsheetTemplateTypeID=self.flowsheet_template_type_id, + FlowsheetTypeID=self.flowsheet_type_id, + InstantValueTaken=self.instant_value_taken, + PatientID=prediction["empi"], + PatientIDType=self.patient_id_type, + UserID=self.user_id, + UserIDType=self.user_id_type, + Value=prediction["score"], ) + print(response) + + ok = response.Success + if ok: + cur.execute( + sql.epic.notification.insert, + {"prediction_id": prediction["id"]}, + ) + else: + cur.execute( + sql.epic.notification_error.insert, + { + "description": response.text, + "name": response.reason, + "prediction_id": prediction["id"], + }, + ) + + +class Verifier(Interconnect): + """Verifier.""" + + YAML = "!interconnectverifier" + def __call__(self): """__call__.""" postgres = self.epic.postgres sql = postgres.sql - with postgres.listen(sql.notification.listen) as listen: - with postgres.commit() as cur: - self.recover(cur) - self.listen(listen) + with self.interconnect() as interconnect: + with postgres.listen(sql.notification.listen) as listen: + with postgres.commit() as cur: + self.recover(cur, interconnect) + self.listen(listen, cur, interconnect) + + def on_notify(self, event, cur, interconnect): + """On notify.""" + super.on_notify(event, cur, interconnect) + sql = self.postgres.sql + cur.execute(sql.notification.recent, event.id) + for each in cur.fetchall(): + self.soap(each, cur, interconnect) - def recover(self, cur): + def recover(self, cur, interconnect): """Recover.""" sql = self.epic.postgres.sql cur.execute(sql.epic.verification.recover) for each in cur.fetchall(): - self.call_uri(each, cur) - - def call_uri(self, notification, cur): - """Call uri.""" - epic = self.epic - sql = epic.postgres.sql - # TODO add notification flowsheet ids to data? - request = Request(self.uri, data=None) - with urlopen(request, epic.uri_timeout) as response: - if response.ok: - cur.execute( - sql.epic.verification.insert, - {"notification_id": notification.id}, - ) - else: - cur.execute( - sql.epic.verification_error.insert, - { - "description": response.text, - "name": response.reason, - "notification_id": notification.id, - }, - ) + self.soap(each, cur, interconnect) + + def soap(self, notification, cur, interconnect): + """Soap.""" + postgres = self.postgres + sql = postgres.sql + response = interconnect.GetFlowsheetRows( + ContactID=notification["csn"], + ContactTypeID=self.contact_type_id, + FlowsheetRowIDs=[notification["flowsheet_row_id"]], + LookBackHours=self.look_back_hours, + PatientID=notification["empi"], + PatienTypeID=self.patient_type_id, + UserID=self.user_id, + UserTypeID=self.user_type_id, + ) + + print(response) + + ok = response.Success + + if ok: + cur.execute( + sql.epic.verification.insert, + {"notification_id": notification["id"]}, + ) + else: + cur.execute( + sql.epic.verification_error.insert, + { + "description": response.text, + "name": response.reason, + "notification_id": notification["id"], + }, + ) + + +def main(empi, csn): + """Main.""" + from os import getcwd + from os.path import join as pathjoin + + from cfgenvy import Parser + + from .asset import Asset + + Asset.as_yaml_type() + Notifier.as_yaml_type() + Postgres.as_yaml_type() + parser = Parser() + cwd = getcwd() + notifier = parser.parse( + argv=( + "-c", + pathjoin(cwd, "local", "test.wsdl.yaml"), + "-e", + pathjoin(cwd, "secrets", "test.wsdl.env"), + ) + ) + + notification = { + "csn": csn, + "empi": empi, + "score": 0, + } + + with notifier.interconnect() as interconnect: + with notifier.postgres.commit() as cur: + notifier.soap(notification, cur, interconnect) + + +if __name__ == "__main__": + main( + csn="278820881", + empi="833065951", + ) From c2da5ef896577e9183ba6ccf33adb64d65ded256 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Wed, 18 Aug 2021 15:32:14 -0400 Subject: [PATCH 16/33] Remove soap --- src/dsdk/epic.py | 376 +++++++++++++++++++++++++---------------------- 1 file changed, 202 insertions(+), 174 deletions(-) diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index 3184a38..6720de7 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- """Epic.""" +from base64 import b64encode from contextlib import contextmanager +from datetime import datetime from logging import getLogger from select import select -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Tuple +from urllib.parse import quote from cfgenvy import yaml_type @@ -12,20 +15,16 @@ try: from requests import Session - from zeep import Client - from zeep.transports import Transport except ImportError: - Client = None Session = None - Transport = None logger = getLogger(__name__) -class Interconnect: # pylint: disable=too-many-instance-attributes - """Interconnect.""" +class Epic: # pylint: disable=too-many-instance-attributes + """Epic.""" - YAML = "!interconnect" + YAML = "!epic" @classmethod def as_yaml_type(cls, tag: Optional[str] = None): @@ -49,90 +48,74 @@ def _yaml_repr(cls, dumper, self, *, tag): def __init__( self, - *, - authorization: str, + client_id: str, # 00000000-0000-0000-0000-000000000000 cookie: str, - postgres: Postgres, - service_name: str, - urn: str, - wsdl: str, + password: str, + url: str, + flowsheet_id: str, comment: str = "Not for clinical use.", - connect_timeout: int = 5, - contact_id_type: str = "csn", - flowsheet_id: str = "3040015333", + contact_id_type: str = "CSN", + flowsheet_id_type: str = "internal", flowsheet_template_id: str = "3040005300", flowsheet_template_id_type: str = "internal", - lookback_hours: int = 72, - operation_timeout: int = 3, - patient_id_type: str = "uid", - user_id: str = "pennsignals", + patient_id_type: str = "UID", + username: str = "Pennsignals", user_id_type: str = "external", - ) -> None: + user_id: str = "PENNSIGNALS", + ): """__init__.""" - self.authorization = authorization - self.cookie = cookie - self.postgres = postgres - self.wsdl = wsdl + self.authorization = b"Basic " + \ + b64encode(f"EMP${username}:{password}".encode("utf-8")) + self.client_id = client_id self.comment = comment self.contact_id_type = contact_id_type - self.connect_timeout = connect_timeout + self.cookie = cookie self.flowsheet_id = flowsheet_id + self.flowsheet_id_type = flowsheet_id_type self.flowsheet_template_id = flowsheet_template_id self.flowsheet_template_id_type = flowsheet_template_id_type - self.lookback_hours = lookback_hours - self.operation_timeout = operation_timeout self.patient_id_type = patient_id_type + self.url = url self.user_id = user_id self.user_id_type = user_id_type - self.urn = urn + + def __call__(self): + """__call__.""" + with self.session() as session: + with self.listener() as listener: + with self.postgres.commit() as cur: + self.recover(cur, session) + for each in listener: + self.on_notify(each, cur, session) def as_yaml(self) -> Dict[str, Any]: """As yaml.""" return { "authorization": self.authorization, + "client_id": self.client_id, "comment": self.comment, "contact_id_type": self.contact_id_type, "cookie": self.cookie, "flowsheet_id": self.flowsheet_id, + "flowsheet_id_type": self.flowsheet_id_type, "flowsheet_template_id": self.flowsheet_template_id, "flowsheet_template_id_type": self.flowsheet_template_id_type, - "lookback_hours": self.lookback_hours, "patient_id_type": self.patient_id_type, - "postgres": self.postgres, - "urn": self.urn, + "url": self.url, "user_id": self.user_id, "user_id_type": self.user_id_type, - "wsdl": self.wsdl, } @contextmanager - def interconnect(self): - """Interconnect soap client.""" - session = Session() - session.headers.update( - { - "authorization": self.authorization, - "cookie": self.cookie, - } - ) - session.verify = False - transport = Transport( - session=session, - timeout=self.connect_timeout, - opration_timeout=self.operation_timeout, - ) - client = Client( - wsdl=self.wsdl, - service_name=self.service_name, - transport=transport, - ) - yield client + def listener(self): + """Listener.""" + raise NotImplementedError() - def listen(self, listen, cur, interconnect): + def listen(self, listen, cur, session): """Listen.""" while True: readers, _, exceptions = select( - [listen], [], [listen], self.epic.poll_timeout + [listen], [], [listen], self.poll_timeout ) if exceptions: break @@ -142,7 +125,7 @@ def listen(self, listen, cur, interconnect): while listen.notifies: yield listen.notified.pop() - def on_notify(self, event, cur, intervconnect): + def on_notify(self, event, cur, session): """On postgres notify handler.""" logger.debug( "NOTIFY: %(id)s.%(channel)s.%(payload)s", @@ -153,148 +136,189 @@ def on_notify(self, event, cur, intervconnect): }, ) - def soap(self, event, cur, interconnect): - """Soap call.""" + def on_success(self, entity, cur, message): + """On success.""" raise NotImplementedError() + def on_error(self, entity, cur, message): + """On error.""" + raise NotImplementedError() + + @contextmanager + def session(self): + """Session.""" + session = Session() + session.verify = False + session.headers.update({ + "Authorization": self.authorization, + "Cookie": self.cookie, + "Epic-Client-ID": self.client_id, + "Epic-User-ID": self.user_id, + "Epic-User-IDType": self.user_id_type, + }) + yield session + -class Notifier(Interconnect): +class Notifier(Epic): """Notifier.""" - YAML = "!interconnectnotifier" + YAML = "!epicnotifier" - def __call__(self): - """__call__.""" - postgres = self.epic.postgres + @classmethod + def as_yaml_type(cls, tag: Optional[str] = None): + """As yaml type.""" + yaml_type( + cls, + tag or cls.YAML, + init=cls._yaml_init, + repr=cls._yaml_repr, + ) + + @classmethod + def _yaml_init(cls, loader, node): + """Yaml init.""" + return cls(**loader.construct_mapping(node, deep=True)) + + @classmethod + def _yaml_repr(cls, dumper, self, *, tag): + """Yaml repr.""" + return dumper.represent_mapper(tag, self.as_yaml()) + + @contextmanager + def listener(self): + """Listener.""" + postgres = self.postgres sql = postgres.sql - with self.interconnect() as interconnect: - with postgres.listen(sql.prediction.listen) as listener: - with postgres.commit() as cur: - self.recover(cur, interconnect) - for each in listener: - self.on_notify(each, cur, interconnect) + with postgres.listen(sql.prediction.listen) as listener: + yield listener - def on_notify(self, event, cur, interconnect): + def on_notify(self, event, cur, session): """On notify.""" - super.on_notify(event, cur, interconnect) + super.on_notify(event, cur, session) sql = self.postgres.sql cur.execute(sql.prediction.recent, event.id) for each in cur.fetchall(): - self.soap(each, cur, interconnect) + ok, message = self.rest(each, session) + if ok: + self.on_success(each, cur, message) + else: + self.on_error(each, cur, message) - def recover(self, cur, interconnect): + def recover(self, cur, session): """Recover.""" sql = self.epic.postgres.sql cur.execute(sql.epic.notification.recover) for each in cur.fetchall(): - self.soap(each, cur, interconnect) - - def soap(self, prediction, cur, interconnect): - """Soap.""" - sql = self.postgres.sql - print(dir(interconnect)) - response = interconnect.service.AddFlowsheetValue( - Comment=self.comment, - ContactID=prediction["csn"], - ContactIDType=self.contact_id_type, - FlowsheetID=self.flowsheet_id, - FlowsheetTemplateID=self.flowsheet_template_id, - FlowsheetTemplateTypeID=self.flowsheet_template_type_id, - FlowsheetTypeID=self.flowsheet_type_id, - InstantValueTaken=self.instant_value_taken, - PatientID=prediction["empi"], - PatientIDType=self.patient_id_type, - UserID=self.user_id, - UserIDType=self.user_id_type, - Value=prediction["score"], + ok, message = self.rest(each, session) + if ok: + self.on_success(each, cur, message) + else: + self.on_error(each, cur, message) + + def rest(self, prediction, session) -> Tuple[bool, Any]: + """Rest.""" + query = { + "comment": self.comment, + "contact_id_type": self.contact_id_type, + "flowsheet_id": self.flowsheet_id, + "flowsheet_id_type": self.flowsheet_id_type, + "flowsheet_template_id": self.flowsheet_template_id, + "flowsheet_template_id_type": self.flowsheet_template_id_type, + "patient_id_type": self.patient_id_type, + "user_id": self.user_id, + "user_id_type": self.user_id_type, + } + query.update({ + "contact_id": prediction["csn"], + "instant_value_taken": prediction["create_on"].strftime( + "%Y-%m-%dT%H:%M:%SZ"), + "patient_id": prediction["empi"], + "value": prediction["score"], + }) + + url = self.url.format(**{ + key: quote(value.encode('utf-8')) + for key, value in query.items()}) + logger.info(url) + + response = session.post( + url=url, + data={}, ) - print(response) - - ok = response.Success - if ok: - cur.execute( - sql.epic.notification.insert, - {"prediction_id": prediction["id"]}, - ) - else: - cur.execute( - sql.epic.notification_error.insert, - { - "description": response.text, - "name": response.reason, - "prediction_id": prediction["id"], - }, - ) + return response.status_code == 200, response -class Verifier(Interconnect): +class Verifier(Epic): """Verifier.""" - YAML = "!interconnectverifier" + YAML = "!epicverifier" - def __call__(self): - """__call__.""" - postgres = self.epic.postgres + @contextmanager + def listener(self): + """Listener.""" + postgres = self.postgres sql = postgres.sql - with self.interconnect() as interconnect: - with postgres.listen(sql.notification.listen) as listen: - with postgres.commit() as cur: - self.recover(cur, interconnect) - self.listen(listen, cur, interconnect) + with postgres.listen(sql.notification.listen) as listener: + yield listener - def on_notify(self, event, cur, interconnect): + def on_notify(self, event, cur, session): """On notify.""" - super.on_notify(event, cur, interconnect) + super.on_notify(event, cur, session) sql = self.postgres.sql cur.execute(sql.notification.recent, event.id) for each in cur.fetchall(): - self.soap(each, cur, interconnect) - - def recover(self, cur, interconnect): - """Recover.""" - sql = self.epic.postgres.sql - cur.execute(sql.epic.verification.recover) - for each in cur.fetchall(): - self.soap(each, cur, interconnect) - - def soap(self, notification, cur, interconnect): - """Soap.""" - postgres = self.postgres - sql = postgres.sql - response = interconnect.GetFlowsheetRows( - ContactID=notification["csn"], - ContactTypeID=self.contact_type_id, - FlowsheetRowIDs=[notification["flowsheet_row_id"]], - LookBackHours=self.look_back_hours, - PatientID=notification["empi"], - PatienTypeID=self.patient_type_id, - UserID=self.user_id, - UserTypeID=self.user_type_id, + ok, response = self.rest(each, session) + if ok: + self.on_success(each, session, response) + else: + self.on_error(each, session, response) + + def on_success(self, notification, cur, response): + """On sucess.""" + cur.execute( + self.postgres.sql.epic.verification.insert, + {"prediction_id": notification["id"]}, ) - print(response) + def on_error(self, notification, cur, response): + """On error.""" + cur.execute( + self.postgres.sql.epic.verification_error.insert, + { + "description": response.text, + "name": response.reason, + "prediction_id": notification["id"], + }, + ) - ok = response.Success + def rest(self, notification, session) -> Tuple[bool, Any]: + """Rest.""" + data = { + "ContactIDType": self.contact_id_type, + "LookbackHours": self.lookback_hours, + "PatientIDType": self.patient_id_type, + "UserID": self.user_id, + "UserIDType": self.user_id_type, + + data.update({ + "ContactID": notification["csn"], + "FlowsheetRowIDs": [{ + "ID": self.flowsheet_id, + "IDType": self.flowsheet_id_type, + }], + "PatientID": notification["empi"], + }) + response = session.post( + url=self.url, + data=data, + ) - if ok: - cur.execute( - sql.epic.verification.insert, - {"notification_id": notification["id"]}, - ) - else: - cur.execute( - sql.epic.verification_error.insert, - { - "description": response.text, - "name": response.reason, - "notification_id": notification["id"], - }, - ) + return response.status_code == 200, response -def main(empi, csn): - """Main.""" +def test_notifier(csn, empi, score): + """Rest.""" from os import getcwd from os.path import join as pathjoin @@ -303,32 +327,36 @@ def main(empi, csn): from .asset import Asset Asset.as_yaml_type() - Notifier.as_yaml_type() Postgres.as_yaml_type() + Notifier.as_yaml_type() parser = Parser() cwd = getcwd() + notifier = parser.parse( argv=( "-c", - pathjoin(cwd, "local", "test.wsdl.yaml"), + pathjoin(cwd, "local", "test.notifier.yaml"), "-e", - pathjoin(cwd, "secrets", "test.wsdl.env"), + pathjoin(cwd, "secrets", "test.notifier.env"), ) ) - notification = { + prediction = { + "create_on": datetime.utcnow(), "csn": csn, "empi": empi, - "score": 0, + "score": score, } - with notifier.interconnect() as interconnect: - with notifier.postgres.commit() as cur: - notifier.soap(notification, cur, interconnect) + with notifier.session() as session: + ok, response = notifier.rest(prediction, session) + print(ok) + print(response) if __name__ == "__main__": - main( + test_notifier( csn="278820881", - empi="833065951", + empi="8330651951", + score="0.5", ) From f1ea25e7bdf80735b3598a8a96ef475bc996408f Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Wed, 18 Aug 2021 17:07:43 -0400 Subject: [PATCH 17/33] Blacken code --- pyproject.toml | 2 +- setup.py | 3 +- src/dsdk/epic.py | 174 +++++++++++++++++++++++++++++++++++++---------- 3 files changed, 140 insertions(+), 39 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d24d4b0..034dd1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ known_first_party = ["dsdk"] default_section = "THIRDPARTY" [tool.pylint.basic] -good-names = '''a,b,c,d,df,e,i,id,logger,n,on,tz''' +good-names = '''a,b,c,d,df,e,i,id,logger,n,on,ok,tz''' [tool.pylint.message_control] disable = '''duplicate-code,C0330''' diff --git a/setup.py b/setup.py index eff56d5..a64bfe5 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ "wheel>=0.35.1", ) -EPIC_REQUIRES = ("zeep",) +EPIC_REQUIRES = ("requests",) PYMSSQL_REQUIRES = ("cython>=0.29.21", "pymssql>=2.1.4") @@ -50,6 +50,7 @@ "types-python-dateutil", "types-pymssql", "types-pyyaml", + "types-requests", ) setup( diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index 6720de7..4a1ceeb 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -46,7 +46,7 @@ def _yaml_repr(cls, dumper, self, *, tag): """Yaml repr.""" return dumper.represent_mapper(tag, self.as_yaml()) - def __init__( + def __init__( # pylint: disable=too-many-arguments,too-many-locals self, client_id: str, # 00000000-0000-0000-0000-000000000000 cookie: str, @@ -58,14 +58,18 @@ def __init__( flowsheet_id_type: str = "internal", flowsheet_template_id: str = "3040005300", flowsheet_template_id_type: str = "internal", + lookback_hours: int = 72, patient_id_type: str = "UID", + poll_timeout: int = 300, username: str = "Pennsignals", user_id_type: str = "external", user_id: str = "PENNSIGNALS", ): """__init__.""" - self.authorization = b"Basic " + \ - b64encode(f"EMP${username}:{password}".encode("utf-8")) + self.authorization = b"Basic " + b64encode( + f"EMP${username}:{password}".encode("utf-8") + ) + self.postgres = None self.client_id = client_id self.comment = comment self.contact_id_type = contact_id_type @@ -74,7 +78,9 @@ def __init__( self.flowsheet_id_type = flowsheet_id_type self.flowsheet_template_id = flowsheet_template_id self.flowsheet_template_id_type = flowsheet_template_id_type + self.lookback_hours = lookback_hours self.patient_id_type = patient_id_type + self.poll_timeout = poll_timeout self.url = url self.user_id = user_id self.user_id_type = user_id_type @@ -100,7 +106,9 @@ def as_yaml(self) -> Dict[str, Any]: "flowsheet_id_type": self.flowsheet_id_type, "flowsheet_template_id": self.flowsheet_template_id, "flowsheet_template_id_type": self.flowsheet_template_id_type, + "lookback_hours": self.lookback_hours, "patient_id_type": self.patient_id_type, + "poll_timeout": self.poll_timeout, "url": self.url, "user_id": self.user_id, "user_id_type": self.user_id_type, @@ -125,7 +133,12 @@ def listen(self, listen, cur, session): while listen.notifies: yield listen.notified.pop() - def on_notify(self, event, cur, session): + def on_notify( # pylint: disable: unused-argument,no-self-use + self, + event, + cur, + session, + ): """On postgres notify handler.""" logger.debug( "NOTIFY: %(id)s.%(channel)s.%(payload)s", @@ -136,26 +149,43 @@ def on_notify(self, event, cur, session): }, ) - def on_success(self, entity, cur, message): + def on_success(self, entity, cur, response): """On success.""" raise NotImplementedError() - def on_error(self, entity, cur, message): + def on_error(self, entity, cur, response): """On error.""" raise NotImplementedError() + def recover(self, cur, session): + """Recover.""" + sql = self.postgres.sql + cur.execute(sql.epic.prediction.recover) + for each in cur.fetchall(): + ok, message = self.rest(each, session) + if ok: + self.on_success(each, cur, message) + else: + self.on_error(each, cur, message) + + def rest(self, entity, session): + """Rest.""" + raise NotImplementedError() + @contextmanager def session(self): """Session.""" session = Session() session.verify = False - session.headers.update({ - "Authorization": self.authorization, - "Cookie": self.cookie, - "Epic-Client-ID": self.client_id, - "Epic-User-ID": self.user_id, - "Epic-User-IDType": self.user_id_type, - }) + session.headers.update( + { + "Authorization": self.authorization, + "Cookie": self.cookie, + "Epic-Client-ID": self.client_id, + "Epic-User-ID": self.user_id, + "Epic-User-IDType": self.user_id_type, + } + ) yield session @@ -204,9 +234,27 @@ def on_notify(self, event, cur, session): else: self.on_error(each, cur, message) + def on_success(self, notification, cur, response): + """On success.""" + cur.execute( + self.postgres.sql.epic.notification.insert, + {"prediction_id": notification["id"]}, + ) + + def on_error(self, notification, cur, response): + """On error.""" + cur.execute( + self.postgres.sql.epic.notification_error.insert, + { + "description": response.text, + "name": response.reason, + "prediction_id": notification["id"], + }, + ) + def recover(self, cur, session): """Recover.""" - sql = self.epic.postgres.sql + sql = self.postgres.sql cur.execute(sql.epic.notification.recover) for each in cur.fetchall(): ok, message = self.rest(each, session) @@ -228,18 +276,25 @@ def rest(self, prediction, session) -> Tuple[bool, Any]: "user_id": self.user_id, "user_id_type": self.user_id_type, } - query.update({ - "contact_id": prediction["csn"], - "instant_value_taken": prediction["create_on"].strftime( - "%Y-%m-%dT%H:%M:%SZ"), - "patient_id": prediction["empi"], - "value": prediction["score"], - }) - - url = self.url.format(**{ - key: quote(value.encode('utf-8')) - for key, value in query.items()}) - logger.info(url) + query.update( + { + "contact_id": prediction["csn"], + "instant_value_taken": prediction["create_on"].strftime( + "%Y-%m-%dT%H:%M:%SZ" + ), + "patient_id": prediction["empi"], + "value": prediction["score"], + } + ) + + url = self.url.format( + **{ + key: quote(value.encode("utf-8")) + for key, value in query.items() + } + ) + + print(url) response = session.post( url=url, @@ -275,7 +330,7 @@ def on_notify(self, event, cur, session): self.on_error(each, session, response) def on_success(self, notification, cur, response): - """On sucess.""" + """On success.""" cur.execute( self.postgres.sql.epic.verification.insert, {"prediction_id": notification["id"]}, @@ -300,15 +355,23 @@ def rest(self, notification, session) -> Tuple[bool, Any]: "PatientIDType": self.patient_id_type, "UserID": self.user_id, "UserIDType": self.user_id_type, + } + data.update( + **{ + "ContactID": notification["csn"], + "FlowsheetRowIDs": [ + { + "ID": self.flowsheet_id, + "Type": self.flowsheet_id_type, + } + ], + "PatientID": notification["empi"], + } + ) + + print(self.url) + print(data) - data.update({ - "ContactID": notification["csn"], - "FlowsheetRowIDs": [{ - "ID": self.flowsheet_id, - "IDType": self.flowsheet_id_type, - }], - "PatientID": notification["empi"], - }) response = session.post( url=self.url, data=data, @@ -351,11 +414,48 @@ def test_notifier(csn, empi, score): with notifier.session() as session: ok, response = notifier.rest(prediction, session) print(ok) - print(response) + print(response.text) + + +def test_verifier(csn, empi, score): + """Test verifier.""" + from os import getcwd + from os.path import join as pathjoin + + from cfgenvy import Parser + + from .asset import Asset + + Asset.as_yaml_type() + Postgres.as_yaml_type() + Verifier.as_yaml_type() + parser = Parser() + cwd = getcwd() + + verifier = parser.parse( + argv=( + "-c", + pathjoin(cwd, "local", "test.verifier.yaml"), + "-e", + pathjoin(cwd, "secrets", "test.verifier.env"), + ) + ) + + notification = { + "create_on": datetime.utcnow(), + "csn": csn, + "empi": empi, + "score": score, + } + + with verifier.session() as session: + ok, response = verifier.rest(notification, session) + print(ok) + print(response.text) if __name__ == "__main__": - test_notifier( + test_verifier( csn="278820881", empi="8330651951", score="0.5", From 28a14540654851d2d882a28db79357691dfab5e5 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Thu, 19 Aug 2021 12:08:28 -0400 Subject: [PATCH 18/33] Fix json payload for verifier --- assets/postgres/epic/errors/insert.sql | 10 --- .../epic/notifications/errors/insert.sql | 10 +++ assets/postgres/epic/notifications/insert.sql | 2 +- .../postgres/epic/notifications/recover.sql | 23 +++++ .../epic/verifications/errors/insert.sql | 10 +++ assets/postgres/epic/verifications/insert.sql | 6 ++ assets/postgres/predictions/recover.sql | 6 +- src/dsdk/epic.py | 84 ++++++++----------- 8 files changed, 89 insertions(+), 62 deletions(-) delete mode 100644 assets/postgres/epic/errors/insert.sql create mode 100644 assets/postgres/epic/notifications/errors/insert.sql create mode 100644 assets/postgres/epic/notifications/recover.sql create mode 100644 assets/postgres/epic/verifications/errors/insert.sql create mode 100644 assets/postgres/epic/verifications/insert.sql diff --git a/assets/postgres/epic/errors/insert.sql b/assets/postgres/epic/errors/insert.sql deleted file mode 100644 index 3ab0ec9..0000000 --- a/assets/postgres/epic/errors/insert.sql +++ /dev/null @@ -1,10 +0,0 @@ -insert into epic_errors ( - prediction_id, - error_name, - error_description -) -select - %(prediction_id)s, - %(error_name)s, - %(error_description)s -returning * diff --git a/assets/postgres/epic/notifications/errors/insert.sql b/assets/postgres/epic/notifications/errors/insert.sql new file mode 100644 index 0000000..902277f --- /dev/null +++ b/assets/postgres/epic/notifications/errors/insert.sql @@ -0,0 +1,10 @@ +insert into epic_notification_errors ( + prediction_id, + name, + description +) +select + %(prediction_id)s, + %(name)s, + %(description)s +returning * diff --git a/assets/postgres/epic/notifications/insert.sql b/assets/postgres/epic/notifications/insert.sql index fad6279..6768220 100644 --- a/assets/postgres/epic/notifications/insert.sql +++ b/assets/postgres/epic/notifications/insert.sql @@ -2,5 +2,5 @@ insert into epic_notifications ( prediction_id ) select - %s + %(prediction_id)s returning * diff --git a/assets/postgres/epic/notifications/recover.sql b/assets/postgres/epic/notifications/recover.sql new file mode 100644 index 0000000..2d79207 --- /dev/null +++ b/assets/postgres/epic/notifications/recover.sql @@ -0,0 +1,23 @@ +select + n.id, + p.id as prediction_id, + p.csn, + p.empi, + p.score, + r.as_of +from + runs as r + join predictions as p on + p.run_id = r.id + and upper(r.interval) != 'infinity' + join epic_notifications as n on + n.prediction_id = p.id + left join epic_verifications as v on + v.notification_id = n.id + left join epic_verification_errors as e on + e.notification_id = n.id +where + v.id is null + and e.id is null +order by + p.id, n.id diff --git a/assets/postgres/epic/verifications/errors/insert.sql b/assets/postgres/epic/verifications/errors/insert.sql new file mode 100644 index 0000000..6251b7a --- /dev/null +++ b/assets/postgres/epic/verifications/errors/insert.sql @@ -0,0 +1,10 @@ +insert into epic_verification_errors ( + notification_id, + name, + description, +) +select + %(notification_id)s, + %(name)s, + %(description)s +returning * diff --git a/assets/postgres/epic/verifications/insert.sql b/assets/postgres/epic/verifications/insert.sql new file mode 100644 index 0000000..eae7e48 --- /dev/null +++ b/assets/postgres/epic/verifications/insert.sql @@ -0,0 +1,6 @@ +insert into epic_verifications ( + notification_id +) +select + %(notification_id)s +returning * diff --git a/assets/postgres/predictions/recover.sql b/assets/postgres/predictions/recover.sql index a52e5d3..5eedec6 100644 --- a/assets/postgres/predictions/recover.sql +++ b/assets/postgres/predictions/recover.sql @@ -6,14 +6,14 @@ select p.score, r.as_of from - runs as r on + runs as r join predictions as p on p.run_id = r.id and upper(r.interval) != 'infinity' left join epic_notifications as n on n.prediction_id = p.prediction_id - left join epic_errors as e on - e.prediction_id = e.prediction_id + left join epic_notification_errors as e on + e.prediction_id = p.prediction_id and e.acknowledged_on is null where n.id is null diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index 4a1ceeb..100ac74 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -55,9 +55,9 @@ def __init__( # pylint: disable=too-many-arguments,too-many-locals flowsheet_id: str, comment: str = "Not for clinical use.", contact_id_type: str = "CSN", - flowsheet_id_type: str = "internal", + flowsheet_id_type: str = "external", flowsheet_template_id: str = "3040005300", - flowsheet_template_id_type: str = "internal", + flowsheet_template_id_type: str = "external", lookback_hours: int = 72, patient_id_type: str = "UID", poll_timeout: int = 300, @@ -180,6 +180,7 @@ def session(self): session.headers.update( { "Authorization": self.authorization, + "Content-Type": "application/json", "Cookie": self.cookie, "Epic-Client-ID": self.client_id, "Epic-User-ID": self.user_id, @@ -266,26 +267,22 @@ def recover(self, cur, session): def rest(self, prediction, session) -> Tuple[bool, Any]: """Rest.""" query = { - "comment": self.comment, - "contact_id_type": self.contact_id_type, - "flowsheet_id": self.flowsheet_id, - "flowsheet_id_type": self.flowsheet_id_type, - "flowsheet_template_id": self.flowsheet_template_id, - "flowsheet_template_id_type": self.flowsheet_template_id_type, - "patient_id_type": self.patient_id_type, - "user_id": self.user_id, - "user_id_type": self.user_id_type, + "Comment": self.comment, + "ContactID": prediction["csn"], + "ContactIDType": self.contact_id_type, + "FlowsheetID": self.flowsheet_id, + "FlowsheetIDType": self.flowsheet_id_type, + "FlowsheetTemplateID": self.flowsheet_template_id, + "FlowsheetTemplateIDType": self.flowsheet_template_id_type, + "InstantValueTaken": prediction["as_of"].strftime( + "%Y-%m-%dT%H:%M:%SZ" + ), + "PatientID": prediction["empi"], + "PatientIDType": self.patient_id_type, + "UserID": self.user_id, + "UserIDType": self.user_id_type, + "Value": prediction["score"], } - query.update( - { - "contact_id": prediction["csn"], - "instant_value_taken": prediction["create_on"].strftime( - "%Y-%m-%dT%H:%M:%SZ" - ), - "patient_id": prediction["empi"], - "value": prediction["score"], - } - ) url = self.url.format( **{ @@ -332,51 +329,42 @@ def on_notify(self, event, cur, session): def on_success(self, notification, cur, response): """On success.""" cur.execute( - self.postgres.sql.epic.verification.insert, + self.postgres.sql.epic.verifications.insert, {"prediction_id": notification["id"]}, ) def on_error(self, notification, cur, response): """On error.""" cur.execute( - self.postgres.sql.epic.verification_error.insert, + self.postgres.sql.epic.verifications.errors.insert, { "description": response.text, "name": response.reason, - "prediction_id": notification["id"], + "notification_id": notification["id"], }, ) def rest(self, notification, session) -> Tuple[bool, Any]: """Rest.""" - data = { + json = { + "ContactID": notification["csn"], "ContactIDType": self.contact_id_type, + "FlowsheetRowIDs": [ + { + "ID": self.flowsheet_id, + "IDType": self.flowsheet_id_type, + } + ], "LookbackHours": self.lookback_hours, + "PatientID": notification["empi"], "PatientIDType": self.patient_id_type, "UserID": self.user_id, "UserIDType": self.user_id_type, } - data.update( - **{ - "ContactID": notification["csn"], - "FlowsheetRowIDs": [ - { - "ID": self.flowsheet_id, - "Type": self.flowsheet_id_type, - } - ], - "PatientID": notification["empi"], - } - ) - - print(self.url) - print(data) - response = session.post( url=self.url, - data=data, + json=json, ) - return response.status_code == 200, response @@ -405,7 +393,7 @@ def test_notifier(csn, empi, score): ) prediction = { - "create_on": datetime.utcnow(), + "as_of": datetime.utcnow(), "csn": csn, "empi": empi, "score": score, @@ -414,7 +402,7 @@ def test_notifier(csn, empi, score): with notifier.session() as session: ok, response = notifier.rest(prediction, session) print(ok) - print(response.text) + print(response.json()) def test_verifier(csn, empi, score): @@ -442,7 +430,7 @@ def test_verifier(csn, empi, score): ) notification = { - "create_on": datetime.utcnow(), + "as_of": datetime.utcnow(), "csn": csn, "empi": empi, "score": score, @@ -451,11 +439,11 @@ def test_verifier(csn, empi, score): with verifier.session() as session: ok, response = verifier.rest(notification, session) print(ok) - print(response.text) + print(response.json()) if __name__ == "__main__": - test_verifier( + test_notifier( csn="278820881", empi="8330651951", score="0.5", From 4e820ee752ae22a385775f92ac4f24791bdc47b5 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Thu, 19 Aug 2021 16:54:37 -0400 Subject: [PATCH 19/33] Blacken code --- assets/postgres/epic/notifications/recent.sql | 31 ++ .../postgres/epic/notifications/recover.sql | 1 + assets/postgres/predictions/recent.sql | 28 ++ assets/postgres/predictions/recover.sql | 2 +- postgres/sql/patchdb.d/007.epic.sql | 2 +- setup.py | 16 +- src/dsdk/epic.py | 357 +++++++++--------- src/dsdk/persistor.py | 1 + 8 files changed, 253 insertions(+), 185 deletions(-) create mode 100644 assets/postgres/epic/notifications/recent.sql create mode 100644 assets/postgres/predictions/recent.sql diff --git a/assets/postgres/epic/notifications/recent.sql b/assets/postgres/epic/notifications/recent.sql new file mode 100644 index 0000000..2d856cf --- /dev/null +++ b/assets/postgres/epic/notifications/recent.sql @@ -0,0 +1,31 @@ +with args as ( + select + cast(%(notification_id)s as int) as notification_id +) +select + n.id + p.id as prediction_id, + p.run_id, + p.csn, + p.empi, + p.score, + r.as_of +from + args + join runs as r + upper(r.interval) != 'infinity' + join predictions as p on + p.run_id = r.id + join epic_notifications as n on + n.prediction_id = p.id + and n.id <= args.notification_id + left join epic_verifcations as v on + v.notification_id = n.id + left join epic_verification_errors as e on + e.notification_id = n.id + and e.acknowledged_on is null +where + v.id is null + and e.id is null +order by + r.id, p.id diff --git a/assets/postgres/epic/notifications/recover.sql b/assets/postgres/epic/notifications/recover.sql index 2d79207..965a810 100644 --- a/assets/postgres/epic/notifications/recover.sql +++ b/assets/postgres/epic/notifications/recover.sql @@ -16,6 +16,7 @@ from v.notification_id = n.id left join epic_verification_errors as e on e.notification_id = n.id + and e.acknowledged_on is null where v.id is null and e.id is null diff --git a/assets/postgres/predictions/recent.sql b/assets/postgres/predictions/recent.sql new file mode 100644 index 0000000..3597bbc --- /dev/null +++ b/assets/postgres/predictions/recent.sql @@ -0,0 +1,28 @@ +with args as ( + select + cast(%(prediction_id)s as int) as prediction_id +) +select + p.id, + p.run_id, + p.csn, + p.empi, + p.score, + r.as_of +from + args + join runs as r + upper(r.interval) != 'infinity' + join predictions as p on + p.id <= args.prediction_id + and p.run_id = r.id + left join epic_notifications as n on + n.prediction_id = p.id + left join epic_notification_errors as e on + e.prediction_id = p.id + and e.acknowledged_on is null +where + n.id is null + and e.id is null +order by + r.id, p.id diff --git a/assets/postgres/predictions/recover.sql b/assets/postgres/predictions/recover.sql index 5eedec6..fbbbe29 100644 --- a/assets/postgres/predictions/recover.sql +++ b/assets/postgres/predictions/recover.sql @@ -19,4 +19,4 @@ where n.id is null and e.id is null order by - p.id + r.id, p.id diff --git a/postgres/sql/patchdb.d/007.epic.sql b/postgres/sql/patchdb.d/007.epic.sql index 5fe215a..f62d2c1 100644 --- a/postgres/sql/patchdb.d/007.epic.sql +++ b/postgres/sql/patchdb.d/007.epic.sql @@ -48,7 +48,7 @@ begin on delete cascade on update cascade ); - create trigger epci_verifications_inserted after insert on epic_verifications + create trigger epic_verifications_inserted after insert on epic_verifications referencing new table as inserted for each statement execute procedure call_notify(); diff --git a/setup.py b/setup.py index a64bfe5..d017932 100644 --- a/setup.py +++ b/setup.py @@ -12,12 +12,11 @@ "numpy>=1.15.4", "pandas>=0.23.4", "pip>=21.2.2", + "requests>=2.26.0", "setuptools>=57.4.0", "wheel>=0.35.1", ) -EPIC_REQUIRES = ("requests",) - PYMSSQL_REQUIRES = ("cython>=0.29.21", "pymssql>=2.1.4") PSYCOPG2_REQUIRES = ("psycopg2-binary>=2.8.6",) @@ -56,17 +55,14 @@ setup( entry_points={ "console_scripts": [ - "epic-notify = dsdk.epic:Notify.main", - "epic-validate = dsdk.epic:Validate.main", + "epic-notify = dsdk.epic:Notifier.main", + "epic-validate = dsdk.epic:Validator.main", + "epic-notify-test = dsdk.epic:Notifier.test", + "epic-validate-test = dsdk.epic:Notifier.test", ] }, extras_require={ - "all": ( - EPIC_REQUIRES - + PSYCOPG2_REQUIRES - + PYMSSQL_REQUIRES - + TEST_REQUIRES - ), + "all": (PSYCOPG2_REQUIRES + PYMSSQL_REQUIRES + TEST_REQUIRES), "psycopg2": PSYCOPG2_REQUIRES, "pymssql": PYMSSQL_REQUIRES, "test": TEST_REQUIRES, diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index 100ac74..eaa9682 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -5,19 +5,19 @@ from contextlib import contextmanager from datetime import datetime from logging import getLogger -from select import select -from typing import Any, Dict, Optional, Tuple +from os import getcwd +from os.path import join as pathjoin +from select import select # pylint: disable=no-name-in-module +from typing import Any, Dict, Generator, Optional, Tuple, Union, cast from urllib.parse import quote -from cfgenvy import yaml_type +from cfgenvy import Parser, yaml_type +from requests import Session +from requests.exceptions import ConnectionError as RequestsConnectionError +from requests.exceptions import HTTPError, Timeout from .postgres import Persistor as Postgres -try: - from requests import Session -except ImportError: - Session = None - logger = getLogger(__name__) @@ -53,6 +53,7 @@ def __init__( # pylint: disable=too-many-arguments,too-many-locals password: str, url: str, flowsheet_id: str, + postgres: Postgres, comment: str = "Not for clinical use.", contact_id_type: str = "CSN", flowsheet_id_type: str = "external", @@ -60,16 +61,17 @@ def __init__( # pylint: disable=too-many-arguments,too-many-locals flowsheet_template_id_type: str = "external", lookback_hours: int = 72, patient_id_type: str = "UID", - poll_timeout: int = 300, + poll_timeout: int = 60, + operation_timeout: int = 5, username: str = "Pennsignals", user_id_type: str = "external", user_id: str = "PENNSIGNALS", ): """__init__.""" - self.authorization = b"Basic " + b64encode( - f"EMP${username}:{password}".encode("utf-8") - ) - self.postgres = None + self.authorization = ( + b"Basic " + b64encode(f"EMP${username}:{password}".encode("utf-8")) + ).decode("utf-8") + self.postgres = postgres self.client_id = client_id self.comment = comment self.contact_id_type = contact_id_type @@ -79,6 +81,7 @@ def __init__( # pylint: disable=too-many-arguments,too-many-locals self.flowsheet_template_id = flowsheet_template_id self.flowsheet_template_id_type = flowsheet_template_id_type self.lookback_hours = lookback_hours + self.operation_timeout = operation_timeout self.patient_id_type = patient_id_type self.poll_timeout = poll_timeout self.url = url @@ -91,7 +94,7 @@ def __call__(self): with self.listener() as listener: with self.postgres.commit() as cur: self.recover(cur, session) - for each in listener: + for each in self.listen(listener): self.on_notify(each, cur, session) def as_yaml(self) -> Dict[str, Any]: @@ -107,6 +110,7 @@ def as_yaml(self) -> Dict[str, Any]: "flowsheet_template_id": self.flowsheet_template_id, "flowsheet_template_id_type": self.flowsheet_template_id_type, "lookback_hours": self.lookback_hours, + "operation_timeut": self.operation_timeout, "patient_id_type": self.patient_id_type, "poll_timeout": self.poll_timeout, "url": self.url, @@ -115,11 +119,11 @@ def as_yaml(self) -> Dict[str, Any]: } @contextmanager - def listener(self): + def listener(self) -> Generator[Any, None, None]: """Listener.""" raise NotImplementedError() - def listen(self, listen, cur, session): + def listen(self, listen) -> Generator[Any, None, None]: """Listen.""" while True: readers, _, exceptions = select( @@ -133,11 +137,11 @@ def listen(self, listen, cur, session): while listen.notifies: yield listen.notified.pop() - def on_notify( # pylint: disable: unused-argument,no-self-use + def on_notify( # pylint: disable=unused-argument,no-self-use self, event, cur, - session, + session: Session, ): """On postgres notify handler.""" logger.debug( @@ -149,42 +153,46 @@ def on_notify( # pylint: disable: unused-argument,no-self-use }, ) - def on_success(self, entity, cur, response): + def on_success(self, entity, cur, content: Dict[str, Any]): """On success.""" raise NotImplementedError() - def on_error(self, entity, cur, response): + def on_error(self, entity, cur, content: Exception): """On error.""" raise NotImplementedError() - def recover(self, cur, session): + def recover(self, cur, session: Session): """Recover.""" sql = self.postgres.sql cur.execute(sql.epic.prediction.recover) for each in cur.fetchall(): - ok, message = self.rest(each, session) - if ok: - self.on_success(each, cur, message) - else: - self.on_error(each, cur, message) + ok, content = self.rest(each, session) + if not ok: + self.on_error(each, cur, cast(Exception, content)) + continue + self.on_success(each, cur, cast(Dict[str, Any], content)) - def rest(self, entity, session): + def rest( + self, + entity, + session: Session, + ) -> Tuple[bool, Union[Exception, Dict[str, Any]]]: """Rest.""" raise NotImplementedError() @contextmanager - def session(self): + def session(self) -> Generator[Any, None, None]: """Session.""" session = Session() session.verify = False session.headers.update( { - "Authorization": self.authorization, - "Content-Type": "application/json", - "Cookie": self.cookie, - "Epic-Client-ID": self.client_id, - "Epic-User-ID": self.user_id, - "Epic-User-IDType": self.user_id_type, + "authorization": self.authorization, + "content-type": "application/json", + "cookie": self.cookie, + "epic-client-id": self.client_id, + "epic-user-id": self.user_id, + "epic-user-idtype": self.user_id_type, } ) yield session @@ -195,6 +203,42 @@ class Notifier(Epic): YAML = "!epicnotifier" + @classmethod + def test( + cls, + csn="278820881", + empi="8330651951", + id=0, # pylint: disable=redefined-builtin + score="0.5", + ): + """Test epic API.""" + cls.as_yaml_type() + Postgres.as_yaml_type() + parser = Parser() + cwd = getcwd() + + notifier = parser.parse( + argv=( + "-c", + pathjoin(cwd, "local", "test.notifier.yaml"), + "-e", + pathjoin(cwd, "secrets", "test.notifier.env"), + ) + ) + + prediction = { + "as_of": datetime.utcnow(), + "csn": csn, + "empi": empi, + "id": id, + "score": score, + } + + with notifier.session() as session: + ok, response = notifier.rest(prediction, session) + print(ok) + print(response) + @classmethod def as_yaml_type(cls, tag: Optional[str] = None): """As yaml type.""" @@ -216,11 +260,11 @@ def _yaml_repr(cls, dumper, self, *, tag): return dumper.represent_mapper(tag, self.as_yaml()) @contextmanager - def listener(self): + def listener(self) -> Generator[Any, None, None]: """Listener.""" postgres = self.postgres sql = postgres.sql - with postgres.listen(sql.prediction.listen) as listener: + with postgres.listen(sql.predictions.listen) as listener: yield listener def on_notify(self, event, cur, session): @@ -229,76 +273,80 @@ def on_notify(self, event, cur, session): sql = self.postgres.sql cur.execute(sql.prediction.recent, event.id) for each in cur.fetchall(): - ok, message = self.rest(each, session) - if ok: - self.on_success(each, cur, message) - else: - self.on_error(each, cur, message) + ok, content = self.rest(each, session) + if not ok: + self.on_error(each, cur, cast(Exception, content)) + continue + self.on_success(each, cur, cast(Dict[str, Any], content)) - def on_success(self, notification, cur, response): + def on_success(self, entity, cur, content: Dict[str, Any]): """On success.""" cur.execute( self.postgres.sql.epic.notification.insert, - {"prediction_id": notification["id"]}, + {"prediction_id": entity["id"]}, ) - def on_error(self, notification, cur, response): + def on_error(self, entity, cur, content: Exception): """On error.""" cur.execute( - self.postgres.sql.epic.notification_error.insert, + self.postgres.sql.epic.notifications.errors.insert, { - "description": response.text, - "name": response.reason, - "prediction_id": notification["id"], + "description": str(content), + "name": content.__class__.__name__, + "prediction_id": entity["id"], }, ) - def recover(self, cur, session): + def recover(self, cur, session: Session): """Recover.""" sql = self.postgres.sql cur.execute(sql.epic.notification.recover) for each in cur.fetchall(): - ok, message = self.rest(each, session) - if ok: - self.on_success(each, cur, message) - else: - self.on_error(each, cur, message) + ok, content = self.rest(each, session) + if not ok: + self.on_error(each, cur, cast(Exception, content)) + continue + self.on_success(each, cur, cast(Dict[str, Any], content)) - def rest(self, prediction, session) -> Tuple[bool, Any]: + def rest( + self, + entity, + session: Session, + ) -> Tuple[bool, Union[Exception, Dict[str, Any]]]: """Rest.""" query = { "Comment": self.comment, - "ContactID": prediction["csn"], + "ContactID": entity["csn"], "ContactIDType": self.contact_id_type, "FlowsheetID": self.flowsheet_id, "FlowsheetIDType": self.flowsheet_id_type, "FlowsheetTemplateID": self.flowsheet_template_id, "FlowsheetTemplateIDType": self.flowsheet_template_id_type, - "InstantValueTaken": prediction["as_of"].strftime( + "InstantValueTaken": entity["as_of"].strftime( "%Y-%m-%dT%H:%M:%SZ" ), - "PatientID": prediction["empi"], + "PatientID": entity["empi"], "PatientIDType": self.patient_id_type, "UserID": self.user_id, "UserIDType": self.user_id_type, - "Value": prediction["score"], + "Value": entity["score"], } - url = self.url.format( **{ key: quote(value.encode("utf-8")) for key, value in query.items() } ) - - print(url) - - response = session.post( - url=url, - data={}, - ) - - return response.status_code == 200, response + try: + response = session.post( + url=url, + json={}, + timeout=self.operation_timeout, + ) + response.raise_for_status() + except (RequestsConnectionError, HTTPError, Timeout) as e: + return False, e + return True, response.json() class Verifier(Epic): @@ -306,48 +354,88 @@ class Verifier(Epic): YAML = "!epicverifier" + @classmethod + def test( + cls, + csn="278820881", + empi="8330651951", + id=0, # pylint: disable=redefined-builtin + score="0.5", + ): + """Test verifier.""" + cls.as_yaml_type() + Postgres.as_yaml_type() + parser = Parser() + cwd = getcwd() + + verifier = parser.parse( + argv=( + "-c", + pathjoin(cwd, "local", "test.verifier.yaml"), + "-e", + pathjoin(cwd, "secrets", "test.verifier.env"), + ) + ) + + notification = { + "as_of": datetime.utcnow(), + "csn": csn, + "empi": empi, + "id": id, + "score": score, + } + + with verifier.session() as session: + ok, response = verifier.rest(notification, session) + print(ok) + print(response) + @contextmanager - def listener(self): + def listener(self) -> Generator[Any, None, None]: """Listener.""" postgres = self.postgres sql = postgres.sql with postgres.listen(sql.notification.listen) as listener: yield listener - def on_notify(self, event, cur, session): + def on_notify(self, event, cur, session: Session): """On notify.""" - super.on_notify(event, cur, session) + super().on_notify(event, cur, session) sql = self.postgres.sql cur.execute(sql.notification.recent, event.id) for each in cur.fetchall(): - ok, response = self.rest(each, session) - if ok: - self.on_success(each, session, response) - else: - self.on_error(each, session, response) + ok, content = self.rest(each, session) + if not ok: + self.on_error(each, cur, cast(Exception, content)) + continue + self.on_success(each, cur, cast(Dict[str, Any], content)) - def on_success(self, notification, cur, response): + def on_success(self, entity, cur, content: Dict[str, Any]): """On success.""" cur.execute( self.postgres.sql.epic.verifications.insert, - {"prediction_id": notification["id"]}, + {"notification_id": entity["id"]}, ) - def on_error(self, notification, cur, response): + def on_error(self, entity, cur, content: Exception): """On error.""" cur.execute( self.postgres.sql.epic.verifications.errors.insert, { - "description": response.text, - "name": response.reason, - "notification_id": notification["id"], + "description": str(content), + "name": content.__class__.__name__, + "notification_id": entity["id"], }, ) - def rest(self, notification, session) -> Tuple[bool, Any]: + def rest( + self, + entity, + session: Session, + ) -> Tuple[bool, Union[Exception, Dict[str, Any]]]: """Rest.""" json = { - "ContactID": notification["csn"], + "ContactID": entity["csn"], "ContactIDType": self.contact_id_type, "FlowsheetRowIDs": [ { @@ -356,95 +444,18 @@ def rest(self, notification, session) -> Tuple[bool, Any]: } ], "LookbackHours": self.lookback_hours, - "PatientID": notification["empi"], + "PatientID": entity["empi"], "PatientIDType": self.patient_id_type, "UserID": self.user_id, "UserIDType": self.user_id_type, } - response = session.post( - url=self.url, - json=json, - ) - return response.status_code == 200, response - - -def test_notifier(csn, empi, score): - """Rest.""" - from os import getcwd - from os.path import join as pathjoin - - from cfgenvy import Parser - - from .asset import Asset - - Asset.as_yaml_type() - Postgres.as_yaml_type() - Notifier.as_yaml_type() - parser = Parser() - cwd = getcwd() - - notifier = parser.parse( - argv=( - "-c", - pathjoin(cwd, "local", "test.notifier.yaml"), - "-e", - pathjoin(cwd, "secrets", "test.notifier.env"), - ) - ) - - prediction = { - "as_of": datetime.utcnow(), - "csn": csn, - "empi": empi, - "score": score, - } - - with notifier.session() as session: - ok, response = notifier.rest(prediction, session) - print(ok) - print(response.json()) - - -def test_verifier(csn, empi, score): - """Test verifier.""" - from os import getcwd - from os.path import join as pathjoin - - from cfgenvy import Parser - - from .asset import Asset - - Asset.as_yaml_type() - Postgres.as_yaml_type() - Verifier.as_yaml_type() - parser = Parser() - cwd = getcwd() - - verifier = parser.parse( - argv=( - "-c", - pathjoin(cwd, "local", "test.verifier.yaml"), - "-e", - pathjoin(cwd, "secrets", "test.verifier.env"), - ) - ) - - notification = { - "as_of": datetime.utcnow(), - "csn": csn, - "empi": empi, - "score": score, - } - - with verifier.session() as session: - ok, response = verifier.rest(notification, session) - print(ok) - print(response.json()) - - -if __name__ == "__main__": - test_notifier( - csn="278820881", - empi="8330651951", - score="0.5", - ) + try: + response = session.post( + url=self.url, + json=json, + timeout=self.operation_timeout, + ) + response.raise_for_status() + except (RequestsConnectionError, HTTPError, Timeout) as e: + return False, e + return True, response.json() diff --git a/src/dsdk/persistor.py b/src/dsdk/persistor.py index ebf7f6f..2e5e969 100644 --- a/src/dsdk/persistor.py +++ b/src/dsdk/persistor.py @@ -233,6 +233,7 @@ class Persistor(AbstractPersistor): @classmethod def as_yaml_type(cls, tag: Optional[str] = None) -> None: """As yaml type.""" + Asset.as_yaml_type() yaml_type( cls, tag or cls.YAML, From 41a98e08e1c30cd959fec67ed51d9992b267f15d Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Fri, 20 Aug 2021 17:24:43 -0400 Subject: [PATCH 20/33] Add server mock --- src/dsdk/epic.py | 189 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 185 insertions(+), 4 deletions(-) diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index eaa9682..d6ee0f1 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -1,14 +1,29 @@ # -*- coding: utf-8 -*- """Epic.""" +from __future__ import annotations + from base64 import b64encode from contextlib import contextmanager from datetime import datetime +from http.server import BaseHTTPRequestHandler, HTTPServer +from json import dumps as json_dumps, loads as json_loads from logging import getLogger from os import getcwd from os.path import join as pathjoin +from re import compile as re_compile from select import select # pylint: disable=no-name-in-module -from typing import Any, Dict, Generator, Optional, Tuple, Union, cast +from typing import ( + Any, + Callable, + Dict, + Generator, + List, + Optional, + Tuple, + Union, + cast, +) from urllib.parse import quote from cfgenvy import Parser, yaml_type @@ -21,6 +36,9 @@ logger = getLogger(__name__) +DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" + + class Epic: # pylint: disable=too-many-instance-attributes """Epic.""" @@ -322,9 +340,7 @@ def rest( "FlowsheetIDType": self.flowsheet_id_type, "FlowsheetTemplateID": self.flowsheet_template_id, "FlowsheetTemplateIDType": self.flowsheet_template_id_type, - "InstantValueTaken": entity["as_of"].strftime( - "%Y-%m-%dT%H:%M:%SZ" - ), + "InstantValueTaken": entity["as_of"].strftime(DATETIME_FORMAT), "PatientID": entity["empi"], "PatientIDType": self.patient_id_type, "UserID": self.user_id, @@ -459,3 +475,168 @@ def rest( except (RequestsConnectionError, HTTPError, Timeout) as e: return False, e return True, response.json() + + +def parse_path(path: str) -> Tuple[str, Dict[str, str]]: + """Parse path.""" + url, *query_list = path.split("?", 1) + query = "".join(query_list) + param_list = query.split("&") + params: Dict[str, str] = {} + for each in param_list: + key, *values = each.split("=", 1) + params[key] = "".join(values) + return (url, params) + + +def equals(expected: Any) -> Callable[..., None]: + """Equals.""" + + def _check(key: str, value: Any): + if value != expected: + raise ValueError(f"{key} actual: {value}, expected: {expected}") + + return _check + + +def matches(expected: str) -> Callable[..., None]: + """Matches.""" + pattern = re_compile(expected) + + def _check(key: str, value: str): + if not pattern.match(value): + raise ValueError(f"{key} actual: {value}, pattern: {expected}") + + return _check + + +def is_float(key: str, value: str): + """Is float.""" + actual = str(float(value)) + if actual != value: + raise ValueError(f"{key} actual: {value}, is not float") + + +def is_datetime(key: str, value: str): + """Is datetime.""" + actual = datetime.strptime(value, DATETIME_FORMAT).strftime( + DATETIME_FORMAT + ) + if actual != value: + raise ValueError(f"{key} actual: {value}, is not datetime") + + +class Server(HTTPServer): + """Server. + + The errors returned do not match the API. + """ + + def __init__( + self, + server_address, + handler_class, + add_flowsheet_value_url: str, + get_flowsheet_rows_url: str, + ): + """__init__.""" + super().__init__(server_address, handler_class) + self.by_flowsheet_id: Dict[str, Any] = {} + self.dispatch_url = { + parse_path(path)[0].lower(): method + for path, method in ( + (add_flowsheet_value_url, self.add_flowsheet_value), + (get_flowsheet_rows_url, self.get_flowsheet_rows), + ) + } + + def __call__(self): + """__call__.""" + self.serve_forever() + + def add_flowsheet_value(self, request, url, params, body): + """Add flowsheet value.""" + errors = request.check_headers() + try: + for key, validate in ( + ("Comment", None), + ("ContactID", None), + ("ContactIDType", None), + ("FlowsheetID", None), + ("FlowsheetIDType", None), + ("FlowsheetTemplateID", None), + ("FlowsheetTemplateIDType", None), + ("PatientID", None), + ("PatientIDType", equals("internal")), + ("UserID", None), + ("UserIDType", equals("internal")), + ("Value", is_float), + ("InstantValueTaken", is_datetime), + ): + value = params[key] + if validate is not None: + validate(key, value) + except (KeyError, ValueError) as e: + errors.append(e) + + if errors: + request.send_response(400) + request.send_headers("content-type", "application/json") + request.end_headers() + request.wfile.write( + json_dumps({"Success": False, "Errors": [ + f"{error.__class__.__name__}: {error}" + for error in errors + ]}).encode('utf-8')) + request.wfile.close() + return + + comment = params["Comment"] + contact_id = params["ContectID"] + contact_id_type = params["ContactIDType"] + flowsheet_id = params["FlowsheetID"] + flowsheet_id_type = params["FlowsheetIDType"] + flowsheet_template_id = params["FlowsheetTemplateID"] + flowsheet_template_id_type = params["FlowsheetTemplateIDType"] + patient_id = params["PatientID"] + patient_id_type = params["PatientIDType"] + user_id = params["UserID"] + user_id_type = params["UserIDType"] + value = params["Value"] + instant_value_taken = params["InstantValueTaken"] + + def get_flowsheet_rows(self, request, url, params, body): + """Get flowsheet rows.""" + errors = request.check_headers() + json = json_loads(body) + + +class RequestHandler(BaseHTTPRequestHandler): + """Request Handler.""" + + def do_POST(self): # noqa: N802 + """Do post.""" + url, params = parse_path(self.path) + url = url.lower() + content_length = int(self.headers.get("content-length", 0)) + body = self.rfile.read(content_length) + self.server.dispatch[url](self, url, params, body) + + def check_headers(self) -> List[Exception]: + """Check headers.""" + errors: List[Exception] = [] + for key, validate in ( + ("authorization", None), + ("content-type", equals("application/json")), + ("cookie", None), + ("epic-client-id", matches(r"\d{8}-\d{4}-\d{4}-\d{4}-\d{12}")), + ("epic-user-id", equals("PENNSIGNALS")), + ("epic-user-idtype", equals("external")), + ): + try: + value = self.headers.get(key, None) + if validate: + validate(key, value) + except (KeyError, ValueError) as e: + errors.append(e) + return errors From d6617652040eebef05e43aecbdf16337c36dc177 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 23 Aug 2021 13:05:31 -0400 Subject: [PATCH 21/33] Update configuration --- local/.gitignore | 2 ++ local/configuration.yaml | 18 ------------------ local/notifier.yaml | 14 ++++++++++++++ local/verifier.yaml | 14 ++++++++++++++ pyproject.toml | 2 +- secrets/.gitignore | 2 ++ secrets/example.notifier.env | 8 ++++++++ secrets/example.verifier.env | 8 ++++++++ src/dsdk/epic.py | 35 +++++++++++++++++++++++------------ 9 files changed, 72 insertions(+), 31 deletions(-) create mode 100644 local/notifier.yaml create mode 100644 local/verifier.yaml create mode 100644 secrets/example.notifier.env create mode 100644 secrets/example.verifier.env diff --git a/local/.gitignore b/local/.gitignore index b87b8a0..af29b09 100644 --- a/local/.gitignore +++ b/local/.gitignore @@ -1,3 +1,5 @@ * !.gitignore !configuration.yaml +!notifier.yaml +!verifier.yaml diff --git a/local/configuration.yaml b/local/configuration.yaml index 7610cdb..73ea058 100644 --- a/local/configuration.yaml +++ b/local/configuration.yaml @@ -1,21 +1,3 @@ -notification: - epic: !epic &epic - authorization: ${INTERCONNECT_AUTHORIZATION} - comment: "Not for clinical use." - cookie: ${INTERCONNECT_COOKIE} - contact_type_id: csn - flowsheet_id: ${INTERCONNECT_FLOWSHEET_ID} - flowsheet_id_type: internal - flowsheet_template_id: 3040005300 - flowsheet_template_id_type: internal - lookback_hours: 72 - patient_id_type: uid - user_id: pennsignals - user_id_type: external - uri: ${INTERCONNECT_HOST}${INTERCONNECT_NOTIFICATION_URI} -verification: !verification - epic: *epic - uri: ${INTERCONNECT_HOST}${INTERCONNECT_VERIFICATION_URI} postgres: !postgres database: ${POSTGRES_DATABASE} host: ${POSTRES_HOST} diff --git a/local/notifier.yaml b/local/notifier.yaml new file mode 100644 index 0000000..8ccc680 --- /dev/null +++ b/local/notifier.yaml @@ -0,0 +1,14 @@ +!epicnotifier +client_id: ${EPIC_CLIENT_ID} +cookie: ${EPIC_COOKIE} +flowsheet_id: ${EPIC_FLOWSHEET_ID} +flowsheet_template_id: ${EPIC_FLOWSHEET_TEMPLATE_ID} +password: ${EPIC_PASSWORD} +postgres: !postgres + database: ${POSTGRES_DATABASE} + host: ${POSTRES_HOST} + password: ${POSTGRES_PASSWORD} + username: ${POSTGRES_USERNAME} +username: ${EPIC_USERNAME} +user_id: ${EPIC_USER_ID} +url: ${EPIC_HOST}api/epic/2011/clinical/patient/addflowsheetvalue/flowsheetvalue?PatientID={PatientID}&PatientIDType={PatientIDType}&ContactID={ContactID}&ContactIDType={ContactIDType}&UserID={UserID}&UserIDType={UserIDType}&FlowsheetID={FlowsheetID}&FlowsheetIDType={FlowsheetIDType}&Value={Value}&Comment={Comment}&InstantValueTaken={InstantValueTaken}&FlowsheetTemplateID={FlowsheetTemplateID}&FlowsheetTemplateIDType={FlowsheetTemplateIDType} diff --git a/local/verifier.yaml b/local/verifier.yaml new file mode 100644 index 0000000..33aee13 --- /dev/null +++ b/local/verifier.yaml @@ -0,0 +1,14 @@ +!epicverifier +client_id: ${EPIC_CLIENT_ID} +cookie: ${EPIC_COOKIE} +flowsheet_id: ${EPIC_FLOWSHEET_ID} +flowsheet_template_id: ${EPIC_FLOWHSEET_TEMPLATE_ID} +password: ${EPIC_PASSWORD} +postgres: !postgres + database: ${POSTGRES_DATABASE} + host: ${POSTRES_HOST} + password: ${POSTGRES_PASSWORD} + username: ${POSTGRES_USERNAME} +username: ${EPIC_USERNAME} +user_id: ${EPIC_USER_ID} +url: ${EPIC_HOST}api/epic/2014/clinical/patient/getflowsheetrows/flowsheetrows diff --git a/pyproject.toml b/pyproject.toml index 034dd1f..d7f3ae1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ known_first_party = ["dsdk"] default_section = "THIRDPARTY" [tool.pylint.basic] -good-names = '''a,b,c,d,df,e,i,id,logger,n,on,ok,tz''' +good-names = '''a,b,c,d,df,do_POST,e,i,id,logger,n,on,ok,tz''' [tool.pylint.message_control] disable = '''duplicate-code,C0330''' diff --git a/secrets/.gitignore b/secrets/.gitignore index d6b7ef3..914d7a6 100644 --- a/secrets/.gitignore +++ b/secrets/.gitignore @@ -1,2 +1,4 @@ * !.gitignore +!example.notifier.env +!example.verifier.env diff --git a/secrets/example.notifier.env b/secrets/example.notifier.env new file mode 100644 index 0000000..64379f1 --- /dev/null +++ b/secrets/example.notifier.env @@ -0,0 +1,8 @@ +EPIC_CLIENT_ID=00000000-0000-0000-0000-000000000000 +EPIC_COOKIE=ASP.NET_SessionId=000000000000000000000000 +EPIS_FLOWSHEET_ID="0000000000" +EPIC_FLOWSHEET_TEMPLATE_ID="0000000000" +EPIC_HOST=https://epic/interconnect-prd-web/ +EPIC_PASSWORD=password +EPIC_USERNAME=Epicuser +EPIC_USER_ID=EPICUSER diff --git a/secrets/example.verifier.env b/secrets/example.verifier.env new file mode 100644 index 0000000..9191474 --- /dev/null +++ b/secrets/example.verifier.env @@ -0,0 +1,8 @@ +EPIC_CLIENT_ID=00000000-0000-0000-0000-000000000000 +EPIC_COOKIE=ASP.NET_SessionId=000000000000000000000000 +EPIC_FLOWSHEET_ID="0000000000" +EPIC_FLOWSHEET_TEMPLATE_ID="0000000000" +EPIC_HOST=https://epic/interconnect-prd-web/ +EPIC_PASSWORD=password +EPIC_USERNAME=Epicuser +EPIC_USER_ID=EPISUSER diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index d6ee0f1..f5839e2 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -7,7 +7,8 @@ from contextlib import contextmanager from datetime import datetime from http.server import BaseHTTPRequestHandler, HTTPServer -from json import dumps as json_dumps, loads as json_loads +from json import dumps as json_dumps +from json import loads as json_loads from logging import getLogger from os import getcwd from os.path import join as pathjoin @@ -71,19 +72,19 @@ def __init__( # pylint: disable=too-many-arguments,too-many-locals password: str, url: str, flowsheet_id: str, + flowsheet_template_id: str, postgres: Postgres, + user_id: str, + username: str, comment: str = "Not for clinical use.", contact_id_type: str = "CSN", flowsheet_id_type: str = "external", - flowsheet_template_id: str = "3040005300", flowsheet_template_id_type: str = "external", lookback_hours: int = 72, patient_id_type: str = "UID", poll_timeout: int = 60, operation_timeout: int = 5, - username: str = "Pennsignals", user_id_type: str = "external", - user_id: str = "PENNSIGNALS", ): """__init__.""" self.authorization = ( @@ -225,7 +226,9 @@ class Notifier(Epic): def test( cls, csn="278820881", + config: Optional[str] = None, empi="8330651951", + env: Optional[str] = None, id=0, # pylint: disable=redefined-builtin score="0.5", ): @@ -238,9 +241,9 @@ def test( notifier = parser.parse( argv=( "-c", - pathjoin(cwd, "local", "test.notifier.yaml"), + config or pathjoin(cwd, "local", "notifier.yaml"), "-e", - pathjoin(cwd, "secrets", "test.notifier.env"), + env or pathjoin(cwd, "secrets", "staging.notifier.env"), ) ) @@ -374,7 +377,9 @@ class Verifier(Epic): def test( cls, csn="278820881", + config: Optional[str] = None, empi="8330651951", + env: Optional[str] = None, id=0, # pylint: disable=redefined-builtin score="0.5", ): @@ -387,9 +392,9 @@ def test( verifier = parser.parse( argv=( "-c", - pathjoin(cwd, "local", "test.verifier.yaml"), + config or pathjoin(cwd, "local", "verifier.yaml"), "-e", - pathjoin(cwd, "secrets", "test.verifier.env"), + env or pathjoin(cwd, "secrets", "staging.verifier.env"), ) ) @@ -584,10 +589,16 @@ def add_flowsheet_value(self, request, url, params, body): request.send_headers("content-type", "application/json") request.end_headers() request.wfile.write( - json_dumps({"Success": False, "Errors": [ - f"{error.__class__.__name__}: {error}" - for error in errors - ]}).encode('utf-8')) + json_dumps( + { + "Errors": [ + f"{error.__class__.__name__}: {error}" + for error in errors + ], + "Success": False, + } + ).encode("utf-8") + ) request.wfile.close() return From efe191a203ece281a224ac30bb19d2291ec398d2 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 23 Aug 2021 13:49:04 -0400 Subject: [PATCH 22/33] Egress extends cfgenvy parser --- docker-compose.yml | 55 ++++++++++++++++++++--- setup.py | 9 ++-- src/dsdk/epic.py | 106 ++++++++++++++++++++++---------------------- src/dsdk/service.py | 9 +--- 4 files changed, 111 insertions(+), 68 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3a2290b..9677fe8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,20 @@ version: "3.8" volumes: postgres_data: +environment: &environment +- EPIC_CLIENT_ID=00000000-0000-0000-0000-000000000000 +- EPIC_COOKIE=ASP.NET_SessionId=000000000000000000000000 +- EPIC_FLOWSHEET_ID=0000000000 +- EPIC_HOST=https://epic/interconnect-prd-web/ +- EPIC_PASSWORD=epic +- EPIC_USERNAME=Epicuser +- EPIC_USER_ID=EPICUSER +- POSTGRES_HOST=postgres +- POSTGRES_USERNAME=postgres +- POSTGRES_PASSWORD=postgres +- POSTGRES_DATABASE=test +- POSTGRES_PORT=5432 + services: postgres: @@ -30,10 +44,41 @@ services: depends_on: - postgres environment: - - POSTGRES_HOST=postgres - - POSTGRES_USERNAME=postgres - - POSTGRES_PASSWORD=postgres - - POSTGRES_DATABASE=test - - POSTGRES_PORT=5432 + << *environment + - CONFIGURATION=/local/test.yaml + volumes: + - ./assets/sql:/sql + - ./local:/local + + notifier: + build: + context: . + target: epic.notifier + depends_on: + - epic + - postgres + environment: + << *environment + - CONFIG=/local/test.notifier.yaml + volumes: + - ./assets/sql:/sql + + verifier: + build: + context: . + target: epic.verfier + depends_on: + - epic + - postgres + environment: + << *environment + - CONFIG=/local/test.verifier.yaml volumes: - ./assets/sql:/sql + + epic: + build: + context: . + target: epic.server + environment: + - CONFIG=/local/test.server.yaml diff --git a/setup.py b/setup.py index d017932..7006d9c 100644 --- a/setup.py +++ b/setup.py @@ -55,10 +55,11 @@ setup( entry_points={ "console_scripts": [ - "epic-notify = dsdk.epic:Notifier.main", - "epic-validate = dsdk.epic:Validator.main", - "epic-notify-test = dsdk.epic:Notifier.test", - "epic-validate-test = dsdk.epic:Notifier.test", + "epic.server = dsdk.epic:Server.main" + "epic.notify = dsdk.epic:Notifier.main", + "epic.validate = dsdk.epic:Validator.main", + "epic.notify.test.api = dsdk.epic:Notifier.test", + "epic.validate.test.api = dsdk.epic:Notifier.test", ] }, extras_require={ diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index f5839e2..30883a1 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -7,8 +7,7 @@ from contextlib import contextmanager from datetime import datetime from http.server import BaseHTTPRequestHandler, HTTPServer -from json import dumps as json_dumps -from json import loads as json_loads +from json import dumps, loads from logging import getLogger from os import getcwd from os.path import join as pathjoin @@ -20,6 +19,7 @@ Dict, Generator, List, + Mapping, Optional, Tuple, Union, @@ -28,11 +28,20 @@ from urllib.parse import quote from cfgenvy import Parser, yaml_type +from pkg_resources import DistributionNotFound, get_distribution from requests import Session from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import HTTPError, Timeout from .postgres import Persistor as Postgres +from .util import configure_logger + +try: + __version__ = get_distribution("dsdk").version +except DistributionNotFound: + # package is not installed + pass + logger = getLogger(__name__) @@ -40,14 +49,19 @@ DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" -class Epic: # pylint: disable=too-many-instance-attributes - """Epic.""" +class Egress(Parser): # pylint: disable=too-many-instance-attributes + """Egress.""" + + ON = dumps({"key": "%s.on"}) + END = dumps({"key": "%s.end"}) + YAML = "!egress" - YAML = "!epic" + VERSION = __version__ @classmethod def as_yaml_type(cls, tag: Optional[str] = None): """As yaml type.""" + Postgres.as_yaml_type() yaml_type( cls, tag or cls.YAML, @@ -55,6 +69,26 @@ def as_yaml_type(cls, tag: Optional[str] = None): repr=cls._yaml_repr, ) + @classmethod + @contextmanager + def context( + cls, + key: str, + argv: Optional[List[str]] = None, + env: Optional[Mapping[str, str]] = None, + ): + """Context.""" + configure_logger("dsdk") + logger.info(cls.ON, key) + yield cls.parse(argv=argv, env=env) + logger.info(cls.END, key) + + @classmethod + def main(cls): + """Main.""" + with cls.context("main") as service: + service() + @classmethod def _yaml_init(cls, loader, node): """Yaml init.""" @@ -67,7 +101,7 @@ def _yaml_repr(cls, dumper, self, *, tag): def __init__( # pylint: disable=too-many-arguments,too-many-locals self, - client_id: str, # 00000000-0000-0000-0000-000000000000 + client_id: str, cookie: str, password: str, url: str, @@ -217,7 +251,7 @@ def session(self) -> Generator[Any, None, None]: yield session -class Notifier(Epic): +class Notifier(Egress): """Notifier.""" YAML = "!epicnotifier" @@ -226,27 +260,11 @@ class Notifier(Epic): def test( cls, csn="278820881", - config: Optional[str] = None, empi="8330651951", - env: Optional[str] = None, id=0, # pylint: disable=redefined-builtin score="0.5", ): """Test epic API.""" - cls.as_yaml_type() - Postgres.as_yaml_type() - parser = Parser() - cwd = getcwd() - - notifier = parser.parse( - argv=( - "-c", - config or pathjoin(cwd, "local", "notifier.yaml"), - "-e", - env or pathjoin(cwd, "secrets", "staging.notifier.env"), - ) - ) - prediction = { "as_of": datetime.utcnow(), "csn": csn, @@ -254,11 +272,11 @@ def test( "id": id, "score": score, } - - with notifier.session() as session: - ok, response = notifier.rest(prediction, session) - print(ok) - print(response) + with cls.context("test.notifier.api") as notifier: + with notifier.session() as session: + ok, response = notifier.rest(prediction, session) + print(ok) + print(response) @classmethod def as_yaml_type(cls, tag: Optional[str] = None): @@ -368,7 +386,7 @@ def rest( return True, response.json() -class Verifier(Epic): +class Verifier(Egress): """Verifier.""" YAML = "!epicverifier" @@ -377,27 +395,11 @@ class Verifier(Epic): def test( cls, csn="278820881", - config: Optional[str] = None, empi="8330651951", - env: Optional[str] = None, id=0, # pylint: disable=redefined-builtin score="0.5", ): """Test verifier.""" - cls.as_yaml_type() - Postgres.as_yaml_type() - parser = Parser() - cwd = getcwd() - - verifier = parser.parse( - argv=( - "-c", - config or pathjoin(cwd, "local", "verifier.yaml"), - "-e", - env or pathjoin(cwd, "secrets", "staging.verifier.env"), - ) - ) - notification = { "as_of": datetime.utcnow(), "csn": csn, @@ -405,11 +407,11 @@ def test( "id": id, "score": score, } - - with verifier.session() as session: - ok, response = verifier.rest(notification, session) - print(ok) - print(response) + with cls.context("test.verifier.api") as verifier: + with verifier.session() as session: + ok, response = verifier.rest(notification, session) + print(ok) + print(response) @contextmanager def listener(self) -> Generator[Any, None, None]: @@ -589,7 +591,7 @@ def add_flowsheet_value(self, request, url, params, body): request.send_headers("content-type", "application/json") request.end_headers() request.wfile.write( - json_dumps( + dumps( { "Errors": [ f"{error.__class__.__name__}: {error}" @@ -619,7 +621,7 @@ def add_flowsheet_value(self, request, url, params, body): def get_flowsheet_rows(self, request, url, params, body): """Get flowsheet rows.""" errors = request.check_headers() - json = json_loads(body) + json = loads(body) class RequestHandler(BaseHTTPRequestHandler): diff --git a/src/dsdk/service.py b/src/dsdk/service.py index ef233f8..065737d 100644 --- a/src/dsdk/service.py +++ b/src/dsdk/service.py @@ -203,6 +203,8 @@ class Service(Parser): # pylint: disable=too-many-instance-attributes @classmethod def as_yaml_type(cls, tag: Optional[str] = None) -> None: """As yaml type.""" + Asset.as_yaml_type() + Interval.as_yaml_type() yaml_type( cls, tag or cls.YAML, @@ -242,13 +244,6 @@ def validate_gold(cls): with cls.context("validate_gold") as service: service.on_validate_gold() - @classmethod - def yaml_types(cls) -> None: - """Yaml types.""" - Asset.as_yaml_type() - Interval.as_yaml_type() - cls.as_yaml_type() - @classmethod def _yaml_init(cls, loader, node): """Yaml init.""" From afb3bc651abe500d737caf2e0cb4b5f469ddb09a Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 23 Aug 2021 14:38:56 -0400 Subject: [PATCH 23/33] Egress api tests work as console scripts --- docker-compose.yml | 10 +++++----- local/notifier.yaml | 7 ++++++- local/verifier.yaml | 9 +++++++-- readme.md | 5 +++++ setup.py | 10 +++++++--- src/dsdk/epic.py | 27 ++++++--------------------- 6 files changed, 36 insertions(+), 32 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9677fe8..3dcde88 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,11 +11,11 @@ environment: &environment - EPIC_PASSWORD=epic - EPIC_USERNAME=Epicuser - EPIC_USER_ID=EPICUSER +- POSTGRES_DATABASE=test - POSTGRES_HOST=postgres -- POSTGRES_USERNAME=postgres - POSTGRES_PASSWORD=postgres -- POSTGRES_DATABASE=test - POSTGRES_PORT=5432 +- POSTGRES_USERNAME=postgres services: @@ -45,7 +45,7 @@ services: - postgres environment: << *environment - - CONFIGURATION=/local/test.yaml + - CONFIG=/local/test.yaml volumes: - ./assets/sql:/sql - ./local:/local @@ -59,7 +59,7 @@ services: - postgres environment: << *environment - - CONFIG=/local/test.notifier.yaml + - CONFIG=/local/notifier.yaml volumes: - ./assets/sql:/sql @@ -72,7 +72,7 @@ services: - postgres environment: << *environment - - CONFIG=/local/test.verifier.yaml + - CONFIG=/local/verifier.yaml volumes: - ./assets/sql:/sql diff --git a/local/notifier.yaml b/local/notifier.yaml index 8ccc680..7e804b3 100644 --- a/local/notifier.yaml +++ b/local/notifier.yaml @@ -6,8 +6,13 @@ flowsheet_template_id: ${EPIC_FLOWSHEET_TEMPLATE_ID} password: ${EPIC_PASSWORD} postgres: !postgres database: ${POSTGRES_DATABASE} - host: ${POSTRES_HOST} + host: ${POSTGRES_HOST} password: ${POSTGRES_PASSWORD} + port: 5432 + sql: !asset + ext: .sql + path: ./assets/postgres + tables: [] username: ${POSTGRES_USERNAME} username: ${EPIC_USERNAME} user_id: ${EPIC_USER_ID} diff --git a/local/verifier.yaml b/local/verifier.yaml index 33aee13..22aa269 100644 --- a/local/verifier.yaml +++ b/local/verifier.yaml @@ -2,12 +2,17 @@ client_id: ${EPIC_CLIENT_ID} cookie: ${EPIC_COOKIE} flowsheet_id: ${EPIC_FLOWSHEET_ID} -flowsheet_template_id: ${EPIC_FLOWHSEET_TEMPLATE_ID} +flowsheet_template_id: ${EPIC_FLOWSHEET_TEMPLATE_ID} password: ${EPIC_PASSWORD} postgres: !postgres database: ${POSTGRES_DATABASE} - host: ${POSTRES_HOST} + host: ${POSTGRES_HOST} password: ${POSTGRES_PASSWORD} + port: 5432 + sql: !asset + ext: .sql + path: ./assets/postgres + tables: [] username: ${POSTGRES_USERNAME} username: ${EPIC_USERNAME} user_id: ${EPIC_USER_ID} diff --git a/readme.md b/readme.md index 92cccf5..b8d486a 100644 --- a/readme.md +++ b/readme.md @@ -35,6 +35,11 @@ Session: docker-compose down deactivate +Test epic egress to production with non-production empi, csn, score, flowsheet_id and flowsheet_template_id: + + epic.notify.api.test -c ./local/notifier.yaml -e ./secrets/staging.notifier.env + epic.notify.api.test -c ./local/verifier.yaml -e ./secrets/staging.verifier.env + Rebuild the postgres container and remove the docker volume if the database schema is changed. ## CI/CD Lint & Test: diff --git a/setup.py b/setup.py index 7006d9c..236b7a6 100644 --- a/setup.py +++ b/setup.py @@ -57,9 +57,13 @@ "console_scripts": [ "epic.server = dsdk.epic:Server.main" "epic.notify = dsdk.epic:Notifier.main", - "epic.validate = dsdk.epic:Validator.main", - "epic.notify.test.api = dsdk.epic:Notifier.test", - "epic.validate.test.api = dsdk.epic:Notifier.test", + "epic.verify = dsdk.epic:Verifier.main", + # epic.notify.api.test + # -c ./local/notifier.yaml -e ./secrets/staging.notifier.env + "epic.notify.api.test = dsdk.epic:Notifier.test", + # epic.verify.api.test + # -c ./local/verifier.yaml -e ./secrets/staging.verifier.env + "epic.verify.api.test = dsdk.epic:Verifier.test", ] }, extras_require={ diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index 30883a1..1502eab 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -34,7 +34,7 @@ from requests.exceptions import HTTPError, Timeout from .postgres import Persistor as Postgres -from .util import configure_logger +from .utils import configure_logger try: __version__ = get_distribution("dsdk").version @@ -89,6 +89,11 @@ def main(cls): with cls.context("main") as service: service() + @classmethod + def yaml_types(cls): + """Yaml types.""" + cls.as_yaml_type() + @classmethod def _yaml_init(cls, loader, node): """Yaml init.""" @@ -278,26 +283,6 @@ def test( print(ok) print(response) - @classmethod - def as_yaml_type(cls, tag: Optional[str] = None): - """As yaml type.""" - yaml_type( - cls, - tag or cls.YAML, - init=cls._yaml_init, - repr=cls._yaml_repr, - ) - - @classmethod - def _yaml_init(cls, loader, node): - """Yaml init.""" - return cls(**loader.construct_mapping(node, deep=True)) - - @classmethod - def _yaml_repr(cls, dumper, self, *, tag): - """Yaml repr.""" - return dumper.represent_mapper(tag, self.as_yaml()) - @contextmanager def listener(self) -> Generator[Any, None, None]: """Listener.""" From 4680dddf5a0944e4c72bd3edea08d080c6d0d550 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 23 Aug 2021 15:45:26 -0400 Subject: [PATCH 24/33] Push docker environment variables into safe env file --- docker-compose.yml | 32 +++++++++++--------------------- local/.gitignore | 1 + local/docker.env | 12 ++++++++++++ 3 files changed, 24 insertions(+), 21 deletions(-) create mode 100644 local/docker.env diff --git a/docker-compose.yml b/docker-compose.yml index 3dcde88..421a0fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,20 +3,6 @@ version: "3.8" volumes: postgres_data: -environment: &environment -- EPIC_CLIENT_ID=00000000-0000-0000-0000-000000000000 -- EPIC_COOKIE=ASP.NET_SessionId=000000000000000000000000 -- EPIC_FLOWSHEET_ID=0000000000 -- EPIC_HOST=https://epic/interconnect-prd-web/ -- EPIC_PASSWORD=epic -- EPIC_USERNAME=Epicuser -- EPIC_USER_ID=EPICUSER -- POSTGRES_DATABASE=test -- POSTGRES_HOST=postgres -- POSTGRES_PASSWORD=postgres -- POSTGRES_PORT=5432 -- POSTGRES_USERNAME=postgres - services: postgres: @@ -44,10 +30,10 @@ services: depends_on: - postgres environment: - << *environment - CONFIG=/local/test.yaml + - ENV=/local/docker.env volumes: - - ./assets/sql:/sql + - ./assets:/assets - ./local:/local notifier: @@ -58,10 +44,11 @@ services: - epic - postgres environment: - << *environment - CONFIG=/local/notifier.yaml + - ENV=/local/docker.env volumes: - - ./assets/sql:/sql + - ./assets:/assets + - ./local:/local verifier: build: @@ -71,14 +58,17 @@ services: - epic - postgres environment: - << *environment - CONFIG=/local/verifier.yaml + - ENV=/local/docker.env volumes: - - ./assets/sql:/sql + - ./assets:/assets + - ./local:/local epic: build: context: . target: epic.server environment: - - CONFIG=/local/test.server.yaml + - CONFIG=/local/server.yaml + volumes: + - ./local:/local diff --git a/local/.gitignore b/local/.gitignore index af29b09..10165bf 100644 --- a/local/.gitignore +++ b/local/.gitignore @@ -1,5 +1,6 @@ * !.gitignore +!docker.env !configuration.yaml !notifier.yaml !verifier.yaml diff --git a/local/docker.env b/local/docker.env new file mode 100644 index 0000000..ea1ad10 --- /dev/null +++ b/local/docker.env @@ -0,0 +1,12 @@ +EPIC_CLIENT_ID=00000000-0000-0000-0000-000000000000 +EPIC_COOKIE=ASP.NET_SessionId=000000000000000000000000 +EPIC_FLOWSHEET_ID=0000000000 +EPIC_HOST=https://epic/interconnect-prd-web/ +EPIC_PASSWORD=epic +EPIC_USERNAME=Epicuser +EPIC_USER_ID=EPICUSER +POSTGRES_DATABASE=test +POSTGRES_HOST=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_PORT=5432 +POSTGRES_USERNAME=postgres From 2a2b82f95bdfb7f67d63c7c5fc8220939e7de7ac Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Wed, 25 Aug 2021 10:57:41 -0400 Subject: [PATCH 25/33] Add epic image --- buster.dockerfile | 10 ++++++++++ docker-compose.yml | 19 +++++++++++++++---- local/.gitignore | 2 +- local/docker.env | 12 ------------ local/epic.yaml | 4 ++++ local/notifier.yaml | 2 +- local/verifier.yaml | 2 +- secrets/.gitignore | 3 +-- secrets/docker.env | 16 ++++++++++++++++ secrets/example.notifier.env | 8 -------- secrets/example.verifier.env | 8 -------- setup.py | 2 +- src/dsdk/epic.py | 8 ++++---- 13 files changed, 54 insertions(+), 42 deletions(-) delete mode 100644 local/docker.env create mode 100644 local/epic.yaml create mode 100644 secrets/docker.env delete mode 100644 secrets/example.notifier.env delete mode 100644 secrets/example.verifier.env diff --git a/buster.dockerfile b/buster.dockerfile index cadf9c8..37f3b51 100644 --- a/buster.dockerfile +++ b/buster.dockerfile @@ -25,6 +25,16 @@ RUN \ apt-get -qq autoremove -y --purge && \ rm -rf /var/lib/apt/lists/* +FROM python:3.9.6-slim-buster as epic +LABEL name="epic" +WORKDIR /tmp +ENV PATH /root/.local/bin:$PATH +COPY --from=build /root/.local /root/.local +COPY --from=build /root/assets /tmp/assets +COPY --from=build /usr/bin/tini /usr/bin +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["epic"] + FROM build as test ARG IFLAGS LABEL name="dsdk.test" diff --git a/docker-compose.yml b/docker-compose.yml index 421a0fe..06637cf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,9 @@ services: target: test depends_on: - postgres + - notifier + - verifier + - epic environment: - CONFIG=/local/test.yaml - ENV=/local/docker.env @@ -39,7 +42,8 @@ services: notifier: build: context: . - target: epic.notifier + target: epic + command: epic.notify depends_on: - epic - postgres @@ -53,7 +57,8 @@ services: verifier: build: context: . - target: epic.verfier + target: epic + command: epic.verify depends_on: - epic - postgres @@ -67,8 +72,14 @@ services: epic: build: context: . - target: epic.server + target: epic environment: - - CONFIG=/local/server.yaml + - CONFIG=/local/epic.yaml + - ENV=/local/docker.env + expose: + - "80" + ports: + - "80:80" + restart: always volumes: - ./local:/local diff --git a/local/.gitignore b/local/.gitignore index 10165bf..b91cff5 100644 --- a/local/.gitignore +++ b/local/.gitignore @@ -1,6 +1,6 @@ * !.gitignore -!docker.env !configuration.yaml !notifier.yaml !verifier.yaml +!epic.yaml diff --git a/local/docker.env b/local/docker.env deleted file mode 100644 index ea1ad10..0000000 --- a/local/docker.env +++ /dev/null @@ -1,12 +0,0 @@ -EPIC_CLIENT_ID=00000000-0000-0000-0000-000000000000 -EPIC_COOKIE=ASP.NET_SessionId=000000000000000000000000 -EPIC_FLOWSHEET_ID=0000000000 -EPIC_HOST=https://epic/interconnect-prd-web/ -EPIC_PASSWORD=epic -EPIC_USERNAME=Epicuser -EPIC_USER_ID=EPICUSER -POSTGRES_DATABASE=test -POSTGRES_HOST=postgres -POSTGRES_PASSWORD=postgres -POSTGRES_PORT=5432 -POSTGRES_USERNAME=postgres diff --git a/local/epic.yaml b/local/epic.yaml new file mode 100644 index 0000000..c9b0e67 --- /dev/null +++ b/local/epic.yaml @@ -0,0 +1,4 @@ +!epic +address: 0.0.0.0:80 +add_flowsheet_value_path: ${EPIC_ROOT}${EPIC_ADD_FLOWSHEET_VALUE_PATH} +get_flowsheet_rows_path: ${EPIC_ROOT}${EPIC_GET_FLOWSHEET_ROWS_PATH} diff --git a/local/notifier.yaml b/local/notifier.yaml index 7e804b3..9f3259e 100644 --- a/local/notifier.yaml +++ b/local/notifier.yaml @@ -16,4 +16,4 @@ postgres: !postgres username: ${POSTGRES_USERNAME} username: ${EPIC_USERNAME} user_id: ${EPIC_USER_ID} -url: ${EPIC_HOST}api/epic/2011/clinical/patient/addflowsheetvalue/flowsheetvalue?PatientID={PatientID}&PatientIDType={PatientIDType}&ContactID={ContactID}&ContactIDType={ContactIDType}&UserID={UserID}&UserIDType={UserIDType}&FlowsheetID={FlowsheetID}&FlowsheetIDType={FlowsheetIDType}&Value={Value}&Comment={Comment}&InstantValueTaken={InstantValueTaken}&FlowsheetTemplateID={FlowsheetTemplateID}&FlowsheetTemplateIDType={FlowsheetTemplateIDType} +url: ${EPIC_URL}${EPIC_ROOT}${EPIC_ADD_FLOWSHEET_VALUE_PATH} diff --git a/local/verifier.yaml b/local/verifier.yaml index 22aa269..7060b20 100644 --- a/local/verifier.yaml +++ b/local/verifier.yaml @@ -16,4 +16,4 @@ postgres: !postgres username: ${POSTGRES_USERNAME} username: ${EPIC_USERNAME} user_id: ${EPIC_USER_ID} -url: ${EPIC_HOST}api/epic/2014/clinical/patient/getflowsheetrows/flowsheetrows +url: ${EPIC_URL}${EPIC_ROOT}${EPIC_GET_FLOWSHEET_ROWS_PATH} diff --git a/secrets/.gitignore b/secrets/.gitignore index 914d7a6..b8dfb1e 100644 --- a/secrets/.gitignore +++ b/secrets/.gitignore @@ -1,4 +1,3 @@ * !.gitignore -!example.notifier.env -!example.verifier.env +!docker.env diff --git a/secrets/docker.env b/secrets/docker.env new file mode 100644 index 0000000..ee14b71 --- /dev/null +++ b/secrets/docker.env @@ -0,0 +1,16 @@ +EPIC_ADD_FLOWSHEET_VALUE_PATH=api/epic/2011/clinical/patient/addflowsheetvalue/flowsheetvalue?PatientID={PatientID}&PatientIDType={PatientIDType}&ContactID={ContactID}&ContactIDType={ContactIDType}&UserID={UserID}&UserIDType={UserIDType}&FlowsheetID={FlowsheetID}&FlowsheetIDType={FlowsheetIDType}&Value={Value}&Comment={Comment}&InstantValueTaken={InstantValueTaken}&FlowsheetTemplateID={FlowsheetTemplateID}&FlowsheetTemplateIDType={FlowsheetTemplateIDType} +EPIC_CLIENT_ID=00000000-0000-0000-0000-000000000000 +EPIC_COOKIE=ASP.NET_SessionId=000000000000000000000000 +EPIC_FLOWSHEET_ID=0000000000 +EPIC_FLOWSHEET_TEMPLATE_ID=0000000000 +EPIC_GET_FLOWSHEET_ROWS_PATH=api/epic/2014/Clinical/Patient/getflowsheetrows/flowsheetrows +EPIC_PASSWORD=epic +EPIC_ROOT=interconnect-prd-web/ +EPIC_URL=https://epic/ +EPIC_USERNAME=Epicuser +EPIC_USER_ID=EPICUSER +POSTGRES_DATABASE=test +POSTGRES_HOST=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_PORT=5432 +POSTGRES_USERNAME=postgres diff --git a/secrets/example.notifier.env b/secrets/example.notifier.env deleted file mode 100644 index 64379f1..0000000 --- a/secrets/example.notifier.env +++ /dev/null @@ -1,8 +0,0 @@ -EPIC_CLIENT_ID=00000000-0000-0000-0000-000000000000 -EPIC_COOKIE=ASP.NET_SessionId=000000000000000000000000 -EPIS_FLOWSHEET_ID="0000000000" -EPIC_FLOWSHEET_TEMPLATE_ID="0000000000" -EPIC_HOST=https://epic/interconnect-prd-web/ -EPIC_PASSWORD=password -EPIC_USERNAME=Epicuser -EPIC_USER_ID=EPICUSER diff --git a/secrets/example.verifier.env b/secrets/example.verifier.env deleted file mode 100644 index 9191474..0000000 --- a/secrets/example.verifier.env +++ /dev/null @@ -1,8 +0,0 @@ -EPIC_CLIENT_ID=00000000-0000-0000-0000-000000000000 -EPIC_COOKIE=ASP.NET_SessionId=000000000000000000000000 -EPIC_FLOWSHEET_ID="0000000000" -EPIC_FLOWSHEET_TEMPLATE_ID="0000000000" -EPIC_HOST=https://epic/interconnect-prd-web/ -EPIC_PASSWORD=password -EPIC_USERNAME=Epicuser -EPIC_USER_ID=EPISUSER diff --git a/setup.py b/setup.py index 236b7a6..14f5319 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ setup( entry_points={ "console_scripts": [ - "epic.server = dsdk.epic:Server.main" + "epic = dsdk.epic:Server.main" "epic.notify = dsdk.epic:Notifier.main", "epic.verify = dsdk.epic:Verifier.main", # epic.notify.api.test diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index 1502eab..16fe779 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -528,8 +528,8 @@ def __init__( self, server_address, handler_class, - add_flowsheet_value_url: str, - get_flowsheet_rows_url: str, + add_flowsheet_value_path: str, + get_flowsheet_rows_path: str, ): """__init__.""" super().__init__(server_address, handler_class) @@ -537,8 +537,8 @@ def __init__( self.dispatch_url = { parse_path(path)[0].lower(): method for path, method in ( - (add_flowsheet_value_url, self.add_flowsheet_value), - (get_flowsheet_rows_url, self.get_flowsheet_rows), + (add_flowsheet_value_path, self.add_flowsheet_value), + (get_flowsheet_rows_path, self.get_flowsheet_rows), ) } From 9cbfd7d69f75e4654851092e6bd5a46dff1a7b83 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Wed, 25 Aug 2021 12:03:54 -0400 Subject: [PATCH 26/33] Use tmp as WORKDIR --- buster.dockerfile | 6 +++--- postgres/dockerfile | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/buster.dockerfile b/buster.dockerfile index 37f3b51..121fd12 100644 --- a/buster.dockerfile +++ b/buster.dockerfile @@ -2,7 +2,7 @@ ARG IFLAGS="--quiet --no-cache-dir --user" FROM python:3.9.6-slim-buster as build ARG IFLAGS -WORKDIR /root +WORKDIR /tmp ENV PATH /root/.local/bin:$PATH ENV TINI_VERSION v0.19.0 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini @@ -30,7 +30,7 @@ LABEL name="epic" WORKDIR /tmp ENV PATH /root/.local/bin:$PATH COPY --from=build /root/.local /root/.local -COPY --from=build /root/assets /tmp/assets +COPY --from=build /tmp/assets /tmp/assets COPY --from=build /usr/bin/tini /usr/bin ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["epic"] @@ -38,7 +38,7 @@ CMD ["epic"] FROM build as test ARG IFLAGS LABEL name="dsdk.test" -WORKDIR /root +WORKDIR /tmp RUN \ pip install ${IFLAGS} ".[all]" ENTRYPOINT [ "/usr/bin/tini", "--" ] diff --git a/postgres/dockerfile b/postgres/dockerfile index c8523f6..f026dc6 100644 --- a/postgres/dockerfile +++ b/postgres/dockerfile @@ -1,4 +1,4 @@ -FROM timescale/timescaledb-postgis:latest-pg12 as postgres +FROM timescale/timescaledb-postgis:latest-pg13 as postgres LABEL name=postgres COPY ./docker-entrypoint.sh /usr/local/bin/ COPY ./sql/initdb.d/*.sql /docker-entrypoint-initdb.d/ From 9954a76391e1b6cf100d0b1fa7fcecb01c4ace83 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Wed, 25 Aug 2021 15:33:20 -0400 Subject: [PATCH 27/33] Unify secrets to a single env file --- readme.md | 4 ++-- setup.py | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/readme.md b/readme.md index b8d486a..eaa8ceb 100644 --- a/readme.md +++ b/readme.md @@ -37,8 +37,8 @@ Session: Test epic egress to production with non-production empi, csn, score, flowsheet_id and flowsheet_template_id: - epic.notify.api.test -c ./local/notifier.yaml -e ./secrets/staging.notifier.env - epic.notify.api.test -c ./local/verifier.yaml -e ./secrets/staging.verifier.env + epic.notify.api.test -c ./local/notifier.yaml -e ./secrets/staging.env + epic.notify.api.test -c ./local/verifier.yaml -e ./secrets/staging.env Rebuild the postgres container and remove the docker volume if the database schema is changed. diff --git a/setup.py b/setup.py index 14f5319..963ef0a 100644 --- a/setup.py +++ b/setup.py @@ -58,11 +58,7 @@ "epic = dsdk.epic:Server.main" "epic.notify = dsdk.epic:Notifier.main", "epic.verify = dsdk.epic:Verifier.main", - # epic.notify.api.test - # -c ./local/notifier.yaml -e ./secrets/staging.notifier.env "epic.notify.api.test = dsdk.epic:Notifier.test", - # epic.verify.api.test - # -c ./local/verifier.yaml -e ./secrets/staging.verifier.env "epic.verify.api.test = dsdk.epic:Verifier.test", ] }, From bc50f3fb25dd92e0df01cb73e7c542bb0d199fe8 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Wed, 25 Aug 2021 16:17:58 -0400 Subject: [PATCH 28/33] Pass tests with staging.env against api --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 963ef0a..df836b7 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ ( "cfgenvy@" "git+https://github.com/pennsignals/cfgenvy.git" - "@1.1.0#egg=cfgenvy" + "@1.2.1#egg=cfgenvy" ), "numpy>=1.15.4", "pandas>=0.23.4", From 2bfc760c049fb0b5a4005f3f528c3bc0d34d7d0a Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 30 Aug 2021 15:32:07 -0400 Subject: [PATCH 29/33] Adding storage for flowsheets --- src/dsdk/epic.py | 234 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 195 insertions(+), 39 deletions(-) diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index 16fe779..ca91412 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -9,8 +9,6 @@ from http.server import BaseHTTPRequestHandler, HTTPServer from json import dumps, loads from logging import getLogger -from os import getcwd -from os.path import join as pathjoin from re import compile as re_compile from select import select # pylint: disable=no-name-in-module from typing import ( @@ -518,6 +516,12 @@ def is_datetime(key: str, value: str): raise ValueError(f"{key} actual: {value}, is not datetime") +def is_lookback(key: str, value: int): + """Is lookback.""" + if not (0 <= value and value <= 72): + raise ValueError(f"{key} actual {value}, is not (0 <= value <= 72") + + class Server(HTTPServer): """Server. @@ -533,12 +537,12 @@ def __init__( ): """__init__.""" super().__init__(server_address, handler_class) - self.by_flowsheet_id: Dict[str, Any] = {} + self.by_user: Dict[str, Any] = {} self.dispatch_url = { parse_path(path)[0].lower(): method for path, method in ( - (add_flowsheet_value_path, self.add_flowsheet_value), - (get_flowsheet_rows_path, self.get_flowsheet_rows), + (add_flowsheet_value_path, AddFlowsheetValue()), + (get_flowsheet_rows_path, GetFlowsheetRows()), ) } @@ -546,8 +550,89 @@ def __call__(self): """__call__.""" self.serve_forever() - def add_flowsheet_value(self, request, url, params, body): - """Add flowsheet value.""" + +class AddFlowsheetValue: + """AddFlowsheetValue.""" + + def __call__( + self, request, url, params, body, + ): + """__call__.""" + if self.is_invalid(request, url, params, body): + return + + comment = params["Comment"] + contact_id = params["ContectID"] + contact_id_type = params["ContactIDType"].upper() + flowsheet_id = params["FlowsheetID"] + flowsheet_id_type = params["FlowsheetIDType"].upper() + # flowsheet_template_id = params["FlowsheetTemplateID"] + # flowsheet_template_id_type = params["FlowsheetTemplateIDType"] + patient_id = params["PatientID"] + patient_id_type = params["PatientIDType"].upper() + user_id = params["UserID"] + user_id_type = params["UserIDType"].upper() + value = str(params["Value"]) + instant = params["InstantValueTaken"].isoformat() + 'Z' + + key = (flowsheet_id, flowsheet_id_type) + by_patient = self.by_flowsheet.get(key) + if by_patient is None: + by_patient = self.by_flowsheet[key] = {} + + key = (patient_id, patient_id_type) + by_contact = by_patient.get(key) + if by_contact is None: + by_contact = by_patient[key] = {} + + key = (contact_id, contact_id_type) + rows = by_contact.get(key) + if rows is None: + rows = by_contact[key] = { + "FlowsheetRows": [ + { + "FlowsheetColumns": [], + "Name": "Test Message", + "Occurrence": 28, # ??? + "Unit": "", + }, + ], + } + + flowsheet_columns = rows["FlowsheetRows"]["FlowsheetColumns"] + + entry = { + "Comment": comment, + "FlowsheetRowID": [ + { + "ID": flowsheet_id, + "Type": flowsheet_id_type, + } + ], + "FormattedValue": value, + "Instant": instant, + "LinesDrainsAirwaysID": None, + "OrderIDs": [], + "RawValue": value, + "UserEnteredBy": [ + { + "ID": user_id, + "Type": user_id_type, + }, + ], + } + + flowsheet_columns.append(entry) + response = { + "Errors": [], + "Success": True, + } + return request.respond(200, response) + + def is_invalid( + self, request, url, params, body, + ) -> Optional[List[Exception]]: + """Is invalid.""" errors = request.check_headers() try: for key, validate in ( @@ -571,42 +656,105 @@ def add_flowsheet_value(self, request, url, params, body): except (KeyError, ValueError) as e: errors.append(e) - if errors: - request.send_response(400) - request.send_headers("content-type", "application/json") - request.end_headers() - request.wfile.write( - dumps( - { - "Errors": [ - f"{error.__class__.__name__}: {error}" - for error in errors - ], - "Success": False, - } - ).encode("utf-8") - ) - request.wfile.close() + if not errors: + return False + + # TODO: + # - errors do not match the spec + response = { + "Errors": [ + f"{error.__class__.__name__}: {error}" + for error in errors + ], + "Success": False, + } + request.respond(400, response) + return True + + +class GetFlowsheetRows: + """Get flowsheet rows.""" + + def __call__( + self, request, url, params, body, + ): + """__call__.""" + json = loads(body) + if self.is_invalid(request, url, params, json): return - comment = params["Comment"] - contact_id = params["ContectID"] - contact_id_type = params["ContactIDType"] - flowsheet_id = params["FlowsheetID"] - flowsheet_id_type = params["FlowsheetIDType"] - flowsheet_template_id = params["FlowsheetTemplateID"] - flowsheet_template_id_type = params["FlowsheetTemplateIDType"] - patient_id = params["PatientID"] - patient_id_type = params["PatientIDType"] - user_id = params["UserID"] - user_id_type = params["UserIDType"] - value = params["Value"] - instant_value_taken = params["InstantValueTaken"] + contact_id = json["ContactID"] + contact_id_type = json["ContactIDType"] + patient_id = json["PatientID"] + patient_id_type = json["PatientIDType"] + # TODO: lookback_hours = json["LookbackHours"] + # user_id = json["UserID"] + # user_id_type = json["UserIDType"] + flowsheet_row_ids = json["FlowsheetRowIDs"] + + # TODO FIXME + + default = {'FlowsheetRows': []} + + key = (flowsheet_id, flowsheet_id_type.lower()) + by_patient = self.by_flowsheet.get(key) + if by_patient is None: + return request.respond(request, 400, default) + + key = (patient_id, patient_id_type) + by_contact = by_patient.get(key) + if by_contact is None: + return request.respond(request, 400, default) - def get_flowsheet_rows(self, request, url, params, body): - """Get flowsheet rows.""" + key = (contact_id, contact_id_type) + body = by_contact.get(key) + if body is None: + return request.respond(request, 400, default) + + return request.respond(request, 200, body) + + def is_invalid(self, request, url, params, body): + """Is invalid.""" errors = request.check_headers() - json = loads(body) + try: + for key, validate in ( + ("ContactID", None), + ("ContactIDType", None), + ("FlowsheetID", None), + ("FlowsheetIDType", None), + ("LookbackHours", is_lookback), + ("PatientID", None), + ("PatientIDType", equals("internal")), + ("UserID", None), + ("UserIDType", equals("internal")), + ): + value = body[key] + if validate is not None: + validate(key, value) + except (KeyError, ValueError) as e: + errors.append(e) + + try: + key = "FlowsheetRowIDs" + value = body[key] + for each in value: + _ = each["ID"] + _ = each["IDType"] + except (KeyError, ValueError) as e: + errors.append(e) + + if not errors: + return False + + response = { + "Errors": [ + f"{error.__class__.__name__}: {error}" + for error in errors + ], + "Success": False, + } + request.respond(400, response) + return True class RequestHandler(BaseHTTPRequestHandler): @@ -638,3 +786,11 @@ def check_headers(self) -> List[Exception]: except (KeyError, ValueError) as e: errors.append(e) return errors + + def respond(self, code, response) -> None: + """Respond.""" + self.send_response(code) + self.send_headers("content-type", "application/json") + self.end_headers() + self.wfile.write(dumps(response).encode("utf-8")) + self.wfile.close() From 49c40ce3fabc55705ae9b6ed69d773b811669452 Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Mon, 30 Aug 2021 17:13:40 -0400 Subject: [PATCH 30/33] Tailor flowsheet entries added to flowsheets query --- src/dsdk/epic.py | 266 +++++++++++++++++++++++++++-------------------- 1 file changed, 155 insertions(+), 111 deletions(-) diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index ca91412..70adeb3 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -3,9 +3,10 @@ from __future__ import annotations +from argparse import Namespace from base64 import b64encode from contextlib import contextmanager -from datetime import datetime +from datetime import datetime, timedelta from http.server import BaseHTTPRequestHandler, HTTPServer from json import dumps, loads from logging import getLogger @@ -479,11 +480,11 @@ def parse_path(path: str) -> Tuple[str, Dict[str, str]]: return (url, params) -def equals(expected: Any) -> Callable[..., None]: +def equals(expected: str) -> Callable[..., None]: """Equals.""" - def _check(key: str, value: Any): - if value != expected: + def _check(key: str, value: str): + if value.lower() != expected.lower(): raise ValueError(f"{key} actual: {value}, expected: {expected}") return _check @@ -518,8 +519,8 @@ def is_datetime(key: str, value: str): def is_lookback(key: str, value: int): """Is lookback.""" - if not (0 <= value and value <= 72): - raise ValueError(f"{key} actual {value}, is not (0 <= value <= 72") + if value < 0 or value > 72: + raise ValueError(f"{key} actual {value}, is not between 0 and 72") class Server(HTTPServer): @@ -537,7 +538,10 @@ def __init__( ): """__init__.""" super().__init__(server_address, handler_class) - self.by_user: Dict[str, Any] = {} + self.by_patient: Dict[ + Tuple[str, str], + Dict[Tuple[str, str], Dict[Tuple[str, str], Namespace]], + ] = {} self.dispatch_url = { parse_path(path)[0].lower(): method for path, method in ( @@ -546,92 +550,103 @@ def __init__( ) } + def by_patient_and_contact( + self, + entry: Namespace, + ) -> Dict[Tuple[str, str], Namespace]: + """By patient and contact.""" + key = (entry.patient_id, entry.patient_id_type) + by_contact = self.by_patient.get(key) + if by_contact is None: + by_contact = self.by_patient[key] = {} + + key = (entry.contact_id, entry.contact_id_type) + by_flowsheet = by_contact.get(key) + if by_flowsheet is None: + by_flowsheet = by_contact[key] = {} + return by_flowsheet + def __call__(self): """__call__.""" self.serve_forever() -class AddFlowsheetValue: - """AddFlowsheetValue.""" +class ResponseHandler: + """ResponseHandler.""" def __call__( - self, request, url, params, body, + self, + request, + url, + params, + body, ): """__call__.""" - if self.is_invalid(request, url, params, body): - return + raise NotImplementedError() - comment = params["Comment"] - contact_id = params["ContectID"] - contact_id_type = params["ContactIDType"].upper() - flowsheet_id = params["FlowsheetID"] - flowsheet_id_type = params["FlowsheetIDType"].upper() - # flowsheet_template_id = params["FlowsheetTemplateID"] - # flowsheet_template_id_type = params["FlowsheetTemplateIDType"] - patient_id = params["PatientID"] - patient_id_type = params["PatientIDType"].upper() - user_id = params["UserID"] - user_id_type = params["UserIDType"].upper() - value = str(params["Value"]) - instant = params["InstantValueTaken"].isoformat() + 'Z' - - key = (flowsheet_id, flowsheet_id_type) - by_patient = self.by_flowsheet.get(key) - if by_patient is None: - by_patient = self.by_flowsheet[key] = {} - - key = (patient_id, patient_id_type) - by_contact = by_patient.get(key) - if by_contact is None: - by_contact = by_patient[key] = {} - - key = (contact_id, contact_id_type) - rows = by_contact.get(key) - if rows is None: - rows = by_contact[key] = { - "FlowsheetRows": [ - { - "FlowsheetColumns": [], - "Name": "Test Message", - "Occurrence": 28, # ??? - "Unit": "", - }, - ], - } + def is_invalid( + self, + request, + url, + params, + body, + ) -> bool: + """Is invalid.""" + raise NotImplementedError() - flowsheet_columns = rows["FlowsheetRows"]["FlowsheetColumns"] - entry = { - "Comment": comment, - "FlowsheetRowID": [ - { - "ID": flowsheet_id, - "Type": flowsheet_id_type, - } - ], - "FormattedValue": value, - "Instant": instant, - "LinesDrainsAirwaysID": None, - "OrderIDs": [], - "RawValue": value, - "UserEnteredBy": [ - { - "ID": user_id, - "Type": user_id_type, - }, - ], - } +class AddFlowsheetValue(ResponseHandler): + """AddFlowsheetValue.""" + + def __call__( + self, + request, + url, + params, + body, + ) -> None: + """__call__.""" + if self.is_invalid(request, url, params, body): + return + + entry = Namespace() + entry.comment = params["Comment"] + entry.contact_id = params["ContectID"] + entry.contact_id_type = params["ContactIDType"].lower() + entry.flowsheet_id = params["FlowsheetID"] + entry.flowsheet_id_type = params["FlowsheetIDType"].lower() + entry.flowsheet_template_id = params["FlowsheetTemplateID"] + entry.flowsheet_template_id_type = params[ + "FlowsheetTemplateIDType" + ].lower() + entry.instant = params["InstantValueTaken"] + entry.patient_id = params["PatientID"] + entry.patient_id_type = params["PatientIDType"].lower() + entry.user_id = params["UserID"] + entry.user_id_type = params["UserIDType"].lower() + entry.value = params["Value"] + + by_flowsheet = request.server.by_patient_and_contact(entry) + + key = (entry.flowsheet_id, entry.flowsheet_id_type) + entries = by_flowsheet.get(key) + if entries is None: + entries = by_flowsheet[key] = [] + entries.append(entry) - flowsheet_columns.append(entry) response = { "Errors": [], "Success": True, } - return request.respond(200, response) + request.respond(200, response) def is_invalid( - self, request, url, params, body, - ) -> Optional[List[Exception]]: + self, + request, + url, + params, + body, + ) -> bool: """Is invalid.""" errors = request.check_headers() try: @@ -663,8 +678,7 @@ def is_invalid( # - errors do not match the spec response = { "Errors": [ - f"{error.__class__.__name__}: {error}" - for error in errors + f"{error.__class__.__name__}: {error}" for error in errors ], "Success": False, } @@ -672,46 +686,77 @@ def is_invalid( return True -class GetFlowsheetRows: +class GetFlowsheetRows(ResponseHandler): """Get flowsheet rows.""" def __call__( - self, request, url, params, body, - ): + self, + request, + url, + params, + body, + ) -> None: """__call__.""" json = loads(body) if self.is_invalid(request, url, params, json): return - contact_id = json["ContactID"] - contact_id_type = json["ContactIDType"] - patient_id = json["PatientID"] - patient_id_type = json["PatientIDType"] - # TODO: lookback_hours = json["LookbackHours"] - # user_id = json["UserID"] - # user_id_type = json["UserIDType"] - flowsheet_row_ids = json["FlowsheetRowIDs"] - - # TODO FIXME - - default = {'FlowsheetRows': []} - - key = (flowsheet_id, flowsheet_id_type.lower()) - by_patient = self.by_flowsheet.get(key) - if by_patient is None: - return request.respond(request, 400, default) - - key = (patient_id, patient_id_type) - by_contact = by_patient.get(key) - if by_contact is None: - return request.respond(request, 400, default) - - key = (contact_id, contact_id_type) - body = by_contact.get(key) - if body is None: - return request.respond(request, 400, default) - - return request.respond(request, 200, body) + query = Namespace() + query.contact_id = json["ContactID"] + query.contact_id_type = json["ContactIDType"].lower() + query.patient_id = json["PatientID"] + query.patient_id_type = json["PatientIDType"].lower() + query.lookback_hours = json["LookbackHours"] + query.lookback = datetime.now() - timedelta( + hours=-int(query.lookback_hours) + ) + query.user_id = json["UserID"] + query.user_id_type = json["UserIDType"].lower() + query.flowsheet_row_ids = [ + Namespace( + flowsheet_id=each["ID"], + flowsheet_id_type=each["IDType"].lower(), + ) + for each in json["FlowsheetRowIDs"] + ] + # TODO some errors here if flowsheets are empty + # implying patient or contact DNE? + by_flowsheet = request.server.by_patient_and_contact(query) + response = { + "FlowsheetRows": [ + { + "FlowsheetColumns": [ + { + "Comment": entry.comment, + "FlowsheetRowID": [ + { + "ID": entry.flowsheet_id, + "IDType": entry.flowsheet_id_type.upper(), + }, + ], + "FormattedValue": entry.value, + "Instant": entry.instant.isoformat() + "Z", + "LinesDrainsAirwaysID": None, + "OrderIDs": [], + "RawValue": entry.value, + "UserEnteredBy": [ + { + "ID": entry.user_id, + "IDType": entry.user_id_type.upper(), + }, + ], + } + for key in query.flowsheet_row_ids + for entry in by_flowsheet(key) + if entry.instant >= query.lookback + ], + "Name": "Penn Signals Test Message", + "Occurence": "28", # ??? + "Unit": "", + } + ], + } + request.respond(200, response) def is_invalid(self, request, url, params, body): """Is invalid.""" @@ -748,8 +793,7 @@ def is_invalid(self, request, url, params, body): response = { "Errors": [ - f"{error.__class__.__name__}: {error}" - for error in errors + f"{error.__class__.__name__}: {error}" for error in errors ], "Success": False, } @@ -790,7 +834,7 @@ def check_headers(self) -> List[Exception]: def respond(self, code, response) -> None: """Respond.""" self.send_response(code) - self.send_headers("content-type", "application/json") + self.send_header("content-type", "application/json") self.end_headers() self.wfile.write(dumps(response).encode("utf-8")) self.wfile.close() From 228fd2dc878a7af4edf6a0b63cf1abe29deafdbe Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Tue, 31 Aug 2021 12:32:51 -0400 Subject: [PATCH 31/33] Add encoding --- src/dsdk/asset.py | 2 +- src/dsdk/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dsdk/asset.py b/src/dsdk/asset.py index b86b5eb..b5e09dc 100644 --- a/src/dsdk/asset.py +++ b/src/dsdk/asset.py @@ -45,7 +45,7 @@ def build(cls, *, path: str, ext: str): s_name, s_ext = splitext(name) if s_ext != ext: continue - with open(child) as fin: + with open(child, encoding="utf-8") as fin: kwargs[s_name] = fin.read() return cls(path=path, ext=ext, **kwargs) diff --git a/src/dsdk/utils.py b/src/dsdk/utils.py index 3b9eb60..8f5936e 100644 --- a/src/dsdk/utils.py +++ b/src/dsdk/utils.py @@ -63,7 +63,7 @@ def chunks(sequence: Sequence[Any], n: int): def dump_json_file(obj: Any, path: str) -> None: """Dump json to file.""" - with open(path, "w") as fout: + with open(path, "w", encoding="utf-8") as fout: json_dump(obj, fout) @@ -87,7 +87,7 @@ def get_tzinfo(key: str) -> tzinfo: def load_json_file(path: str) -> object: """Load json from file.""" - with open(path, "r") as fin: + with open(path, "r", encoding="utf-8") as fin: return json_load(fin) From 51d5b340d01c7b5c8fadcf79efa6c2069412cece Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Thu, 23 Sep 2021 15:06:29 -0400 Subject: [PATCH 32/33] Update csn with one more visible --- src/dsdk/epic.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/dsdk/epic.py b/src/dsdk/epic.py index 70adeb3..a7a36a8 100644 --- a/src/dsdk/epic.py +++ b/src/dsdk/epic.py @@ -263,7 +263,8 @@ class Notifier(Egress): @classmethod def test( cls, - csn="278820881", + # csn="278820881", + csn="218202909", # inpatient admission date is 2019-02-06 at PAH empi="8330651951", id=0, # pylint: disable=redefined-builtin score="0.5", @@ -378,7 +379,8 @@ class Verifier(Egress): @classmethod def test( cls, - csn="278820881", + # csn="278820881", + csn="218202909", # inpatient admission date is 2019-02-06 at PAH empi="8330651951", id=0, # pylint: disable=redefined-builtin score="0.5", From 7cda6456bd3b7f042576efde31ffd3c7a6f7ddff Mon Sep 17 00:00:00 2001 From: Jason Lubken Date: Thu, 23 Sep 2021 15:29:47 -0400 Subject: [PATCH 33/33] Update listen --- assets/postgres/epic/notifications/channel.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/postgres/epic/notifications/channel.sql b/assets/postgres/epic/notifications/channel.sql index ad482c9..2977bff 100644 --- a/assets/postgres/epic/notifications/channel.sql +++ b/assets/postgres/epic/notifications/channel.sql @@ -1 +1 @@ -example.epic_notifications +listen example.epic_notifications