From 50876c339f23ce90bdf163116406b3c469162253 Mon Sep 17 00:00:00 2001 From: Peter Kosztolanyi Date: Sun, 14 Oct 2018 00:08:49 +0100 Subject: [PATCH 1/5] fix to load zero values --- target_postgres/db_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_postgres/db_sync.py b/target_postgres/db_sync.py index 4fba051..3bfcf2c 100644 --- a/target_postgres/db_sync.py +++ b/target_postgres/db_sync.py @@ -150,7 +150,7 @@ def record_to_csv_line(self, record): flatten = flatten_record(record) return ','.join( [ - json.dumps(flatten[name]) if name in flatten and flatten[name] else '' + json.dumps(flatten[name]) if name in flatten and (flatten[name] == 0 or flatten[name]) else '' for name in self.flatten_schema ] ) From 65f2a3e7b9e47bf6b2756441096144b6152cecbc Mon Sep 17 00:00:00 2001 From: Peter Kosztolanyi Date: Sun, 14 Oct 2018 00:52:31 +0100 Subject: [PATCH 2/5] Fixed multipleOf validation issue: convert float to Decimal --- target_postgres/__init__.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/target_postgres/__init__.py b/target_postgres/__init__.py index 15a12b8..3dd408d 100644 --- a/target_postgres/__init__.py +++ b/target_postgres/__init__.py @@ -13,13 +13,24 @@ from tempfile import TemporaryFile import pkg_resources -from jsonschema.validators import Draft4Validator +from jsonschema.validators import Draft4Validator, FormatChecker import singer from target_postgres.db_sync import DbSync logger = singer.get_logger() +def float_to_decimal(value): + '''Walk the given data structure and turn all instances of float into + double.''' + if isinstance(value, float): + return Decimal(str(value)) + if isinstance(value, list): + return [float_to_decimal(child) for child in value] + if isinstance(value, dict): + return {k: float_to_decimal(v) for k, v in value.items()} + return value + def emit_state(state): if state is not None: line = json.dumps(state) @@ -65,7 +76,7 @@ def persist_lines(config, lines): stream = o['stream'] # Validate record - validators[stream].validate(o['record']) + validators[stream].validate(float_to_decimal(o['record'])) sync = stream_to_sync[stream] @@ -93,7 +104,8 @@ def persist_lines(config, lines): raise Exception("Line is missing required key 'stream': {}".format(line)) stream = o['stream'] schemas[stream] = o - validators[stream] = Draft4Validator(o['schema']) + schema = float_to_decimal(o['schema']) + validators[stream] = Draft4Validator(schema, format_checker=FormatChecker()) if 'key_properties' not in o: raise Exception("key_properties field is required") key_properties[stream] = o['key_properties'] From aae1f5e2816bf38311ea1055dbce554258b838be Mon Sep 17 00:00:00 2001 From: Devin Zuczek Date: Tue, 23 Oct 2018 16:22:49 -0400 Subject: [PATCH 3/5] Attempting to fix https://github.com/statsbotco/target-postgres/issues/6 --- target_postgres/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/target_postgres/__init__.py b/target_postgres/__init__.py index 3dd408d..8e7ab43 100644 --- a/target_postgres/__init__.py +++ b/target_postgres/__init__.py @@ -13,7 +13,8 @@ from tempfile import TemporaryFile import pkg_resources -from jsonschema.validators import Draft4Validator, FormatChecker +from jsonschema import validators, Draft4Validator, FormatChecker +from decimal import Decimal import singer from target_postgres.db_sync import DbSync From a95a05862075093f6a445b7702b6c2f1d112839e Mon Sep 17 00:00:00 2001 From: Devin Zuczek Date: Fri, 27 Mar 2020 15:40:51 -0400 Subject: [PATCH 4/5] Decode UTF8 characters from tap --- target_postgres/db_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_postgres/db_sync.py b/target_postgres/db_sync.py index 3bfcf2c..6e07c90 100644 --- a/target_postgres/db_sync.py +++ b/target_postgres/db_sync.py @@ -150,7 +150,7 @@ def record_to_csv_line(self, record): flatten = flatten_record(record) return ','.join( [ - json.dumps(flatten[name]) if name in flatten and (flatten[name] == 0 or flatten[name]) else '' + json.dumps(flatten[name], ensure_ascii=False) if name in flatten and (flatten[name] == 0 or flatten[name]) else '' for name in self.flatten_schema ] ) From 6d6261445de8ca939e8172983f0916d555dc93e5 Mon Sep 17 00:00:00 2001 From: Devin Zuczek Date: Wed, 15 Jul 2020 13:03:14 -0400 Subject: [PATCH 5/5] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9f73a0c..4ee1832 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ py_modules=["target_postgres"], install_requires=[ "singer-python==5.1.1", - "psycopg2==2.7.5", + "psycopg2==2.8.4", "inflection==0.3.1" ], entry_points="""