From 68b6644bec5274784d1217e23eeddced3c3291a7 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 12 Dec 2023 16:49:03 +0100 Subject: [PATCH] test: Slightly refactor fixed config values into `tests/settings.py` (#312) Dear Derek and Edgar, first things first: Thank you so much for your excellent work on MeltanoLabs' `{tap,target}-postgres`. We have been able to build upon them at [^1][^2] without much ado. We fear we can't contribute anything significant back for the time being, other than cosmetic changes, if you like them. In this spirit, this patch makes a humble start. It was derived from https://github.com/crate-workbench/meltano-tap-cratedb/pull/1, and pulls the database-specific configuration values into a `tests/settings.py` file, to make it easier to adjust them. With kind regards, Andreas. [^1]: https://github.com/crate-workbench/meltano-tap-cratedb [^2]: https://github.com/crate-workbench/meltano-target-cratedb --- .gitignore | 1 + tests/__init__.py | 0 tests/settings.py | 5 +++++ tests/test_core.py | 17 +++++++++-------- tests/test_replication_key.py | 7 ++++--- tests/test_selected_columns_only.py | 3 ++- 6 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/settings.py diff --git a/.gitignore b/.gitignore index 00e1675b..2a915d9c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea .vscode/** # Secrets and internal config files **/.secrets/* diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/settings.py b/tests/settings.py new file mode 100644 index 00000000..b29bfc85 --- /dev/null +++ b/tests/settings.py @@ -0,0 +1,5 @@ +# PostgreSQL default schema name. +DB_SCHEMA_NAME = "public" + +# SQLAlchemy connection URL. +DB_SQLALCHEMY_URL = "postgresql://postgres:postgres@localhost:5432/postgres" diff --git a/tests/test_core.py b/tests/test_core.py index 50f470de..52e36425 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -11,8 +11,9 @@ from singer_sdk.testing.runners import TapTestRunner from sqlalchemy import Column, DateTime, Integer, MetaData, Numeric, String, Table, text from sqlalchemy.dialects.postgresql import BIGINT, DATE, JSON, JSONB, TIME, TIMESTAMP -from test_replication_key import TABLE_NAME, TapTestReplicationKey -from test_selected_columns_only import ( +from tests.settings import DB_SCHEMA_NAME, DB_SQLALCHEMY_URL +from tests.test_replication_key import TABLE_NAME, TapTestReplicationKey +from tests.test_selected_columns_only import ( TABLE_NAME_SELECTED_COLUMNS_ONLY, TapTestSelectedColumnsOnly, ) @@ -21,7 +22,7 @@ SAMPLE_CONFIG = { "start_date": pendulum.datetime(2022, 11, 1).to_iso8601_string(), - "sqlalchemy_url": "postgresql://postgres:postgres@localhost:5432/postgres", + "sqlalchemy_url": DB_SQLALCHEMY_URL, } NO_SQLALCHEMY_CONFIG = { @@ -159,7 +160,7 @@ def test_temporal_datatypes(): conn.execute(insert) tap = TapPostgres(config=SAMPLE_CONFIG) tap_catalog = json.loads(tap.catalog_json_text) - altered_table_name = f"public-{table_name}" + altered_table_name = f"{DB_SCHEMA_NAME}-{table_name}" for stream in tap_catalog["streams"]: if stream.get("stream") and altered_table_name not in stream["stream"]: for metadata in stream["metadata"]: @@ -217,7 +218,7 @@ def test_jsonb_json(): conn.execute(insert) tap = TapPostgres(config=SAMPLE_CONFIG) tap_catalog = json.loads(tap.catalog_json_text) - altered_table_name = f"public-{table_name}" + altered_table_name = f"{DB_SCHEMA_NAME}-{table_name}" for stream in tap_catalog["streams"]: if stream.get("stream") and altered_table_name not in stream["stream"]: for metadata in stream["metadata"]: @@ -268,7 +269,7 @@ def test_decimal(): conn.execute(insert) tap = TapPostgres(config=SAMPLE_CONFIG) tap_catalog = json.loads(tap.catalog_json_text) - altered_table_name = f"public-{table_name}" + altered_table_name = f"{DB_SCHEMA_NAME}-{table_name}" for stream in tap_catalog["streams"]: if stream.get("stream") and altered_table_name not in stream["stream"]: for metadata in stream["metadata"]: @@ -354,7 +355,7 @@ def test_invalid_python_dates(): tap = TapPostgres(config=SAMPLE_CONFIG) # Alter config and then check the data comes through as a string tap_catalog = json.loads(tap.catalog_json_text) - altered_table_name = f"public-{table_name}" + altered_table_name = f"{DB_SCHEMA_NAME}-{table_name}" for stream in tap_catalog["streams"]: if stream.get("stream") and altered_table_name not in stream["stream"]: for metadata in stream["metadata"]: @@ -376,7 +377,7 @@ def test_invalid_python_dates(): copied_config["dates_as_string"] = True tap = TapPostgres(config=copied_config) tap_catalog = json.loads(tap.catalog_json_text) - altered_table_name = f"public-{table_name}" + altered_table_name = f"{DB_SCHEMA_NAME}-{table_name}" for stream in tap_catalog["streams"]: if stream.get("stream") and altered_table_name not in stream["stream"]: for metadata in stream["metadata"]: diff --git a/tests/test_replication_key.py b/tests/test_replication_key.py index d64f903d..41118dda 100644 --- a/tests/test_replication_key.py +++ b/tests/test_replication_key.py @@ -9,13 +9,14 @@ from singer_sdk.testing.templates import TapTestTemplate from sqlalchemy import Column, MetaData, String, Table from sqlalchemy.dialects.postgresql import TIMESTAMP +from tests.settings import DB_SCHEMA_NAME, DB_SQLALCHEMY_URL from tap_postgres.tap import TapPostgres TABLE_NAME = "test_replication_key" SAMPLE_CONFIG = { "start_date": pendulum.datetime(2022, 11, 1).to_iso8601_string(), - "sqlalchemy_url": "postgresql://postgres:postgres@localhost:5432/postgres", + "sqlalchemy_url": DB_SQLALCHEMY_URL, } @@ -80,7 +81,7 @@ def test_null_replication_key_with_start_date(): conn.execute(insert) tap = TapPostgres(config=SAMPLE_CONFIG) tap_catalog = json.loads(tap.catalog_json_text) - altered_table_name = f"public-{table_name}" + altered_table_name = f"{DB_SCHEMA_NAME}-{table_name}" for stream in tap_catalog["streams"]: if stream.get("stream") and altered_table_name not in stream["stream"]: for metadata in stream["metadata"]: @@ -137,7 +138,7 @@ def test_null_replication_key_without_start_date(): conn.execute(insert) tap = TapPostgres(config=modified_config) tap_catalog = json.loads(tap.catalog_json_text) - altered_table_name = f"public-{table_name}" + altered_table_name = f"{DB_SCHEMA_NAME}-{table_name}" for stream in tap_catalog["streams"]: if stream.get("stream") and altered_table_name not in stream["stream"]: for metadata in stream["metadata"]: diff --git a/tests/test_selected_columns_only.py b/tests/test_selected_columns_only.py index ac21cf65..0f964b87 100644 --- a/tests/test_selected_columns_only.py +++ b/tests/test_selected_columns_only.py @@ -2,12 +2,13 @@ import json from singer_sdk.testing.templates import TapTestTemplate +from tests.settings import DB_SQLALCHEMY_URL from tap_postgres.tap import TapPostgres TABLE_NAME_SELECTED_COLUMNS_ONLY = "test_selected_columns_only" SAMPLE_CONFIG = { - "sqlalchemy_url": "postgresql://postgres:postgres@localhost:5432/postgres", + "sqlalchemy_url": DB_SQLALCHEMY_URL, }