From 5f9cfbb4d7ea2f03493e3e108554a1230bebb79e Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Thu, 17 Oct 2024 16:10:07 +0100 Subject: [PATCH 01/13] Use pytest --- tests/test_args.py | 16 +++--- tests/test_config.py | 93 +++++++++++++++++---------------- tests/test_connection.py | 43 +++++++-------- tests/test_dataframe.py | 109 +++++++++++++++++++-------------------- tests/test_decrypt.py | 12 ++--- tests/test_encrypt.py | 28 ++++------ tests/test_load.py | 101 +++++++++++++++++++----------------- tests/test_session.py | 40 +++++++------- tests/test_table.py | 36 +++++++------ tests/test_trigger.py | 57 ++++++++++---------- tests/test_view.py | 47 +++++++++-------- 11 files changed, 288 insertions(+), 294 deletions(-) diff --git a/tests/test_args.py b/tests/test_args.py index 0e193aa..55f7588 100644 --- a/tests/test_args.py +++ b/tests/test_args.py @@ -1,14 +1,14 @@ -from unittest import TestCase +import pytest from seq_dbutils import Args -class ArgsTestClass(TestCase): +@pytest.fixture(scope='session') +def args_fixture(): + return Args.initialize_args() - def setUp(self): - self.parser = Args.initialize_args() - def test_initialize_args(self): - parsed = self.parser.parse_args(['TEST']) - config = vars(parsed)['config'][0] - self.assertEqual(config, 'TEST') +def test_initialize_args(args_fixture): + parsed = args_fixture.parse_args(['TEST']) + config = vars(parsed)['config'][0] + assert config == 'TEST' diff --git a/tests/test_config.py b/tests/test_config.py index 92cf2b5..4816f08 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,4 @@ from os.path import abspath, dirname, join -from unittest import TestCase from mock import patch @@ -8,49 +7,49 @@ DATA_DIR = join(dirname(abspath(__file__)), 'data') -class ArgsTestClass(TestCase): - - @staticmethod - @patch('logging.error') - @patch('sys.exit') - def test_initialize_no_file(mock_exit, mock_error): - file = join(DATA_DIR, 'fake.ini') - Config.initialize(file) - mock_exit.assert_called_once() - - @patch('logging.info') - @patch('configparser.ConfigParser.read') - def test_initialize_ok(self, mock_read, mock_info): - file = join(DATA_DIR, 'test_initialize_ok.ini') - Config.initialize(file) - mock_read.assert_called_once_with(file) - - @patch('logging.info') - @patch('configparser.ConfigParser.get') - def test_get_section_config(self, mock_get, mock_info): - required_section = 'Mock' - required_key = 'mock' - Config.get_section_config(required_section, required_key) - mock_get.assert_called_once_with(required_section, required_key) - - @patch('logging.info') - @patch('seq_dbutils.config.Config.get_section_config') - def test_get_db_config_ok(self, mock_get_section, mock_info): - my_str = 'mock' - args = 'TEST' - mock_get_section.return_value = my_str - mock_get_section.encode.return_value = my_str - user, key, host, db = Config.get_db_config(args) - self.assertEqual(mock_get_section.call_count, 4) - assert user == my_str - assert key == b'mock' - assert host == my_str - assert db == my_str - - @staticmethod - @patch('logging.error') - @patch('logging.info') - @patch('sys.exit') - def test_get_db_config_fail(mock_exit, mock_info, mock_error): - Config.get_db_config('error') - mock_exit.assert_called_once() +@patch('logging.error') +@patch('sys.exit') +def test_initialize_no_file(mock_exit, mock_error): + file = join(DATA_DIR, 'fake.ini') + Config.initialize(file) + mock_exit.assert_called_once() + + +@patch('logging.info') +@patch('configparser.ConfigParser.read') +def test_initialize_ok(mock_read, mock_info): + file = join(DATA_DIR, 'test_initialize_ok.ini') + Config.initialize(file) + mock_read.assert_called_once_with(file) + + +@patch('logging.info') +@patch('configparser.ConfigParser.get') +def test_get_section_config(mock_get, mock_info): + required_section = 'Mock' + required_key = 'mock' + Config.get_section_config(required_section, required_key) + mock_get.assert_called_once_with(required_section, required_key) + + +@patch('logging.info') +@patch('seq_dbutils.config.Config.get_section_config') +def test_get_db_config_ok(mock_get_section, mock_info): + my_str = 'mock' + args = 'TEST' + mock_get_section.return_value = my_str + mock_get_section.encode.return_value = my_str + user, key, host, db = Config.get_db_config(args) + assert mock_get_section.call_count == 4 + assert user == my_str + assert key == b'mock' + assert host == my_str + assert db == my_str + + +@patch('logging.error') +@patch('logging.info') +@patch('sys.exit') +def test_get_db_config_fail(mock_exit, mock_info, mock_error): + Config.get_db_config('error') + mock_exit.assert_called_once() diff --git a/tests/test_connection.py b/tests/test_connection.py index f18e41f..e443617 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1,32 +1,25 @@ -from unittest import TestCase - -from mock import patch, Mock +import pytest +from mock import patch from seq_dbutils import Connection -class ConnectionTestClass(TestCase): +@pytest.fixture(scope='session') +def connection_fixture(): + return Connection('me', 'mypassword', 'myhost', 'mydb') + - def setUp(self): - self.user = 'me' - self.pwd = 'mypassword' - self.host = 'myhost' - self.db = 'mydb' - self.connection = Connection(self.user, self.pwd, self.host, self.db) - self.connector_type = 'mysqlconnector' +def test_create_sql_engine_ok(connection_fixture): + with patch('logging.info'): + with patch('sqlalchemy.create_engine') as mock_create: + connection_fixture.create_sql_engine() + mock_create.assert_called_once_with('mysql+mysqlconnector://me:mypassword@myhost/mydb', echo=False) - @patch('logging.info') - @patch('sqlalchemy.create_engine') - def test_create_sql_engine_ok(self, mock_create, mock_info): - self.connection.create_sql_engine() - mock_create.assert_called_once_with( - f'mysql+{self.connector_type}://{self.user}:{self.pwd}@{self.host}/{self.db}', echo=False) - @patch('logging.error') - @patch('logging.info') - @patch('sys.exit') - @patch('sqlalchemy.create_engine') - def test_create_sql_engine_fail(self, mock_create, mock_exit, mock_info, mock_error): - mock_create.side_effect = Mock(side_effect=Exception()) - self.connection.create_sql_engine() - mock_exit.assert_called_once() +def test_create_sql_engine_fail(connection_fixture): + with patch('logging.info'): + with patch('logging.error'): + with patch('sys.exit') as mock_exit: + with patch('sqlalchemy.create_engine', side_effect=Exception()): + connection_fixture.create_sql_engine() + mock_exit.assert_called_once() diff --git a/tests/test_dataframe.py b/tests/test_dataframe.py index 8bb314b..41044f1 100644 --- a/tests/test_dataframe.py +++ b/tests/test_dataframe.py @@ -1,6 +1,5 @@ import datetime -from unittest import TestCase - +import pytest import pandas as pd from mock import patch from mock_alchemy.mocking import AlchemyMagicMock @@ -9,57 +8,55 @@ from seq_dbutils import DataFrameUtils -class DataFrameUtilsTestClass(TestCase): - - def test_apply_date_format_value_blank(self): - result = DataFrameUtils.apply_date_format(None, '%Y-%m-%d') - self.assertIsNone(result) - - @staticmethod - def test_apply_date_format_ok(): - input_date = '2023-10-25' - date_format = '%Y-%m-%d' - result = DataFrameUtils.apply_date_format(input_date, date_format) - expected = datetime.date(2023, 10, 25) - assert result == expected - - @staticmethod - @patch('logging.error') - @patch('sys.exit') - def test_apply_date_format_error(mock_exit, mock_error): - input_date = 'xxxxxxxxxxxx' - date_format = '%Y-%m-%d' - DataFrameUtils.apply_date_format(input_date, date_format) - mock_exit.assert_called_once() - - @staticmethod - def test_apply_date_format_value_unconverted(): - input_date = '2023-10-25 00:00:00' - date_format = '%Y-%m-%d' - result = DataFrameUtils.apply_date_format(input_date, date_format) - expected = datetime.date(2023, 10, 25) - assert result == expected - - @staticmethod - @patch('pandas.read_sql') - def test_get_db_table_col_list(mock_sql): - mock_engine = AlchemyMagicMock(spec=Engine) - DataFrameUtils(mock_engine, 'Test').get_db_table_col_list() - mock_sql.assert_called_once_with('SHOW COLUMNS FROM Test;', mock_engine) - - @staticmethod - @patch('seq_dbutils.DataFrameUtils.get_db_table_col_list', return_value=['col1', 'col3']) - def test_create_db_table_dataframe(mock_get): - mock_engine = AlchemyMagicMock(spec=Engine) - df = pd.DataFrame(data={ - 'col1': ['a', 'b', None], - 'col2': ['some data', 'some more data', None], - 'col3': [None, None, None], - }, columns=['col1', 'col2', 'col3']) - df_result = DataFrameUtils(mock_engine, 'Test').create_db_table_dataframe(df) - df_expected = pd.DataFrame(data={ - 'col1': ['a', 'b'], - 'col3': [None, None], - }, columns=['col1', 'col3']) - mock_get.assert_called_once() - pd.testing.assert_frame_equal(df_result, df_expected) +def test_apply_date_format_value_blank(): + result = DataFrameUtils.apply_date_format(None, '%Y-%m-%d') + assert result is None + + +def test_apply_date_format_ok(): + input_date = '2023-10-25' + date_format = '%Y-%m-%d' + result = DataFrameUtils.apply_date_format(input_date, date_format) + expected = datetime.date(2023, 10, 25) + assert result == expected + + +@patch('logging.error') +@patch('sys.exit') +def test_apply_date_format_error(mock_exit, mock_error): + input_date = 'xxxxxxxxxxxx' + date_format = '%Y-%m-%d' + DataFrameUtils.apply_date_format(input_date, date_format) + mock_exit.assert_called_once() + + +def test_apply_date_format_value_unconverted(): + input_date = '2023-10-25 00:00:00' + date_format = '%Y-%m-%d' + result = DataFrameUtils.apply_date_format(input_date, date_format) + expected = datetime.date(2023, 10, 25) + assert result == expected + + +@patch('pandas.read_sql') +def test_get_db_table_col_list(mock_sql): + mock_engine = AlchemyMagicMock(spec=Engine) + DataFrameUtils(mock_engine, 'Test').get_db_table_col_list() + mock_sql.assert_called_once_with('SHOW COLUMNS FROM Test;', mock_engine) + + +@patch('seq_dbutils.DataFrameUtils.get_db_table_col_list', return_value=['col1', 'col3']) +def test_create_db_table_dataframe(mock_get): + mock_engine = AlchemyMagicMock(spec=Engine) + df = pd.DataFrame(data={ + 'col1': ['a', 'b', None], + 'col2': ['some data', 'some more data', None], + 'col3': [None, None, None], + }, columns=['col1', 'col2', 'col3']) + df_result = DataFrameUtils(mock_engine, 'Test').create_db_table_dataframe(df) + df_expected = pd.DataFrame(data={ + 'col1': ['a', 'b'], + 'col3': [None, None], + }, columns=['col1', 'col3']) + mock_get.assert_called_once() + pd.testing.assert_frame_equal(df_result, df_expected) diff --git a/tests/test_decrypt.py b/tests/test_decrypt.py index 51624ce..3b9557e 100644 --- a/tests/test_decrypt.py +++ b/tests/test_decrypt.py @@ -1,5 +1,4 @@ from os.path import abspath, dirname, join -from unittest import TestCase import seq_dbutils @@ -8,10 +7,7 @@ seq_dbutils.decrypt.BIN_FILE = join(DATA_DIR, 'test_decrypt.bin') -class DecryptTestClass(TestCase): - - @staticmethod - def test_initialize(): - key = '-zITTaJ8LJ_JFjsa6EG3ASlL-yZsxEYRmCX_wjaW34I=' - result = seq_dbutils.Decrypt.initialize(key) - assert result == 'password' +def test_initialize(): + key = '-zITTaJ8LJ_JFjsa6EG3ASlL-yZsxEYRmCX_wjaW34I=' + result = seq_dbutils.Decrypt.initialize(key) + assert result == 'password' diff --git a/tests/test_encrypt.py b/tests/test_encrypt.py index be57fe1..6ce3050 100644 --- a/tests/test_encrypt.py +++ b/tests/test_encrypt.py @@ -1,8 +1,6 @@ -from os import remove -from os.path import abspath, dirname, join, isfile -from unittest import TestCase +from os.path import abspath, dirname, join -from mock import patch +from mock import patch, mock_open import seq_dbutils @@ -12,17 +10,11 @@ seq_dbutils.encrypt.BIN_FILE = TEST_BIN_FILE -class EncryptTestClass(TestCase): - - @staticmethod - @patch('logging.info') - @patch('seq_dbutils.encrypt.getpass') - def test_initialize(mock_pass, mock_info): - mock_pass.return_value = 'password' - seq_dbutils.Encrypt.initialize() - assert isfile(TEST_BIN_FILE) - - @staticmethod - def tearDown(**kwargs): - if isfile(TEST_BIN_FILE): - remove(TEST_BIN_FILE) +def test_initialize(): + key = b'ZmDfcTF7_60GrrY167zsiPd67pEvs0aGOv2oasOM1Pg=' + with patch('logging.info'): + with patch('seq_dbutils.encrypt.getpass', return_value='password'): + with patch('cryptography.fernet.Fernet.generate_key', return_value=key): + with patch('builtins.open', mock_open()) as mock_file: + seq_dbutils.Encrypt.initialize() + mock_file.assert_called_once() diff --git a/tests/test_load.py b/tests/test_load.py index 3aa40ba..ae3ec5f 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -1,6 +1,5 @@ -from unittest import TestCase - import pandas as pd +import pytest from mock import patch, Mock from sqlalchemy import Column, String, Float from sqlalchemy.orm import declarative_base @@ -19,54 +18,64 @@ class MockTable(BASE): mysql_charset = 'utf8' -class LoadTestClass(TestCase): +@pytest.fixture(scope='session') +@patch('sqlalchemy.orm.sessionmaker') +def session_fixture(mock_session): + return mock_session() - @patch('sqlalchemy.orm.sessionmaker') - def setUp(self, mock_session): - self.mock_instance = mock_session() - self.df_data = pd.DataFrame(data={'id1': ['a', 'b', 'c'], - 'id2': ['d', 'b', 'f'], - 'id3': ['g', 'h', 'i']}, - columns=['id1', 'id2', 'id3']) - @patch('logging.info') - def test_bulk_insert_df_table_empty(self, mock_info): - df = pd.DataFrame() - Load(df, self.mock_instance, MockTable).bulk_insert_df_table() +@pytest.fixture(scope='session') +def dataframe_fixture(): + df_data = pd.DataFrame(data={'id1': ['a', 'b', 'c'], + 'id2': ['d', 'b', 'f'], + 'id3': ['g', 'h', 'i']}, + columns=['id1', 'id2', 'id3']) + return df_data + + +def test_bulk_insert_df_table_empty(session_fixture): + df = pd.DataFrame() + with patch('logging.info') as mock_info: + Load(df, session_fixture, MockTable).bulk_insert_df_table() mock_info.assert_called_with('Skipping bulk insert for table \'Mock\' and empty dataframe') - @patch('logging.info') - def test_bulk_insert_df_table_ok(self, mock_info): - Load(self.df_data, self.mock_instance, MockTable).bulk_insert_df_table() - self.mock_instance.bulk_insert_mappings.assert_called_once() - - @patch('logging.error') - @patch('logging.info') - @patch('sys.exit') - def test_bulk_insert_df_table_fail(self, mock_exit, mock_info, mock_error): - self.mock_instance.bulk_insert_mappings = Mock(side_effect=Exception()) - self.mock_instance.rollback = Mock() - Load(self.df_data, self.mock_instance, MockTable).bulk_insert_df_table() - self.mock_instance.rollback.assert_called_once() - mock_exit.assert_called_once() - - @patch('logging.info') - def test_bulk_update_df_table_empty(self, mock_info): + +def test_bulk_insert_df_table_ok(session_fixture, dataframe_fixture): + with patch('logging.info'): + Load(dataframe_fixture, session_fixture, MockTable).bulk_insert_df_table() + session_fixture.bulk_insert_mappings.assert_called_once() + + +def test_bulk_insert_df_table_fail(session_fixture, dataframe_fixture): + with patch('logging.info'): + with patch('logging.error'): + with patch('sys.exit') as mock_exit: + session_fixture.bulk_insert_mappings = Mock(side_effect=Exception()) + session_fixture.rollback = Mock() + Load(dataframe_fixture, session_fixture, MockTable).bulk_insert_df_table() + session_fixture.rollback.assert_called_once() + mock_exit.assert_called_once() + + +def test_bulk_update_df_table_empty(session_fixture): + with patch('logging.info') as mock_info: df = pd.DataFrame() - Load(df, self.mock_instance, MockTable).bulk_update_df_table() + Load(df, session_fixture, MockTable).bulk_update_df_table() mock_info.assert_called_with('Skipping bulk update for table \'Mock\' and empty dataframe') - @patch('logging.info') - def test_bulk_update_df_table_ok(self, mock_info): - Load(self.df_data, self.mock_instance, MockTable).bulk_update_df_table() - self.mock_instance.bulk_update_mappings.assert_called_once() - - @patch('logging.error') - @patch('logging.info') - @patch('sys.exit') - def test_bulk_update_df_table_fail(self, mock_exit, mock_info, mock_error): - self.mock_instance.bulk_update_mappings = Mock(side_effect=Exception()) - self.mock_instance.rollback = Mock() - Load(self.df_data, self.mock_instance, MockTable).bulk_update_df_table() - self.mock_instance.rollback.assert_called_once() - mock_exit.assert_called_once() + +def test_bulk_update_df_table_ok(session_fixture, dataframe_fixture): + with patch('logging.info'): + Load(dataframe_fixture, session_fixture, MockTable).bulk_update_df_table() + session_fixture.bulk_update_mappings.assert_called_once() + + +def test_bulk_update_df_table_fail(session_fixture, dataframe_fixture): + with patch('logging.info'): + with patch('logging.error'): + with patch('sys.exit') as mock_exit: + session_fixture.bulk_update_mappings = Mock(side_effect=Exception()) + session_fixture.rollback = Mock() + Load(dataframe_fixture, session_fixture, MockTable).bulk_update_df_table() + session_fixture.rollback.assert_called_once() + mock_exit.assert_called_once() diff --git a/tests/test_session.py b/tests/test_session.py index dc82295..87f9a94 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -1,29 +1,29 @@ -from unittest import TestCase - +import pytest from mock import patch from mock_alchemy.mocking import AlchemyMagicMock -from sqlalchemy.sql import text from seq_dbutils import Session -class SessionTestClass(TestCase): +@pytest.fixture(scope='session') +def alchemy_fixture(): + return AlchemyMagicMock() - def setUp(self): - self.mock_instance = AlchemyMagicMock() - @patch('logging.info') - def test_log_and_execute_sql(self, mock_info): +def test_log_and_execute_sql(alchemy_fixture): + with patch('logging.info'): sql = 'SELECT * FROM test;' - Session(self.mock_instance).log_and_execute_sql(sql) - self.mock_instance.execute.assert_called_once() - - @patch('logging.info') - def test_commit_changes_false(self, mock_info): - Session(self.mock_instance).commit_changes(False) - self.mock_instance.commit.assert_not_called() - - @patch('logging.info') - def test_commit_changes_true(self, mock_info): - Session(self.mock_instance).commit_changes(True) - self.mock_instance.commit.assert_called_once() + Session(alchemy_fixture).log_and_execute_sql(sql) + alchemy_fixture.execute.assert_called_once() + + +def test_commit_changes_false(alchemy_fixture): + with patch('logging.info'): + Session(alchemy_fixture).commit_changes(False) + alchemy_fixture.commit.assert_not_called() + + +def test_commit_changes_true(alchemy_fixture): + with patch('logging.info'): + Session(alchemy_fixture).commit_changes(True) + alchemy_fixture.commit.assert_called_once() diff --git a/tests/test_table.py b/tests/test_table.py index c92ec37..a7361da 100644 --- a/tests/test_table.py +++ b/tests/test_table.py @@ -1,5 +1,4 @@ -from unittest import TestCase - +import pytest from mock import patch from mock_alchemy.mocking import AlchemyMagicMock from sqlalchemy import Column, String, Float @@ -20,21 +19,26 @@ class MockTable(Table, BASE): mysql_charset = 'utf8' -class TableTestClass(TestCase): +@pytest.fixture(scope='session') +def engine_fixture(): + return AlchemyMagicMock(spec=Engine) + + +@pytest.fixture(scope='session') +def table_fixture(engine_fixture): + mock_engine = engine_fixture + return Table(mock_engine, MockTable) - def setUp(self): - self.mock_engine = AlchemyMagicMock(spec=Engine) - self.table = Table(self.mock_engine, MockTable) - @patch('logging.info') - @patch('sqlalchemy.schema.Table.drop') - def test_drop_table(self, mock_drop, mock_info): - self.table.drop_table() - mock_drop.assert_called_once_with(self.mock_engine) +def test_drop_table(engine_fixture, table_fixture): + with patch('logging.info'): + with patch('sqlalchemy.schema.Table.drop') as mock_drop: + table_fixture.drop_table() + mock_drop.assert_called_once_with(engine_fixture) - @patch('logging.info') - @patch('sqlalchemy.schema.Table.create') - def test_create_table(self, mock_create, mock_info): - self.table.create_table() - mock_create.assert_called_once_with(self.mock_engine) +def test_create_table(engine_fixture, table_fixture): + with patch('logging.info'): + with patch('sqlalchemy.schema.Table.create') as mock_create: + table_fixture.create_table() + mock_create.assert_called_once_with(engine_fixture) diff --git a/tests/test_trigger.py b/tests/test_trigger.py index 53936ad..0bc0788 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -1,6 +1,6 @@ from os.path import abspath, dirname, join -from unittest import TestCase +import pytest from mock import patch from seq_dbutils import Trigger @@ -8,30 +8,31 @@ DATA_DIR = join(dirname(abspath(__file__)), 'data') -class TriggerTestClass(TestCase): - - @patch('sqlalchemy.orm.sessionmaker') - def setUp(self, mock_session): - self.mock_instance = mock_session() - self.trigger_name = 'test_trigger' - self.trigger_filepath = join(DATA_DIR, f'{self.trigger_name}.sql') - self.trigger = Trigger(self.trigger_filepath, self.mock_instance) - - @patch('logging.info') - def test_drop_trigger_if_exists(self, mock_info): - self.trigger.drop_trigger_if_exists() - sql = f"DROP TRIGGER IF EXISTS {self.trigger_name};" - self.mock_instance.execute.assert_called_once() - - @patch('logging.info') - def test_create_trigger(self, mock_info): - self.trigger.create_trigger() - sql = f"""CREATE TRIGGER {self.trigger_name} -BEFORE UPDATE ON Pt - FOR EACH ROW SET NEW.modified = CURRENT_TIMESTAMP;""" - self.mock_instance.execute.assert_called_once() - - @patch('logging.info') - def test_drop_and_create_trigger(self, mock_info): - self.trigger.drop_and_create_trigger() - self.assertEqual(self.mock_instance.execute.call_count, 2) +@pytest.fixture(scope='function') +def instance_fixture(): + with patch('sqlalchemy.orm.sessionmaker') as mock_session: + return mock_session() + + +@pytest.fixture(scope='function') +def trigger_fixture(instance_fixture): + trigger_filepath = join(DATA_DIR, 'test_trigger.sql') + return Trigger(trigger_filepath, instance_fixture) + + +def test_drop_trigger_if_exists(instance_fixture, trigger_fixture): + with patch('logging.info'): + trigger_fixture.drop_trigger_if_exists() + instance_fixture.execute.assert_called_once() + + +def test_create_trigger(instance_fixture, trigger_fixture): + with patch('logging.info'): + trigger_fixture.create_trigger() + instance_fixture.execute.assert_called_once() + + +def test_drop_and_create_trigger(instance_fixture, trigger_fixture): + with patch('logging.info'): + trigger_fixture.drop_and_create_trigger() + assert instance_fixture.execute.call_count == 2 diff --git a/tests/test_view.py b/tests/test_view.py index 948fdae..1782ded 100644 --- a/tests/test_view.py +++ b/tests/test_view.py @@ -1,6 +1,6 @@ from os.path import abspath, dirname, join -from unittest import TestCase +import pytest from mock import patch from seq_dbutils import View @@ -8,28 +8,31 @@ DATA_DIR = join(dirname(abspath(__file__)), 'data') -class ViewTestClass(TestCase): +@pytest.fixture(scope='function') +def instance_fixture(): + with patch('sqlalchemy.orm.sessionmaker') as mock_session: + return mock_session() - @patch('sqlalchemy.orm.sessionmaker') - def setUp(self, mock_session): - self.mock_instance = mock_session() - self.view_name = 'test_view' - self.view_filepath = join(DATA_DIR, f'{self.view_name}.sql') - self.view = View(self.view_filepath, self.mock_instance) - @patch('logging.info') - def test_drop_view_if_exists(self, mock_info): - self.view.drop_view_if_exists(self.mock_instance, self.view_name) - sql = f'DROP VIEW IF EXISTS {self.view_name};' - self.mock_instance.execute.assert_called_once() +@pytest.fixture(scope='function') +def view_fixture(instance_fixture): + view_filepath = join(DATA_DIR, 'test_view.sql') + return View(view_filepath, instance_fixture) - @patch('logging.info') - def test_create_view(self, mock_info): - self.view.create_view() - sql = f'CREATE VIEW {self.view_name} AS \nSELECT * FROM Pt;' - self.mock_instance.execute.assert_called_once() - @patch('logging.info') - def test_drop_and_create_view(self, mock_info): - self.view.drop_and_create_view() - self.assertEqual(self.mock_instance.execute.call_count, 2) +def test_drop_view_if_exists(instance_fixture, view_fixture): + with patch('logging.info'): + view_fixture.drop_view_if_exists(instance_fixture, 'test_view') + instance_fixture.execute.assert_called_once() + + +def test_create_view(instance_fixture, view_fixture): + with patch('logging.info'): + view_fixture.create_view() + instance_fixture.execute.assert_called_once() + + +def test_drop_and_create_view(instance_fixture, view_fixture): + with patch('logging.info'): + view_fixture.drop_and_create_view() + assert instance_fixture.execute.call_count == 2 From e69db3ba3ddffec696f5fc886edcf88221bb72e2 Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Fri, 18 Oct 2024 14:45:25 +0100 Subject: [PATCH 02/13] Fixture renaming / change scopes --- tests/test_args.py | 8 +++---- tests/test_connection.py | 12 +++++----- tests/test_dataframe.py | 2 +- tests/test_load.py | 51 ++++++++++++++++++++-------------------- tests/test_session.py | 20 ++++++++-------- tests/test_table.py | 18 +++++++------- tests/test_trigger.py | 28 +++++++++++----------- tests/test_view.py | 28 +++++++++++----------- 8 files changed, 84 insertions(+), 83 deletions(-) diff --git a/tests/test_args.py b/tests/test_args.py index 55f7588..5d446f7 100644 --- a/tests/test_args.py +++ b/tests/test_args.py @@ -3,12 +3,12 @@ from seq_dbutils import Args -@pytest.fixture(scope='session') -def args_fixture(): +@pytest.fixture() +def args(): return Args.initialize_args() -def test_initialize_args(args_fixture): - parsed = args_fixture.parse_args(['TEST']) +def test_initialize_args(args): + parsed = args.parse_args(['TEST']) config = vars(parsed)['config'][0] assert config == 'TEST' diff --git a/tests/test_connection.py b/tests/test_connection.py index e443617..e9ec5b5 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -4,22 +4,22 @@ from seq_dbutils import Connection -@pytest.fixture(scope='session') -def connection_fixture(): +@pytest.fixture() +def connection(): return Connection('me', 'mypassword', 'myhost', 'mydb') -def test_create_sql_engine_ok(connection_fixture): +def test_create_sql_engine_ok(connection): with patch('logging.info'): with patch('sqlalchemy.create_engine') as mock_create: - connection_fixture.create_sql_engine() + connection.create_sql_engine() mock_create.assert_called_once_with('mysql+mysqlconnector://me:mypassword@myhost/mydb', echo=False) -def test_create_sql_engine_fail(connection_fixture): +def test_create_sql_engine_fail(connection): with patch('logging.info'): with patch('logging.error'): with patch('sys.exit') as mock_exit: with patch('sqlalchemy.create_engine', side_effect=Exception()): - connection_fixture.create_sql_engine() + connection.create_sql_engine() mock_exit.assert_called_once() diff --git a/tests/test_dataframe.py b/tests/test_dataframe.py index 41044f1..48c9502 100644 --- a/tests/test_dataframe.py +++ b/tests/test_dataframe.py @@ -1,5 +1,5 @@ import datetime -import pytest + import pandas as pd from mock import patch from mock_alchemy.mocking import AlchemyMagicMock diff --git a/tests/test_load.py b/tests/test_load.py index ae3ec5f..44ecbd7 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -18,14 +18,15 @@ class MockTable(BASE): mysql_charset = 'utf8' -@pytest.fixture(scope='session') -@patch('sqlalchemy.orm.sessionmaker') -def session_fixture(mock_session): - return mock_session() +@pytest.fixture() +def instance(): + with patch('sqlalchemy.orm.sessionmaker') as mock_session: + return mock_session() + @pytest.fixture(scope='session') -def dataframe_fixture(): +def dataframe(): df_data = pd.DataFrame(data={'id1': ['a', 'b', 'c'], 'id2': ['d', 'b', 'f'], 'id3': ['g', 'h', 'i']}, @@ -33,49 +34,49 @@ def dataframe_fixture(): return df_data -def test_bulk_insert_df_table_empty(session_fixture): +def test_bulk_insert_df_table_empty(instance): df = pd.DataFrame() with patch('logging.info') as mock_info: - Load(df, session_fixture, MockTable).bulk_insert_df_table() + Load(df, instance, MockTable).bulk_insert_df_table() mock_info.assert_called_with('Skipping bulk insert for table \'Mock\' and empty dataframe') -def test_bulk_insert_df_table_ok(session_fixture, dataframe_fixture): +def test_bulk_insert_df_table_ok(instance, dataframe): with patch('logging.info'): - Load(dataframe_fixture, session_fixture, MockTable).bulk_insert_df_table() - session_fixture.bulk_insert_mappings.assert_called_once() + Load(dataframe, instance, MockTable).bulk_insert_df_table() + instance.bulk_insert_mappings.assert_called_once() -def test_bulk_insert_df_table_fail(session_fixture, dataframe_fixture): +def test_bulk_insert_df_table_fail(instance, dataframe): with patch('logging.info'): with patch('logging.error'): with patch('sys.exit') as mock_exit: - session_fixture.bulk_insert_mappings = Mock(side_effect=Exception()) - session_fixture.rollback = Mock() - Load(dataframe_fixture, session_fixture, MockTable).bulk_insert_df_table() - session_fixture.rollback.assert_called_once() + instance.bulk_insert_mappings = Mock(side_effect=Exception()) + instance.rollback = Mock() + Load(dataframe, instance, MockTable).bulk_insert_df_table() + instance.rollback.assert_called_once() mock_exit.assert_called_once() -def test_bulk_update_df_table_empty(session_fixture): +def test_bulk_update_df_table_empty(instance): with patch('logging.info') as mock_info: df = pd.DataFrame() - Load(df, session_fixture, MockTable).bulk_update_df_table() + Load(df, instance, MockTable).bulk_update_df_table() mock_info.assert_called_with('Skipping bulk update for table \'Mock\' and empty dataframe') -def test_bulk_update_df_table_ok(session_fixture, dataframe_fixture): +def test_bulk_update_df_table_ok(instance, dataframe): with patch('logging.info'): - Load(dataframe_fixture, session_fixture, MockTable).bulk_update_df_table() - session_fixture.bulk_update_mappings.assert_called_once() + Load(dataframe, instance, MockTable).bulk_update_df_table() + instance.bulk_update_mappings.assert_called_once() -def test_bulk_update_df_table_fail(session_fixture, dataframe_fixture): +def test_bulk_update_df_table_fail(instance, dataframe): with patch('logging.info'): with patch('logging.error'): with patch('sys.exit') as mock_exit: - session_fixture.bulk_update_mappings = Mock(side_effect=Exception()) - session_fixture.rollback = Mock() - Load(dataframe_fixture, session_fixture, MockTable).bulk_update_df_table() - session_fixture.rollback.assert_called_once() + instance.bulk_update_mappings = Mock(side_effect=Exception()) + instance.rollback = Mock() + Load(dataframe, instance, MockTable).bulk_update_df_table() + instance.rollback.assert_called_once() mock_exit.assert_called_once() diff --git a/tests/test_session.py b/tests/test_session.py index 87f9a94..713d983 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -6,24 +6,24 @@ @pytest.fixture(scope='session') -def alchemy_fixture(): +def session(): return AlchemyMagicMock() -def test_log_and_execute_sql(alchemy_fixture): +def test_log_and_execute_sql(session): with patch('logging.info'): sql = 'SELECT * FROM test;' - Session(alchemy_fixture).log_and_execute_sql(sql) - alchemy_fixture.execute.assert_called_once() + Session(session).log_and_execute_sql(sql) + session.execute.assert_called_once() -def test_commit_changes_false(alchemy_fixture): +def test_commit_changes_false(session): with patch('logging.info'): - Session(alchemy_fixture).commit_changes(False) - alchemy_fixture.commit.assert_not_called() + Session(session).commit_changes(False) + session.commit.assert_not_called() -def test_commit_changes_true(alchemy_fixture): +def test_commit_changes_true(session): with patch('logging.info'): - Session(alchemy_fixture).commit_changes(True) - alchemy_fixture.commit.assert_called_once() + Session(session).commit_changes(True) + session.commit.assert_called_once() diff --git a/tests/test_table.py b/tests/test_table.py index a7361da..e417bed 100644 --- a/tests/test_table.py +++ b/tests/test_table.py @@ -20,25 +20,25 @@ class MockTable(Table, BASE): @pytest.fixture(scope='session') -def engine_fixture(): +def engine(): return AlchemyMagicMock(spec=Engine) @pytest.fixture(scope='session') -def table_fixture(engine_fixture): - mock_engine = engine_fixture +def table(engine): + mock_engine = engine return Table(mock_engine, MockTable) -def test_drop_table(engine_fixture, table_fixture): +def test_drop_table(engine, table): with patch('logging.info'): with patch('sqlalchemy.schema.Table.drop') as mock_drop: - table_fixture.drop_table() - mock_drop.assert_called_once_with(engine_fixture) + table.drop_table() + mock_drop.assert_called_once_with(engine) -def test_create_table(engine_fixture, table_fixture): +def test_create_table(engine, table): with patch('logging.info'): with patch('sqlalchemy.schema.Table.create') as mock_create: - table_fixture.create_table() - mock_create.assert_called_once_with(engine_fixture) + table.create_table() + mock_create.assert_called_once_with(engine) diff --git a/tests/test_trigger.py b/tests/test_trigger.py index 0bc0788..d2f0351 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -8,31 +8,31 @@ DATA_DIR = join(dirname(abspath(__file__)), 'data') -@pytest.fixture(scope='function') -def instance_fixture(): +@pytest.fixture() +def instance(): with patch('sqlalchemy.orm.sessionmaker') as mock_session: return mock_session() -@pytest.fixture(scope='function') -def trigger_fixture(instance_fixture): +@pytest.fixture() +def trigger(instance): trigger_filepath = join(DATA_DIR, 'test_trigger.sql') - return Trigger(trigger_filepath, instance_fixture) + return Trigger(trigger_filepath, instance) -def test_drop_trigger_if_exists(instance_fixture, trigger_fixture): +def test_drop_trigger_if_exists(instance, trigger): with patch('logging.info'): - trigger_fixture.drop_trigger_if_exists() - instance_fixture.execute.assert_called_once() + trigger.drop_trigger_if_exists() + instance.execute.assert_called_once() -def test_create_trigger(instance_fixture, trigger_fixture): +def test_create_trigger(instance, trigger): with patch('logging.info'): - trigger_fixture.create_trigger() - instance_fixture.execute.assert_called_once() + trigger.create_trigger() + instance.execute.assert_called_once() -def test_drop_and_create_trigger(instance_fixture, trigger_fixture): +def test_drop_and_create_trigger(instance, trigger): with patch('logging.info'): - trigger_fixture.drop_and_create_trigger() - assert instance_fixture.execute.call_count == 2 + trigger.drop_and_create_trigger() + assert instance.execute.call_count == 2 diff --git a/tests/test_view.py b/tests/test_view.py index 1782ded..d660867 100644 --- a/tests/test_view.py +++ b/tests/test_view.py @@ -8,31 +8,31 @@ DATA_DIR = join(dirname(abspath(__file__)), 'data') -@pytest.fixture(scope='function') -def instance_fixture(): +@pytest.fixture() +def instance(): with patch('sqlalchemy.orm.sessionmaker') as mock_session: return mock_session() -@pytest.fixture(scope='function') -def view_fixture(instance_fixture): +@pytest.fixture() +def view(instance): view_filepath = join(DATA_DIR, 'test_view.sql') - return View(view_filepath, instance_fixture) + return View(view_filepath, instance) -def test_drop_view_if_exists(instance_fixture, view_fixture): +def test_drop_view_if_exists(instance, view): with patch('logging.info'): - view_fixture.drop_view_if_exists(instance_fixture, 'test_view') - instance_fixture.execute.assert_called_once() + view.drop_view_if_exists(instance, 'test_view') + instance.execute.assert_called_once() -def test_create_view(instance_fixture, view_fixture): +def test_create_view(instance, view): with patch('logging.info'): - view_fixture.create_view() - instance_fixture.execute.assert_called_once() + view.create_view() + instance.execute.assert_called_once() -def test_drop_and_create_view(instance_fixture, view_fixture): +def test_drop_and_create_view(instance, view): with patch('logging.info'): - view_fixture.drop_and_create_view() - assert instance_fixture.execute.call_count == 2 + view.drop_and_create_view() + assert instance.execute.call_count == 2 From 7c48cf43c7acbe67308e4ae30ee7fdef5e33cc0e Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Wed, 30 Oct 2024 15:55:54 +0000 Subject: [PATCH 03/13] Refactoring --- seq_dbutils/config.py | 27 ++++------ seq_dbutils/connection.py | 13 ++--- seq_dbutils/dataframe.py | 17 ------- seq_dbutils/decrypt.py | 2 +- seq_dbutils/load.py | 11 ++-- tests/test_config.py | 104 +++++++++++++++++++++----------------- tests/test_connection.py | 8 ++- tests/test_dataframe.py | 58 +++++++-------------- tests/test_load.py | 31 +++++------- tests/test_session.py | 2 +- 10 files changed, 111 insertions(+), 162 deletions(-) diff --git a/seq_dbutils/config.py b/seq_dbutils/config.py index ebc2d83..7b93698 100644 --- a/seq_dbutils/config.py +++ b/seq_dbutils/config.py @@ -1,6 +1,4 @@ import logging -import os -import sys from configparser import ConfigParser logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') @@ -11,10 +9,10 @@ class Config: @classmethod def initialize(cls, config_file_path): - if not os.path.isfile(config_file_path): - logging.error(f'Config file {config_file_path} does not exist. Exiting...') - sys.exit(1) - cls.configParser.read(config_file_path) + try: + return cls.configParser.read(config_file_path) + except FileNotFoundError: + raise FileNotFoundError(f'Config file {config_file_path} does not exist. Exiting...') @classmethod def get_section_config(cls, required_section, required_key): @@ -22,14 +20,9 @@ def get_section_config(cls, required_section, required_key): @staticmethod def get_db_config(required_section): - assert required_section - try: - logging.info(f"Extracting config for '{required_section}'") - user = Config.get_section_config(required_section, 'user') - key = Config.get_section_config(required_section, 'key').encode() - host = Config.get_section_config(required_section, 'host') - db = Config.get_section_config(required_section, 'db') - return user, key, host, db - except Exception as ex: - logging.error(str(ex)) - sys.exit(1) + logging.info(f"Extracting config for '{required_section}'") + user = Config.get_section_config(required_section, 'user') + key = Config.get_section_config(required_section, 'key').encode() + host = Config.get_section_config(required_section, 'host') + db = Config.get_section_config(required_section, 'db') + return user, key, host, db diff --git a/seq_dbutils/connection.py b/seq_dbutils/connection.py index d85a992..4100a8a 100644 --- a/seq_dbutils/connection.py +++ b/seq_dbutils/connection.py @@ -1,5 +1,4 @@ import logging -import sys import sqlalchemy @@ -20,11 +19,7 @@ def __init__(self, user, pwd, host, db, connector_type='mysqlconnector'): self.connector_type = connector_type def create_sql_engine(self, sql_logging=False): - try: - logging.info(f'Connecting to {self.db} on host {self.host}') - conn_str = f'mysql+{self.connector_type}://{self.user}:{self.pwd}@{self.host}/{self.db}' - sql_engine = sqlalchemy.create_engine(conn_str, echo=sql_logging) - return sql_engine - except Exception as ex: - logging.error(str(ex)) - sys.exit(1) + logging.info(f'Connecting to {self.db} on host {self.host}') + conn_str = f'mysql+{self.connector_type}://{self.user}:{self.pwd}@{self.host}/{self.db}' + sql_engine = sqlalchemy.create_engine(conn_str, echo=sql_logging) + return sql_engine diff --git a/seq_dbutils/dataframe.py b/seq_dbutils/dataframe.py index 26546c7..1872bde 100644 --- a/seq_dbutils/dataframe.py +++ b/seq_dbutils/dataframe.py @@ -1,5 +1,4 @@ import logging -import sys from datetime import datetime import pandas as pd @@ -26,19 +25,3 @@ def create_db_table_dataframe(self, df): df_db_table = df.filter(db_table_col_list, axis=1) df_db_table = df_db_table.dropna(subset=df_db_table.columns, how='all') return df_db_table - - @staticmethod - def apply_date_format(input_date, format_date): - if input_date: - format_time = format_date + ' %H:%M:%S' - try: - input_date = datetime.strptime(input_date, format_date).date() - except ValueError as ex: - if 'unconverted data remains:' in ex.args[0]: - input_date = datetime.strptime(input_date, format_time).date() - else: - logging.error(str(ex)) - sys.exit(1) - else: - input_date = None - return input_date diff --git a/seq_dbutils/decrypt.py b/seq_dbutils/decrypt.py index b29cac5..708cd56 100644 --- a/seq_dbutils/decrypt.py +++ b/seq_dbutils/decrypt.py @@ -15,5 +15,5 @@ def initialize(cls, key): for line in file_object: pwd_encrypted = line pwd_decrypted = cipher_suite.decrypt(pwd_encrypted) - pwd_plain_text = bytes(pwd_decrypted).decode("utf-8") + pwd_plain_text = bytes(pwd_decrypted).decode('utf-8') return pwd_plain_text diff --git a/seq_dbutils/load.py b/seq_dbutils/load.py index 4faacb5..e7fa1f3 100644 --- a/seq_dbutils/load.py +++ b/seq_dbutils/load.py @@ -1,5 +1,4 @@ import logging -import sys import pandas as pd @@ -22,11 +21,10 @@ def bulk_insert_df_table(self): logging.info( f"Bulk inserting into table '{self.table_subclass.__tablename__}'. Number of rows: {len(self.df_table)}") self.session_instance.bulk_insert_mappings(self.table_subclass, self.df_table.to_dict(orient='records')) - except Exception as ex: + except Exception: logging.error('Failed to load data into database. Rolling back...') self.session_instance.rollback() - logging.error(str(ex)) - sys.exit(1) + raise else: logging.info(f"Skipping bulk insert for table '{self.table_subclass.__tablename__}' and empty dataframe") @@ -36,10 +34,9 @@ def bulk_update_df_table(self): logging.info( f"Bulk updating table '{self.table_subclass.__tablename__}'. Number of rows: {len(self.df_table)}") self.session_instance.bulk_update_mappings(self.table_subclass, self.df_table.to_dict(orient='records')) - except Exception as ex: + except Exception: logging.error('Failed to update data in database. Rolling back...') self.session_instance.rollback() - logging.error(str(ex)) - sys.exit(1) + raise else: logging.info(f"Skipping bulk update for table '{self.table_subclass.__tablename__}' and empty dataframe") diff --git a/tests/test_config.py b/tests/test_config.py index 4816f08..22a4263 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,7 @@ +from configparser import NoOptionError from os.path import abspath, dirname, join +import pytest from mock import patch from seq_dbutils import Config @@ -7,49 +9,59 @@ DATA_DIR = join(dirname(abspath(__file__)), 'data') -@patch('logging.error') -@patch('sys.exit') -def test_initialize_no_file(mock_exit, mock_error): - file = join(DATA_DIR, 'fake.ini') - Config.initialize(file) - mock_exit.assert_called_once() - - -@patch('logging.info') -@patch('configparser.ConfigParser.read') -def test_initialize_ok(mock_read, mock_info): - file = join(DATA_DIR, 'test_initialize_ok.ini') - Config.initialize(file) - mock_read.assert_called_once_with(file) - - -@patch('logging.info') -@patch('configparser.ConfigParser.get') -def test_get_section_config(mock_get, mock_info): - required_section = 'Mock' - required_key = 'mock' - Config.get_section_config(required_section, required_key) - mock_get.assert_called_once_with(required_section, required_key) - - -@patch('logging.info') -@patch('seq_dbutils.config.Config.get_section_config') -def test_get_db_config_ok(mock_get_section, mock_info): - my_str = 'mock' - args = 'TEST' - mock_get_section.return_value = my_str - mock_get_section.encode.return_value = my_str - user, key, host, db = Config.get_db_config(args) - assert mock_get_section.call_count == 4 - assert user == my_str - assert key == b'mock' - assert host == my_str - assert db == my_str - - -@patch('logging.error') -@patch('logging.info') -@patch('sys.exit') -def test_get_db_config_fail(mock_exit, mock_info, mock_error): - Config.get_db_config('error') - mock_exit.assert_called_once() +def test_initialize_no_file(): + with patch('configparser.ConfigParser.read', side_effect=FileNotFoundError()): + with pytest.raises(FileNotFoundError): + Config.initialize('fake.ini') + + +def test_initialize_exception(): + with patch('configparser.ConfigParser.read', side_effect=Exception()): + with pytest.raises(Exception): + Config.initialize('fake.ini') + + +def test_initialize_ok(): + data = 'file_contents' + with patch('configparser.ConfigParser.read', return_value=data): + result = Config.initialize('fake.ini') + assert result == data + + +def test_get_section_config_no_option(): + required_section = 'SOME_SECTION' + required_key = 'some_key' + with patch('configparser.ConfigParser.get', side_effect=NoOptionError(required_section, required_key)): + with pytest.raises(NoOptionError): + Config.get_section_config(required_section, required_key) + + +def test_get_section_config_exception(): + required_section = 'SOME_SECTION' + required_key = 'some_key' + with patch('configparser.ConfigParser.get', side_effect=Exception()): + with pytest.raises(Exception): + Config.get_section_config(required_section, required_key) + + +def test_get_section_config_ok(): + required_section = 'SOME_SECTION' + required_key = 'some_key' + data = 'some_config' + with patch('configparser.ConfigParser.get', return_value=data): + result = Config.get_section_config(required_section, required_key) + assert result == data + + +def test_get_db_config_ok(): + result = 'some_config' + with patch('logging.info'): + with patch('seq_dbutils.config.Config.get_section_config') as mock_get_section: + mock_get_section.return_value = result + mock_get_section.encode.return_value = result + user, key, host, db = Config.get_db_config('SOME_SECTION') + assert mock_get_section.call_count == 4 + assert user == result + assert key == b'some_config' + assert host == result + assert db == result diff --git a/tests/test_connection.py b/tests/test_connection.py index e9ec5b5..38a5430 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -18,8 +18,6 @@ def test_create_sql_engine_ok(connection): def test_create_sql_engine_fail(connection): with patch('logging.info'): - with patch('logging.error'): - with patch('sys.exit') as mock_exit: - with patch('sqlalchemy.create_engine', side_effect=Exception()): - connection.create_sql_engine() - mock_exit.assert_called_once() + with patch('sqlalchemy.create_engine', side_effect=Exception()): + with pytest.raises(Exception): + connection.create_sql_engine() diff --git a/tests/test_dataframe.py b/tests/test_dataframe.py index 48c9502..149f926 100644 --- a/tests/test_dataframe.py +++ b/tests/test_dataframe.py @@ -1,6 +1,5 @@ -import datetime - import pandas as pd +import pytest from mock import patch from mock_alchemy.mocking import AlchemyMagicMock from sqlalchemy.engine import Engine @@ -8,55 +7,32 @@ from seq_dbutils import DataFrameUtils -def test_apply_date_format_value_blank(): - result = DataFrameUtils.apply_date_format(None, '%Y-%m-%d') - assert result is None - - -def test_apply_date_format_ok(): - input_date = '2023-10-25' - date_format = '%Y-%m-%d' - result = DataFrameUtils.apply_date_format(input_date, date_format) - expected = datetime.date(2023, 10, 25) - assert result == expected - +@pytest.fixture(scope='session') +def engine(): + return AlchemyMagicMock(spec=Engine) -@patch('logging.error') -@patch('sys.exit') -def test_apply_date_format_error(mock_exit, mock_error): - input_date = 'xxxxxxxxxxxx' - date_format = '%Y-%m-%d' - DataFrameUtils.apply_date_format(input_date, date_format) - mock_exit.assert_called_once() - -def test_apply_date_format_value_unconverted(): - input_date = '2023-10-25 00:00:00' - date_format = '%Y-%m-%d' - result = DataFrameUtils.apply_date_format(input_date, date_format) - expected = datetime.date(2023, 10, 25) - assert result == expected - - -@patch('pandas.read_sql') -def test_get_db_table_col_list(mock_sql): - mock_engine = AlchemyMagicMock(spec=Engine) - DataFrameUtils(mock_engine, 'Test').get_db_table_col_list() - mock_sql.assert_called_once_with('SHOW COLUMNS FROM Test;', mock_engine) +def test_get_db_table_col_list(engine): + field_list = ['field_a', 'field_b', 'field_c'] + df = pd.DataFrame(data={ + 'Field': field_list, + 'Type': ['varchar(45)', 'varchar(45)', 'int'], + }, columns=['Field', 'Type']) + with patch('pandas.read_sql', return_value=df): + result = DataFrameUtils(engine, 'Test').get_db_table_col_list() + assert result == field_list -@patch('seq_dbutils.DataFrameUtils.get_db_table_col_list', return_value=['col1', 'col3']) -def test_create_db_table_dataframe(mock_get): - mock_engine = AlchemyMagicMock(spec=Engine) +def test_create_db_table_dataframe(engine): df = pd.DataFrame(data={ 'col1': ['a', 'b', None], 'col2': ['some data', 'some more data', None], 'col3': [None, None, None], }, columns=['col1', 'col2', 'col3']) - df_result = DataFrameUtils(mock_engine, 'Test').create_db_table_dataframe(df) df_expected = pd.DataFrame(data={ 'col1': ['a', 'b'], 'col3': [None, None], }, columns=['col1', 'col3']) - mock_get.assert_called_once() - pd.testing.assert_frame_equal(df_result, df_expected) + with patch('seq_dbutils.DataFrameUtils.get_db_table_col_list', return_value=['col1', 'col3']): + df_result = DataFrameUtils(engine, 'Test').create_db_table_dataframe(df) + pd.testing.assert_frame_equal(df_result, df_expected) diff --git a/tests/test_load.py b/tests/test_load.py index 44ecbd7..1308c0f 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -24,7 +24,6 @@ def instance(): return mock_session() - @pytest.fixture(scope='session') def dataframe(): df_data = pd.DataFrame(data={'id1': ['a', 'b', 'c'], @@ -47,20 +46,18 @@ def test_bulk_insert_df_table_ok(instance, dataframe): instance.bulk_insert_mappings.assert_called_once() -def test_bulk_insert_df_table_fail(instance, dataframe): +def test_bulk_insert_df_table_exception(instance, dataframe): + instance.bulk_insert_mappings = Mock(side_effect=Exception()) + instance.rollback = Mock() with patch('logging.info'): - with patch('logging.error'): - with patch('sys.exit') as mock_exit: - instance.bulk_insert_mappings = Mock(side_effect=Exception()) - instance.rollback = Mock() - Load(dataframe, instance, MockTable).bulk_insert_df_table() - instance.rollback.assert_called_once() - mock_exit.assert_called_once() + with pytest.raises(Exception): + Load(dataframe, instance, MockTable).bulk_insert_df_table() + instance.rollback.assert_called_once() def test_bulk_update_df_table_empty(instance): + df = pd.DataFrame() with patch('logging.info') as mock_info: - df = pd.DataFrame() Load(df, instance, MockTable).bulk_update_df_table() mock_info.assert_called_with('Skipping bulk update for table \'Mock\' and empty dataframe') @@ -71,12 +68,10 @@ def test_bulk_update_df_table_ok(instance, dataframe): instance.bulk_update_mappings.assert_called_once() -def test_bulk_update_df_table_fail(instance, dataframe): +def test_bulk_update_df_table_exception(instance, dataframe): + instance.bulk_update_mappings = Mock(side_effect=Exception()) + instance.rollback = Mock() with patch('logging.info'): - with patch('logging.error'): - with patch('sys.exit') as mock_exit: - instance.bulk_update_mappings = Mock(side_effect=Exception()) - instance.rollback = Mock() - Load(dataframe, instance, MockTable).bulk_update_df_table() - instance.rollback.assert_called_once() - mock_exit.assert_called_once() + with pytest.raises(Exception): + Load(dataframe, instance, MockTable).bulk_update_df_table() + instance.rollback.assert_called_once() diff --git a/tests/test_session.py b/tests/test_session.py index 713d983..ea11c03 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -11,8 +11,8 @@ def session(): def test_log_and_execute_sql(session): + sql = 'SELECT * FROM test;' with patch('logging.info'): - sql = 'SELECT * FROM test;' Session(session).log_and_execute_sql(sql) session.execute.assert_called_once() From 92b5e14c2b3b058fc97c6b335cc352cc828553ab Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Wed, 6 Nov 2024 13:30:10 +0000 Subject: [PATCH 04/13] Remove f-strings from execute statements --- seq_dbutils/trigger.py | 5 ++--- seq_dbutils/view.py | 5 ++--- tests/test_trigger.py | 2 +- tests/test_view.py | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/seq_dbutils/trigger.py b/seq_dbutils/trigger.py index b18d72c..abc3eae 100644 --- a/seq_dbutils/trigger.py +++ b/seq_dbutils/trigger.py @@ -20,9 +20,8 @@ def drop_and_create_trigger(self): self.create_trigger() def drop_trigger_if_exists(self): - drop_sql = f'DROP TRIGGER IF EXISTS {self.trigger_name};' - logging.info(drop_sql) - self.session_instance.execute(text(drop_sql)) + logging.info(f'DROP TRIGGER IF EXISTS {self.trigger_name}') + self.session_instance.execute('DROP TRIGGER IF EXISTS %s;', (self.trigger_name,)) def create_trigger(self): with open(self.trigger_filepath, 'r') as reader: diff --git a/seq_dbutils/view.py b/seq_dbutils/view.py index 8ef4085..ac3539f 100644 --- a/seq_dbutils/view.py +++ b/seq_dbutils/view.py @@ -21,9 +21,8 @@ def drop_and_create_view(self): @staticmethod def drop_view_if_exists(session_instance, view_name): - drop_sql = f'DROP VIEW IF EXISTS {view_name};' - logging.info(drop_sql) - session_instance.execute(text(drop_sql)) + logging.info(f'DROP VIEW IF EXISTS {view_name}') + session_instance.execute('DROP VIEW IF EXISTS %s;', (view_name,)) def create_view(self): with open(self.view_filepath, 'r') as reader: diff --git a/tests/test_trigger.py b/tests/test_trigger.py index d2f0351..c87296c 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -23,7 +23,7 @@ def trigger(instance): def test_drop_trigger_if_exists(instance, trigger): with patch('logging.info'): trigger.drop_trigger_if_exists() - instance.execute.assert_called_once() + instance.execute.assert_called_with('DROP TRIGGER IF EXISTS %s;', ('test_trigger',)) def test_create_trigger(instance, trigger): diff --git a/tests/test_view.py b/tests/test_view.py index d660867..2180230 100644 --- a/tests/test_view.py +++ b/tests/test_view.py @@ -23,7 +23,7 @@ def view(instance): def test_drop_view_if_exists(instance, view): with patch('logging.info'): view.drop_view_if_exists(instance, 'test_view') - instance.execute.assert_called_once() + instance.execute.assert_called_with('DROP VIEW IF EXISTS %s;', ('test_view',)) def test_create_view(instance, view): From 912802f518e1662ea78365da1090f8a8eab37839 Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Wed, 6 Nov 2024 13:35:02 +0000 Subject: [PATCH 05/13] Obsolete file --- tests/data/test_initialize_ok.ini | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tests/data/test_initialize_ok.ini diff --git a/tests/data/test_initialize_ok.ini b/tests/data/test_initialize_ok.ini deleted file mode 100644 index b29f229..0000000 --- a/tests/data/test_initialize_ok.ini +++ /dev/null @@ -1 +0,0 @@ -[MOCK] From a0b538ac9b51a0483076d5239df46656ab628e3d Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Wed, 6 Nov 2024 14:09:18 +0000 Subject: [PATCH 06/13] Use pytest --- .github/workflows/coverage.yml | 4 ++-- .github/workflows/python-versions.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 9099f76..56a8674 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -21,12 +21,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install coverage codecov mock mock_alchemy + pip install coverage codecov mock mock_alchemy pytest pip install . - name: Run coverage run: | - coverage run --source seq_dbutils -m unittest discover + coverage run --source seq_dbutils -m pytest coverage report -m coverage xml diff --git a/.github/workflows/python-versions.yml b/.github/workflows/python-versions.yml index ef2aa0a..d674f9f 100644 --- a/.github/workflows/python-versions.yml +++ b/.github/workflows/python-versions.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 mock mock_alchemy + pip install flake8 mock mock_alchemy pytest pip install . - name: Lint with flake8 @@ -34,6 +34,6 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --max-complexity=10 --statistics - - name: Test with unittest + - name: Test with pytest run: | - python -m unittest + pytest From 2c7275878272b7904bdb363887a348b529d41e7a Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Wed, 6 Nov 2024 14:11:18 +0000 Subject: [PATCH 07/13] Obsolete import --- seq_dbutils/dataframe.py | 1 - 1 file changed, 1 deletion(-) diff --git a/seq_dbutils/dataframe.py b/seq_dbutils/dataframe.py index 1872bde..9bc901b 100644 --- a/seq_dbutils/dataframe.py +++ b/seq_dbutils/dataframe.py @@ -1,5 +1,4 @@ import logging -from datetime import datetime import pandas as pd from sqlalchemy.engine import Engine From 4b1854eccd449c96bc572cec4eae13b06f436c50 Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Wed, 6 Nov 2024 14:24:31 +0000 Subject: [PATCH 08/13] Upgrade package --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index f80d2de..db4c09a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ test_suite = tests setup_requires = setuptools install_requires = - cryptography ~=41.0.7 + cryptography ~=43.0.1 mysql-connector-python ~=8.1.0 pandas ~=2.1.1 SQLAlchemy ~=2.0.22 From aff308674009e42d573bacee3cd3578267387858 Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Wed, 6 Nov 2024 14:27:37 +0000 Subject: [PATCH 09/13] Update actions versions --- .github/workflows/coverage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 56a8674..6f82670 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.12 - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: "3.12" From 21bcc31eccca083bd2b441805c68ee57d3ec9cf0 Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Wed, 6 Nov 2024 14:29:05 +0000 Subject: [PATCH 10/13] Update actions versions --- .github/workflows/publish-to-test-pypi.yml | 4 ++-- .github/workflows/python-versions.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index 1c79e0b..6ca3b2a 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -13,10 +13,10 @@ jobs: id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.12 - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: "3.12" diff --git a/.github/workflows/python-versions.yml b/.github/workflows/python-versions.yml index d674f9f..7c77e43 100644 --- a/.github/workflows/python-versions.yml +++ b/.github/workflows/python-versions.yml @@ -15,9 +15,9 @@ jobs: python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From 517d41db46eda4e5edf65bc7b5891d96dee1df18 Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Wed, 6 Nov 2024 14:34:18 +0000 Subject: [PATCH 11/13] Update logging --- seq_dbutils/trigger.py | 2 +- seq_dbutils/view.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/seq_dbutils/trigger.py b/seq_dbutils/trigger.py index abc3eae..2ee5b64 100644 --- a/seq_dbutils/trigger.py +++ b/seq_dbutils/trigger.py @@ -20,7 +20,7 @@ def drop_and_create_trigger(self): self.create_trigger() def drop_trigger_if_exists(self): - logging.info(f'DROP TRIGGER IF EXISTS {self.trigger_name}') + logging.info(f'DROP TRIGGER IF EXISTS {self.trigger_name};') self.session_instance.execute('DROP TRIGGER IF EXISTS %s;', (self.trigger_name,)) def create_trigger(self): diff --git a/seq_dbutils/view.py b/seq_dbutils/view.py index ac3539f..8578fba 100644 --- a/seq_dbutils/view.py +++ b/seq_dbutils/view.py @@ -21,7 +21,7 @@ def drop_and_create_view(self): @staticmethod def drop_view_if_exists(session_instance, view_name): - logging.info(f'DROP VIEW IF EXISTS {view_name}') + logging.info(f'DROP VIEW IF EXISTS {view_name};') session_instance.execute('DROP VIEW IF EXISTS %s;', (view_name,)) def create_view(self): From 65f0fd7566f104c9c4a5e814f46b4365f823caf2 Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Wed, 6 Nov 2024 15:17:33 +0000 Subject: [PATCH 12/13] Revert f-string change --- seq_dbutils/trigger.py | 5 +++-- seq_dbutils/view.py | 5 +++-- tests/test_trigger.py | 2 +- tests/test_view.py | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/seq_dbutils/trigger.py b/seq_dbutils/trigger.py index 2ee5b64..b18d72c 100644 --- a/seq_dbutils/trigger.py +++ b/seq_dbutils/trigger.py @@ -20,8 +20,9 @@ def drop_and_create_trigger(self): self.create_trigger() def drop_trigger_if_exists(self): - logging.info(f'DROP TRIGGER IF EXISTS {self.trigger_name};') - self.session_instance.execute('DROP TRIGGER IF EXISTS %s;', (self.trigger_name,)) + drop_sql = f'DROP TRIGGER IF EXISTS {self.trigger_name};' + logging.info(drop_sql) + self.session_instance.execute(text(drop_sql)) def create_trigger(self): with open(self.trigger_filepath, 'r') as reader: diff --git a/seq_dbutils/view.py b/seq_dbutils/view.py index 8578fba..8ef4085 100644 --- a/seq_dbutils/view.py +++ b/seq_dbutils/view.py @@ -21,8 +21,9 @@ def drop_and_create_view(self): @staticmethod def drop_view_if_exists(session_instance, view_name): - logging.info(f'DROP VIEW IF EXISTS {view_name};') - session_instance.execute('DROP VIEW IF EXISTS %s;', (view_name,)) + drop_sql = f'DROP VIEW IF EXISTS {view_name};' + logging.info(drop_sql) + session_instance.execute(text(drop_sql)) def create_view(self): with open(self.view_filepath, 'r') as reader: diff --git a/tests/test_trigger.py b/tests/test_trigger.py index c87296c..d2f0351 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -23,7 +23,7 @@ def trigger(instance): def test_drop_trigger_if_exists(instance, trigger): with patch('logging.info'): trigger.drop_trigger_if_exists() - instance.execute.assert_called_with('DROP TRIGGER IF EXISTS %s;', ('test_trigger',)) + instance.execute.assert_called_once() def test_create_trigger(instance, trigger): diff --git a/tests/test_view.py b/tests/test_view.py index 2180230..d660867 100644 --- a/tests/test_view.py +++ b/tests/test_view.py @@ -23,7 +23,7 @@ def view(instance): def test_drop_view_if_exists(instance, view): with patch('logging.info'): view.drop_view_if_exists(instance, 'test_view') - instance.execute.assert_called_with('DROP VIEW IF EXISTS %s;', ('test_view',)) + instance.execute.assert_called_once() def test_create_view(instance, view): From 05b32a917ecd3c697a281b796bcaf26606089491 Mon Sep 17 00:00:00 2001 From: Laura Thomson Date: Wed, 6 Nov 2024 15:21:46 +0000 Subject: [PATCH 13/13] Update version --- seq_dbutils/_version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seq_dbutils/_version.txt b/seq_dbutils/_version.txt index ece61c6..359a5b9 100644 --- a/seq_dbutils/_version.txt +++ b/seq_dbutils/_version.txt @@ -1 +1 @@ -1.0.6 \ No newline at end of file +2.0.0 \ No newline at end of file