From 8e6c435e5a92bf0b35d0e9c45edab1efc4477813 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Mon, 18 Jul 2022 14:21:15 +0100 Subject: [PATCH 01/17] Added daily scheduling modified: README.md modified: src/deploy.py modified: src/model.py modified: src/sql/agent/create_job_schedule.sql modified: src/sql/db.py modified: test/conftest.py modified: test/test_model.py --- .DS_Store | Bin 0 -> 6148 bytes README.md | 4 ++-- src/deploy.py | 6 +++--- src/model.py | 17 ++++++++++++----- src/sql/agent/create_job_schedule.sql | 4 ++-- src/sql/db.py | 14 ++++++++------ test/conftest.py | 4 ++-- test/test_model.py | 2 +- 8 files changed, 30 insertions(+), 21 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..6abaf3dcc7d4db273cde062d21b946bf2fe3f5d1 GIT binary patch literal 6148 zcmeHK%Wl&^6upzU#7&Eo1xRIqWQlDW(i9;oHfbJ$N<5?*!2(c-YZojWPZ~Q!D2kLd z`~$zhmLGwCVFhO%L3WZZ*dP?mjqaTB%suls8P9l#h_%N40a1mBY`8$HjAV_8df_Wp z&?Q%aOt#^lMEuN)xVI8(3XB3qf&Whd@$PPsM`H>pqU`%CwuEDN?Uyt;^j&JxLzF(D z2h^oU^b}a5!3yu|63-$Z?I)CG_6M}jZeAAWO_HJBN4|N(C`{6N{Rb-)t*t91yHwgK zy>{R6iJSJ*ankCKUW%t@oJZaw-}jz}!K7EY*5+~A5944Y385b#{F(O$cN9lFev2{@Uxs!k(6S39U8HL?WP+d7-lie`p+2Q_PMk(GSkm(6QhXWl z9}yi0-O)oxF@Fg(8F3Wv!KI Date: Mon, 18 Jul 2022 15:53:20 +0100 Subject: [PATCH 02/17] Changed tests to match new config --- src/deploy.py | 1 + test/test_model.py | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/deploy.py b/src/deploy.py index a30b16d..fd65d6f 100644 --- a/src/deploy.py +++ b/src/deploy.py @@ -71,6 +71,7 @@ def deploy_ssis( db.agent_create_job_schedule_occurs_every_n_units( job_name, job_schedule.name, + job_schedule.unit, job_schedule.every_n_unit, job_schedule.schedule_time, ) diff --git a/test/test_model.py b/test/test_model.py index c0e702b..2d738f3 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -51,6 +51,40 @@ def test_FrequencyType_MONTHLY_equals_16(self): assert FrequencyType.MONTHLY.value == expected +class TestUnitTypeFrequencyType: + def test_UnitTypeFrequencyType_MINUTE_equals_4(self): + """ + Test that UnitTypeFrequencyType.MINUTE = 4 + """ + expected = 4 + + assert UnitTypeFrequencyType.MINUTE.value == expected + + def test_UnitTypeFrequencyType_DAILY_equals_4(self): + """ + Test that UnitTypeFrequencyType.DAILY = 4 + """ + expected = 4 + + assert UnitTypeFrequencyType.DAILY.value == expected + +class TestUnitTypeFrequencyInterval: + def test_UnitTypeFrequencyInterval_MINUTE_equals_1(self): + """ + Test that UnitTypeFrequencyInterval.MINUTE = 1 + """ + expected = 1 + + assert UnitTypeFrequencyInterval.MINUTE.value == expected + + def test_UnitTypeFrequencyInterval_DAY_equals_4(self): + """ + Test that UnitTypeFrequencyType.DAILY = 4 + """ + expected = 4 + + assert UnitTypeFrequencyInterval.DAY.value == expected + class TestFrequencyInterval: def test_FrequencyInterval_ONCE_equals_1(self): @@ -201,8 +235,8 @@ def test_SsisDeployment_Job_Schedules_are_set_correctly(self): Test job schedules are set correctly. """ expected = [ - Schedule("Winter Moon", 30, datetime.time(0, 0)), - Schedule("Autumn Mountain", 1440, datetime.time(0, 0)), + Schedule("Winter Moon", "DAY", 30, 200000), + Schedule("Autumn Mountain", "MINUTE", 1440, 000000), ] actual = load_configuration(toml.dumps(TEST_CONFIG)) assert actual.job.schedules == expected From 4ca0b87b0ee3464d93464ca1fa42da7f5bab8d89 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Wed, 20 Jul 2022 10:36:24 +0100 Subject: [PATCH 03/17] Added in week and month parameters into schedule --- src/model.py | 21 +++++++++-- test/conftest.py | 2 ++ test/run_tests.sh | 0 test/test_model.py | 87 ++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 93 insertions(+), 17 deletions(-) mode change 100644 => 100755 test/run_tests.sh diff --git a/src/model.py b/src/model.py index 50fdfc3..5e46d6b 100644 --- a/src/model.py +++ b/src/model.py @@ -31,10 +31,18 @@ class NotifyLevelEmail(Enum): class UnitTypeFrequencyType(Enum): MINUTE = 4 DAY = 4 + WEEK = 8 + MONTH = 16 -class UnitTypeFrequencyInterval(Enum): - MINUTE = 1 - DAY = 4 + +class DayOfWeekFrequencyInterval(Enum): + SUNDAY = 1 + MONDAY = 2 + TUESDAY = 4 + WEDNESDAY = 8 + THURSDAY = 16 + FRIDAY = 32 + SATURDAY = 64 @dataclass @@ -51,7 +59,14 @@ class Schedule: unit: str every_n_units: int schedule_time: str = "000000" + run_days: typing.Optional[typing.List[str]] = None + day_of_month: typing.Optional[int] = None + def __post_init__(self): + if self.unit == "WEEK" and not self.run_days: + raise ConfigurationError("'run_days must be provided.'") + elif self.unit == "MONTH" and not self.day_of_month: + raise ConfigurationError("'day_of_month must be provided.'") @dataclass class Step: diff --git a/test/conftest.py b/test/conftest.py index ac2ba65..86e120e 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -30,6 +30,8 @@ "schedules": [ {"name": "Winter Moon", "unit": "DAY", "every_n_units": 30, "schedule_time": "200000"}, {"name": "Autumn Mountain", "unit": "MINUTE", "every_n_units": 1440}, + {"name": "Peaceful Valley", "unit": "WEEK", "every_n_units": 1, "run_days": ["MONDAY","FRIDAY"]}, + {"name": "Snowfall", "unit": "MONTH", "every_n_units": 1, "day_of_month": 15}, ], }, } diff --git a/test/run_tests.sh b/test/run_tests.sh old mode 100644 new mode 100755 diff --git a/test/test_model.py b/test/test_model.py index 2d738f3..70f89d3 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -11,6 +11,8 @@ from src.model import ( FrequencyInterval, FrequencyType, + DayOfWeekFrequencyInterval, + UnitTypeFrequencyType, NotifyLevelEmail, Schedule, SsisDeployment, @@ -60,31 +62,86 @@ def test_UnitTypeFrequencyType_MINUTE_equals_4(self): assert UnitTypeFrequencyType.MINUTE.value == expected - def test_UnitTypeFrequencyType_DAILY_equals_4(self): + def test_UnitTypeFrequencyType_DAY_equals_4(self): """ - Test that UnitTypeFrequencyType.DAILY = 4 + Test that UnitTypeFrequencyType.DAY = 4 """ expected = 4 - assert UnitTypeFrequencyType.DAILY.value == expected + assert UnitTypeFrequencyType.DAY.value == expected -class TestUnitTypeFrequencyInterval: - def test_UnitTypeFrequencyInterval_MINUTE_equals_1(self): + def test_UnitTypeFrequencyType_WEEK_equals_8(self): """ - Test that UnitTypeFrequencyInterval.MINUTE = 1 + Test that UnitTypeFrequencyType.WEEK = 8 + """ + expected = 8 + + assert UnitTypeFrequencyType.WEEK.value == expected + + def test_UnitTypeFrequencyType_MONTH_equals_16(self): + """ + Test that UnitTypeFrequencyType.MONTH = 16 + """ + expected = 16 + + assert UnitTypeFrequencyType.MONTH.value == expected + +class TestDayOfWeekFrequencyInterval: + def test_DayOfWeekFrequencyInterval_SUNDAY_equals_1(self): + """ + Test that DayOfWeekFrequencyInterval.SUNDAY = 1 """ expected = 1 - assert UnitTypeFrequencyInterval.MINUTE.value == expected + assert DayOfWeekFrequencyInterval.SUNDAY.value == expected - def test_UnitTypeFrequencyInterval_DAY_equals_4(self): + def test_DayOfWeekFrequencyInterval_MONDAY_equals_2(self): + """ + Test that DayOfWeekFrequencyInterval.MONDAY = 2 """ - Test that UnitTypeFrequencyType.DAILY = 4 + expected = 2 + + assert DayOfWeekFrequencyInterval.MONDAY.value == expected + + def test_UnitTypeFrequencyInterval_TUESDAY_equals_4(self): + """ + Test that DayOfWeekFrequencyInterval.TUESDAY = 4 """ expected = 4 - assert UnitTypeFrequencyInterval.DAY.value == expected + assert DayOfWeekFrequencyInterval.TUESDAY.value == expected + + def test_DayOfWeekFrequencyInterval_WEDNESDAY_equals_8(self): + """ + Test that DayOfWeekFrequencyInterval.WEDNESDAY = 8 + """ + expected = 8 + + assert DayOfWeekFrequencyInterval.WEDNESDAY.value == expected + + def test_DayOfWeekFrequencyInterval_THURSDAY_equals_16(self): + """ + Test that DayOfWeekFrequencyInterval.THURSDAY = 16 + """ + expected = 16 + + assert DayOfWeekFrequencyInterval.THURSDAY.value == expected + + def test_DayOfWeekFrequencyInterval_FRIDAY_equals_32(self): + """ + Test that DayOfWeekFrequencyInterval.FRIDAY = 32 + """ + expected = 32 + + assert DayOfWeekFrequencyInterval.FRIDAY.value == expected + + def test_DayOfWeekFrequencyInterval_SATURDAY_equals_64(self): + """ + Test that DayOfWeekFrequencyInterval.SATURDAY = 64 + """ + expected = 64 + assert DayOfWeekFrequencyInterval.SATURDAY.value == expected class TestFrequencyInterval: def test_FrequencyInterval_ONCE_equals_1(self): @@ -226,7 +283,7 @@ def test_SsisDeployment_Number_Job_Schedules_is_set_correctly(self): """ Test number of job schedules is correct. """ - expected = 2 + expected = 4 actual = load_configuration(toml.dumps(TEST_CONFIG)) assert len(actual.job.schedules) == expected @@ -235,8 +292,10 @@ def test_SsisDeployment_Job_Schedules_are_set_correctly(self): Test job schedules are set correctly. """ expected = [ - Schedule("Winter Moon", "DAY", 30, 200000), - Schedule("Autumn Mountain", "MINUTE", 1440, 000000), + Schedule("Winter Moon", "DAY", 30, "200000"), + Schedule("Autumn Mountain", "MINUTE", 1440, "000000"), + Schedule("Peaceful Valley", "WEEK", 1, "000000", ["MONDAY", "FRIDAY"]), + Schedule("Snowfall", "MONTH", 1, "000000", None, 15) ] actual = load_configuration(toml.dumps(TEST_CONFIG)) assert actual.job.schedules == expected @@ -253,7 +312,7 @@ def test_SsisDeployment_job_schedule_count_is_set_correctly(self): """ Test the correct number of job schedules are set. """ - expected = 2 + expected = 4 actual = len(load_configuration(toml.dumps(TEST_CONFIG)).job.schedules) assert actual == expected From df94d29bdb9f809cb81dce09d8cf6ec760d2b83b Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 21 Jul 2022 09:38:42 +0100 Subject: [PATCH 04/17] added test coverage --- README.md | 20 +++++++++- src/config.py | 4 +- src/deploy.py | 21 ++++++----- src/model.py | 53 ++++++++++++++++++++++++++- src/sql-deployment-tools.py | 4 +- src/sql/agent/create_job_schedule.sql | 12 +++--- src/sql/db.py | 24 +++++++----- test/conftest.py | 2 +- test/test_model.py | 22 +++++++++-- 9 files changed, 126 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 7a200e4..86095af 100644 --- a/README.md +++ b/README.md @@ -114,11 +114,29 @@ tsql_command = "SELECT TOP 10 * FROM sys.objects" [[job.schedules]] name = "name1" -every_n_units = 12 +unit = "DAY" +every_n_units = 2 +schedule_time = 200000 [[job.schedules]] name = "name2" +unit = "MINUTE" every_n_units = 111 + +[[job.schedules]] +name = "name3" +unit = "WEEK" +every_n_units = 1 +run_days = [ + "MONDAY", + "TUESDAY" + ] + +[[job.schedules]] +name = "name4" +unit = "MONTH" +every_n_units = 3 +day_of_month = 12 ``` Note, when configuring a T-SQL step in an agent job, the default database diff --git a/src/config.py b/src/config.py index 9eb7c45..7a09a4b 100644 --- a/src/config.py +++ b/src/config.py @@ -1,7 +1,7 @@ import toml -from src.exceptions import ConfigurationError -from src.model import SsisDeployment +from exceptions import ConfigurationError +from model import SsisDeployment def load_configuration(configuration: str) -> SsisDeployment: diff --git a/src/deploy.py b/src/deploy.py index fd65d6f..540c36a 100644 --- a/src/deploy.py +++ b/src/deploy.py @@ -1,13 +1,12 @@ -from src.exceptions import SqlAgentOperatorException -from src.model import SsisDeployment -from src.sql.db import Database +from exceptions import SqlAgentOperatorException +from model import SsisDeployment +from sql.db import Database def deploy_ssis( connection_string: str, ispac_path: str, ssis_deployment: SsisDeployment -): +): db = Database(connection_string) - project_name = ssis_deployment.project folder_name = ssis_deployment.folder environment_name = ssis_deployment.environment @@ -68,12 +67,16 @@ def deploy_ssis( ) for job_schedule in ssis_deployment.job.schedules: - db.agent_create_job_schedule_occurs_every_n_units( + parameters = job_schedule.transform_for_query() + db.agent_create_job_schedule( job_name, job_schedule.name, - job_schedule.unit, - job_schedule.every_n_unit, - job_schedule.schedule_time, + parameters.freq_type, + parameters.freq_interval, + parameters.freq_subday_type, + parameters.freq_subday_interval, + parameters.freq_recurrence_factor, + parameters.active_start_time ) if ssis_deployment.job.notification_email_address: diff --git a/src/model.py b/src/model.py index 5e46d6b..fd662d4 100644 --- a/src/model.py +++ b/src/model.py @@ -5,7 +5,7 @@ from dataclasses_json import config, dataclass_json -from src.exceptions import ConfigurationError +from exceptions import ConfigurationError class FrequencyType(Enum): @@ -52,22 +52,71 @@ class Parameter: sensitive: bool = False +@dataclass +class ScheduleQueryParameters: + freq_type: int + freq_interval: int + freq_subday_type: int + freq_subday_interval: int + freq_recurrence_factor: int + active_start_time: int + + @dataclass_json @dataclass class Schedule: name: str unit: str every_n_units: int - schedule_time: str = "000000" + schedule_time: int = 0 run_days: typing.Optional[typing.List[str]] = None day_of_month: typing.Optional[int] = None + def transform_for_query(self) -> ScheduleQueryParameters: + if self.unit == "MINUTE": + return ScheduleQueryParameters( + UnitTypeFrequencyType.MINUTE.value, + 1, + 4, + self.every_n_units, + 0, + self.schedule_time + ) + if self.unit == "DAY": + return ScheduleQueryParameters( + UnitTypeFrequencyType.DAY.value, + self.every_n_units, + 1, + 0, + 0, + self.schedule_time + ) + if self.unit == "WEEK": + return ScheduleQueryParameters( + UnitTypeFrequencyType.WEEK.value, + sum([DayOfWeekFrequencyInterval[x].value for x in self.run_days]), + 1, + 0, + self.every_n_units, + self.schedule_time + ) + if self.unit == "MONTH": + return ScheduleQueryParameters( + UnitTypeFrequencyType.MONTH.value, + self.day_of_month, + 1, + 0, + self.every_n_units, + self.schedule_time + ) + def __post_init__(self): if self.unit == "WEEK" and not self.run_days: raise ConfigurationError("'run_days must be provided.'") elif self.unit == "MONTH" and not self.day_of_month: raise ConfigurationError("'day_of_month must be provided.'") + @dataclass class Step: name: str diff --git a/src/sql-deployment-tools.py b/src/sql-deployment-tools.py index 6800b12..e7d4290 100644 --- a/src/sql-deployment-tools.py +++ b/src/sql-deployment-tools.py @@ -3,8 +3,8 @@ import os import sys -from src.config import ConfigurationError, load_configuration -from src.deploy import deploy_ssis +from config import ConfigurationError, load_configuration +from deploy import deploy_ssis if __name__ == "__main__": parent_parser = argparse.ArgumentParser(description="SSIS Deployment Helper") diff --git a/src/sql/agent/create_job_schedule.sql b/src/sql/agent/create_job_schedule.sql index dfa4242..37dfaf8 100644 --- a/src/sql/agent/create_job_schedule.sql +++ b/src/sql/agent/create_job_schedule.sql @@ -4,13 +4,13 @@ EXEC msdb.dbo.sp_add_jobschedule @job_name = $job_name , @name = $schedule_name , @enabled = 1 -- Enabled - , @freq_type = 4 -- Daily - , @freq_interval = 1 -- Once - , @freq_subday_type = $frequency_subday_type - , @freq_subday_interval = $every_n_units + , @freq_type = $freq_type + , @freq_interval = $freq_interval + , @freq_subday_type = $freq_subday_type + , @freq_subday_interval = $freq_subday_interval , @freq_relative_interval = 0 - , @freq_recurrence_factor = 1 - , @active_start_time = $hh_mm_ss + , @freq_recurrence_factor = $freq_recurrence_factor + , @active_start_time = $active_start_time , @schedule_id = @schedule_id OUTPUT ; diff --git a/src/sql/db.py b/src/sql/db.py index 998f0e2..f59a257 100644 --- a/src/sql/db.py +++ b/src/sql/db.py @@ -5,8 +5,8 @@ import sqlparams from sql import query -from src.exceptions import SqlAgentOperatorException -from src.model import NotifyLevelEmail +from exceptions import SqlAgentOperatorException +from model import NotifyLevelEmail class Database: @@ -235,22 +235,28 @@ def agent_create_job_step_tsql( self._agent_reset_job_step_flow(job_name) - def agent_create_job_schedule_occurs_every_n_units( + def agent_create_job_schedule( self, job_name: str, schedule_name: str, - unit: str, - every_n_units: int, - schedule_time: str, + freq_type: int, + freq_interval: int, + freq_subday_type: int, + freq_subday_interval: int, + freq_recurrence_factor: int, + active_start_time: int ): self._execute_sql( query.agent_create_job_schedule, { "job_name": job_name, "schedule_name": schedule_name, - "frequency_subday_type": UnitTypeFrequencyInterval[unit].value, - "every_n_units": every_n_units, - "hh_mm_ss": schedule_time + "freq_type": freq_type, + "freq_interval": freq_interval, + "freq_subday_type": freq_subday_type, + "freq_subday_interval": freq_subday_interval, + "freq_recurrence_factor": freq_recurrence_factor, + "active_start_time": active_start_time }, ) diff --git a/test/conftest.py b/test/conftest.py index 86e120e..3556a1a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -28,7 +28,7 @@ }, ], "schedules": [ - {"name": "Winter Moon", "unit": "DAY", "every_n_units": 30, "schedule_time": "200000"}, + {"name": "Winter Moon", "unit": "DAY", "every_n_units": 30, "schedule_time": 200000}, {"name": "Autumn Mountain", "unit": "MINUTE", "every_n_units": 1440}, {"name": "Peaceful Valley", "unit": "WEEK", "every_n_units": 1, "run_days": ["MONDAY","FRIDAY"]}, {"name": "Snowfall", "unit": "MONTH", "every_n_units": 1, "day_of_month": 15}, diff --git a/test/test_model.py b/test/test_model.py index 70f89d3..fc00375 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -12,6 +12,7 @@ FrequencyInterval, FrequencyType, DayOfWeekFrequencyInterval, + ScheduleQueryParameters, UnitTypeFrequencyType, NotifyLevelEmail, Schedule, @@ -292,13 +293,26 @@ def test_SsisDeployment_Job_Schedules_are_set_correctly(self): Test job schedules are set correctly. """ expected = [ - Schedule("Winter Moon", "DAY", 30, "200000"), - Schedule("Autumn Mountain", "MINUTE", 1440, "000000"), - Schedule("Peaceful Valley", "WEEK", 1, "000000", ["MONDAY", "FRIDAY"]), - Schedule("Snowfall", "MONTH", 1, "000000", None, 15) + Schedule("Winter Moon", "DAY", 30, 200000), + Schedule("Autumn Mountain", "MINUTE", 1440, 0), + Schedule("Peaceful Valley", "WEEK", 1, 0, ["MONDAY", "FRIDAY"]), + Schedule("Snowfall", "MONTH", 1, 0, None, 15) ] actual = load_configuration(toml.dumps(TEST_CONFIG)) assert actual.job.schedules == expected + + def test_SsisDeployment_Job_Schedule_Transforms_are_correct(self): + """ + Test schedules transform into sp_schedule_job variables correctly + """ + expected = [ + ScheduleQueryParameters(4,30,1,0,0,200000), + ScheduleQueryParameters(4,1,4,1440,0,0), + ScheduleQueryParameters(8,34,1,0,1,0), + ScheduleQueryParameters(16,15,1,0,1,0) + ] + actual = load_configuration(toml.dumps(TEST_CONFIG)) + assert [x.transform_for_query() for x in actual.job.schedules] == expected def test_SsisDeployment_job_step_count_is_set_correctly(self): """ From 602518a1472588e24dbb1d94811914cf9e619ea7 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 21 Jul 2022 10:11:14 +0100 Subject: [PATCH 05/17] pre-commit fixes --- src/deploy.py | 4 ++-- src/model.py | 11 +++++------ src/sql/db.py | 9 ++++----- test/conftest.py | 21 ++++++++++++++++++--- test/test_model.py | 22 ++++++++++++---------- 5 files changed, 41 insertions(+), 26 deletions(-) diff --git a/src/deploy.py b/src/deploy.py index 540c36a..f16e37b 100644 --- a/src/deploy.py +++ b/src/deploy.py @@ -5,7 +5,7 @@ def deploy_ssis( connection_string: str, ispac_path: str, ssis_deployment: SsisDeployment -): +): db = Database(connection_string) project_name = ssis_deployment.project folder_name = ssis_deployment.folder @@ -76,7 +76,7 @@ def deploy_ssis( parameters.freq_subday_type, parameters.freq_subday_interval, parameters.freq_recurrence_factor, - parameters.active_start_time + parameters.active_start_time, ) if ssis_deployment.job.notification_email_address: diff --git a/src/model.py b/src/model.py index fd662d4..9398918 100644 --- a/src/model.py +++ b/src/model.py @@ -1,4 +1,3 @@ -import datetime import typing from dataclasses import dataclass, field from enum import Enum @@ -80,7 +79,7 @@ def transform_for_query(self) -> ScheduleQueryParameters: 4, self.every_n_units, 0, - self.schedule_time + self.schedule_time, ) if self.unit == "DAY": return ScheduleQueryParameters( @@ -89,7 +88,7 @@ def transform_for_query(self) -> ScheduleQueryParameters: 1, 0, 0, - self.schedule_time + self.schedule_time, ) if self.unit == "WEEK": return ScheduleQueryParameters( @@ -98,7 +97,7 @@ def transform_for_query(self) -> ScheduleQueryParameters: 1, 0, self.every_n_units, - self.schedule_time + self.schedule_time, ) if self.unit == "MONTH": return ScheduleQueryParameters( @@ -107,9 +106,9 @@ def transform_for_query(self) -> ScheduleQueryParameters: 1, 0, self.every_n_units, - self.schedule_time + self.schedule_time, ) - + def __post_init__(self): if self.unit == "WEEK" and not self.run_days: raise ConfigurationError("'run_days must be provided.'") diff --git a/src/sql/db.py b/src/sql/db.py index f59a257..51b5a73 100644 --- a/src/sql/db.py +++ b/src/sql/db.py @@ -1,12 +1,11 @@ -import datetime import os import pyodbc import sqlparams -from sql import query from exceptions import SqlAgentOperatorException from model import NotifyLevelEmail +from sql import query class Database: @@ -244,7 +243,7 @@ def agent_create_job_schedule( freq_subday_type: int, freq_subday_interval: int, freq_recurrence_factor: int, - active_start_time: int + active_start_time: int, ): self._execute_sql( query.agent_create_job_schedule, @@ -256,9 +255,9 @@ def agent_create_job_schedule( "freq_subday_type": freq_subday_type, "freq_subday_interval": freq_subday_interval, "freq_recurrence_factor": freq_recurrence_factor, - "active_start_time": active_start_time + "active_start_time": active_start_time, }, - ) + ) def ssis_install_ispac( self, diff --git a/test/conftest.py b/test/conftest.py index 3556a1a..3bb65da 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -28,10 +28,25 @@ }, ], "schedules": [ - {"name": "Winter Moon", "unit": "DAY", "every_n_units": 30, "schedule_time": 200000}, + { + "name": "Winter Moon", + "unit": "DAY", + "every_n_units": 30, + "schedule_time": 200000, + }, {"name": "Autumn Mountain", "unit": "MINUTE", "every_n_units": 1440}, - {"name": "Peaceful Valley", "unit": "WEEK", "every_n_units": 1, "run_days": ["MONDAY","FRIDAY"]}, - {"name": "Snowfall", "unit": "MONTH", "every_n_units": 1, "day_of_month": 15}, + { + "name": "Peaceful Valley", + "unit": "WEEK", + "every_n_units": 1, + "run_days": ["MONDAY", "FRIDAY"], + }, + { + "name": "Snowfall", + "unit": "MONTH", + "every_n_units": 1, + "day_of_month": 15, + }, ], }, } diff --git a/test/test_model.py b/test/test_model.py index fc00375..24616f6 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -1,5 +1,4 @@ import copy -import datetime from test.conftest import TEST_CONFIG import pytest @@ -9,15 +8,15 @@ from src.config import load_configuration from src.exceptions import ConfigurationError from src.model import ( + DayOfWeekFrequencyInterval, FrequencyInterval, FrequencyType, - DayOfWeekFrequencyInterval, - ScheduleQueryParameters, - UnitTypeFrequencyType, NotifyLevelEmail, Schedule, + ScheduleQueryParameters, SsisDeployment, Step, + UnitTypeFrequencyType, ) @@ -54,6 +53,7 @@ def test_FrequencyType_MONTHLY_equals_16(self): assert FrequencyType.MONTHLY.value == expected + class TestUnitTypeFrequencyType: def test_UnitTypeFrequencyType_MINUTE_equals_4(self): """ @@ -87,6 +87,7 @@ def test_UnitTypeFrequencyType_MONTH_equals_16(self): assert UnitTypeFrequencyType.MONTH.value == expected + class TestDayOfWeekFrequencyInterval: def test_DayOfWeekFrequencyInterval_SUNDAY_equals_1(self): """ @@ -144,6 +145,7 @@ def test_DayOfWeekFrequencyInterval_SATURDAY_equals_64(self): assert DayOfWeekFrequencyInterval.SATURDAY.value == expected + class TestFrequencyInterval: def test_FrequencyInterval_ONCE_equals_1(self): """ @@ -296,20 +298,20 @@ def test_SsisDeployment_Job_Schedules_are_set_correctly(self): Schedule("Winter Moon", "DAY", 30, 200000), Schedule("Autumn Mountain", "MINUTE", 1440, 0), Schedule("Peaceful Valley", "WEEK", 1, 0, ["MONDAY", "FRIDAY"]), - Schedule("Snowfall", "MONTH", 1, 0, None, 15) + Schedule("Snowfall", "MONTH", 1, 0, None, 15), ] actual = load_configuration(toml.dumps(TEST_CONFIG)) assert actual.job.schedules == expected - + def test_SsisDeployment_Job_Schedule_Transforms_are_correct(self): """ Test schedules transform into sp_schedule_job variables correctly """ expected = [ - ScheduleQueryParameters(4,30,1,0,0,200000), - ScheduleQueryParameters(4,1,4,1440,0,0), - ScheduleQueryParameters(8,34,1,0,1,0), - ScheduleQueryParameters(16,15,1,0,1,0) + ScheduleQueryParameters(4, 30, 1, 0, 0, 200000), + ScheduleQueryParameters(4, 1, 4, 1440, 0, 0), + ScheduleQueryParameters(8, 34, 1, 0, 1, 0), + ScheduleQueryParameters(16, 15, 1, 0, 1, 0), ] actual = load_configuration(toml.dumps(TEST_CONFIG)) assert [x.transform_for_query() for x in actual.job.schedules] == expected From 45a12c7c5924c8bb8a3a28dbc62827679e1623a7 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 21 Jul 2022 11:55:30 +0100 Subject: [PATCH 06/17] added schedule keys to readme --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 86095af..a684a68 100644 --- a/README.md +++ b/README.md @@ -163,3 +163,14 @@ sql-deployment-tools deploy --replacement-tokens '{"SECRET_VALUE": "***"}' ```text Driver={SQL Server Native Client 11.0};Server=.;Database=SSISDB;Trusted_Connection=yes; ``` + +### Schedules TOML keys + +| key | type | required | allowable values | description | +|-----------------|-----------|---------------------|---------------------------------------------------------------------------------------|-------------------------------------------------------------------| +| name | str | yes | any string | schedule name | +| unit | str | yes | "MINUTE"
"DAY"
"WEEK"
"MONTH" | time unit | +| every\_n\_units | int | yes | any integer | x value in: repeats every _x unit_s | +| schedule\_time | int | no _default:0_ | integer time value HHMMSS in 24h clock
e.g 0 is midnight, 70000 is 7am, 190000 is 7pm | job scheduled start time in 24h clock | +| run\_days | List[str] | when unit = "WEEK" | str values:
"SUNDAY"
"MONDAY"
"TUESDAY"
"WEDNESDAY"
"THURSDAY"
"FRIDAY" "SATURDAY" | days of the week that job should run | +| day\_of\_month | int | when unit = "MONTH" | any integer | day of the month to run scheduled job. 1 being first day of month | \ No newline at end of file From 0e5251def07eea6c411ed57dc19c2e5f63bc0986 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 21 Jul 2022 12:10:14 +0100 Subject: [PATCH 07/17] fixed spacing --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a684a68..ce3a62a 100644 --- a/README.md +++ b/README.md @@ -172,5 +172,5 @@ Driver={SQL Server Native Client 11.0};Server=.;Database=SSISDB;Trusted_Connecti | unit | str | yes | "MINUTE"
"DAY"
"WEEK"
"MONTH" | time unit | | every\_n\_units | int | yes | any integer | x value in: repeats every _x unit_s | | schedule\_time | int | no _default:0_ | integer time value HHMMSS in 24h clock
e.g 0 is midnight, 70000 is 7am, 190000 is 7pm | job scheduled start time in 24h clock | -| run\_days | List[str] | when unit = "WEEK" | str values:
"SUNDAY"
"MONDAY"
"TUESDAY"
"WEDNESDAY"
"THURSDAY"
"FRIDAY" "SATURDAY" | days of the week that job should run | +| run\_days | List[str] | when unit = "WEEK" | str values:
"SUNDAY"
"MONDAY"
"TUESDAY"
"WEDNESDAY"
"THURSDAY"
"FRIDAY"
"SATURDAY" | days of the week that job should run | | day\_of\_month | int | when unit = "MONTH" | any integer | day of the month to run scheduled job. 1 being first day of month | \ No newline at end of file From 633f0b0ee3639e9888435267c6dfecd9883a28c6 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 21 Jul 2022 12:15:43 +0100 Subject: [PATCH 08/17] removed .DS_Store --- .DS_Store | Bin 6148 -> 0 bytes .gitignore | 3 +++ 2 files changed, 3 insertions(+) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 6abaf3dcc7d4db273cde062d21b946bf2fe3f5d1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%Wl&^6upzU#7&Eo1xRIqWQlDW(i9;oHfbJ$N<5?*!2(c-YZojWPZ~Q!D2kLd z`~$zhmLGwCVFhO%L3WZZ*dP?mjqaTB%suls8P9l#h_%N40a1mBY`8$HjAV_8df_Wp z&?Q%aOt#^lMEuN)xVI8(3XB3qf&Whd@$PPsM`H>pqU`%CwuEDN?Uyt;^j&JxLzF(D z2h^oU^b}a5!3yu|63-$Z?I)CG_6M}jZeAAWO_HJBN4|N(C`{6N{Rb-)t*t91yHwgK zy>{R6iJSJ*ankCKUW%t@oJZaw-}jz}!K7EY*5+~A5944Y385b#{F(O$cN9lFev2{@Uxs!k(6S39U8HL?WP+d7-lie`p+2Q_PMk(GSkm(6QhXWl z9}yi0-O)oxF@Fg(8F3Wv!KI Date: Thu, 21 Jul 2022 12:30:14 +0100 Subject: [PATCH 09/17] precommit fixes --- .gitignore | 2 +- README.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index a9dd5b1..2c30504 100644 --- a/.gitignore +++ b/.gitignore @@ -148,4 +148,4 @@ settings.json test-results/ # Mac Preferences File -.DS_Store \ No newline at end of file +.DS_Store diff --git a/README.md b/README.md index ce3a62a..8caee31 100644 --- a/README.md +++ b/README.md @@ -169,8 +169,8 @@ Driver={SQL Server Native Client 11.0};Server=.;Database=SSISDB;Trusted_Connecti | key | type | required | allowable values | description | |-----------------|-----------|---------------------|---------------------------------------------------------------------------------------|-------------------------------------------------------------------| | name | str | yes | any string | schedule name | -| unit | str | yes | "MINUTE"
"DAY"
"WEEK"
"MONTH" | time unit | +| unit | str | yes | "MINUTE" "DAY" "WEEK" "MONTH" | time unit | | every\_n\_units | int | yes | any integer | x value in: repeats every _x unit_s | -| schedule\_time | int | no _default:0_ | integer time value HHMMSS in 24h clock
e.g 0 is midnight, 70000 is 7am, 190000 is 7pm | job scheduled start time in 24h clock | -| run\_days | List[str] | when unit = "WEEK" | str values:
"SUNDAY"
"MONDAY"
"TUESDAY"
"WEDNESDAY"
"THURSDAY"
"FRIDAY"
"SATURDAY" | days of the week that job should run | -| day\_of\_month | int | when unit = "MONTH" | any integer | day of the month to run scheduled job. 1 being first day of month | \ No newline at end of file +| schedule\_time | int | no _default:0_ | integer time value HHMMSS in 24h clock e.g 0 is midnight, 70000 is 7am, 190000 is 7pm | job scheduled start time in 24h clock | +| run\_days | List[str] | when unit = "WEEK" | str values: "SUNDAY" "MONDAY" "TUESDAY" "WEDNESDAY" "THURSDAY" "FRIDAY" "SATURDAY" | days of the week that job should run | +| day\_of\_month | int | when unit = "MONTH" | any integer | day of the month to run scheduled job. 1 being first day of month | From e36a864f65183a18f88f8d389b826a8957503d8e Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 21 Jul 2022 12:48:29 +0100 Subject: [PATCH 10/17] got tests to run --- src/config.py | 4 ++-- src/deploy.py | 7 ++++--- src/model.py | 2 +- src/sql-deployment-tools.py | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/config.py b/src/config.py index 7a09a4b..9eb7c45 100644 --- a/src/config.py +++ b/src/config.py @@ -1,7 +1,7 @@ import toml -from exceptions import ConfigurationError -from model import SsisDeployment +from src.exceptions import ConfigurationError +from src.model import SsisDeployment def load_configuration(configuration: str) -> SsisDeployment: diff --git a/src/deploy.py b/src/deploy.py index f16e37b..2755255 100644 --- a/src/deploy.py +++ b/src/deploy.py @@ -1,12 +1,13 @@ -from exceptions import SqlAgentOperatorException -from model import SsisDeployment -from sql.db import Database +from src.exceptions import SqlAgentOperatorException +from src.model import SsisDeployment +from src.sql.db import Database def deploy_ssis( connection_string: str, ispac_path: str, ssis_deployment: SsisDeployment ): db = Database(connection_string) + project_name = ssis_deployment.project folder_name = ssis_deployment.folder environment_name = ssis_deployment.environment diff --git a/src/model.py b/src/model.py index 9398918..3d93b9f 100644 --- a/src/model.py +++ b/src/model.py @@ -4,7 +4,7 @@ from dataclasses_json import config, dataclass_json -from exceptions import ConfigurationError +from src.exceptions import ConfigurationError class FrequencyType(Enum): diff --git a/src/sql-deployment-tools.py b/src/sql-deployment-tools.py index e7d4290..6800b12 100644 --- a/src/sql-deployment-tools.py +++ b/src/sql-deployment-tools.py @@ -3,8 +3,8 @@ import os import sys -from config import ConfigurationError, load_configuration -from deploy import deploy_ssis +from src.config import ConfigurationError, load_configuration +from src.deploy import deploy_ssis if __name__ == "__main__": parent_parser = argparse.ArgumentParser(description="SSIS Deployment Helper") From bc657750eb8f58d6daa074468501f59b7bbeaf9a Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 21 Jul 2022 12:51:41 +0100 Subject: [PATCH 11/17] precommit fix --- src/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deploy.py b/src/deploy.py index 2755255..891829e 100644 --- a/src/deploy.py +++ b/src/deploy.py @@ -7,7 +7,7 @@ def deploy_ssis( connection_string: str, ispac_path: str, ssis_deployment: SsisDeployment ): db = Database(connection_string) - + project_name = ssis_deployment.project folder_name = ssis_deployment.folder environment_name = ssis_deployment.environment From 6454aefce6fbbbb4ddb83c2299db3af7a64c9e3d Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 21 Jul 2022 13:14:25 +0100 Subject: [PATCH 12/17] added to pytest.ini to fix package path issue --- pytest.ini | 1 + src/config.py | 4 ++-- src/deploy.py | 6 +++--- src/model.py | 2 +- src/sql-deployment-tools.py | 4 ++-- test/test_config.py | 6 +++--- test/test_model.py | 6 +++--- 7 files changed, 15 insertions(+), 14 deletions(-) diff --git a/pytest.ini b/pytest.ini index c2a30b0..490c27f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,3 @@ [pytest] addopts = --cov=./src/ --cov-config=./.coveragerc --junitxml=./test-results/unit-test-results.xml -v -rF -rP +pythonpath = src diff --git a/src/config.py b/src/config.py index 9eb7c45..7a09a4b 100644 --- a/src/config.py +++ b/src/config.py @@ -1,7 +1,7 @@ import toml -from src.exceptions import ConfigurationError -from src.model import SsisDeployment +from exceptions import ConfigurationError +from model import SsisDeployment def load_configuration(configuration: str) -> SsisDeployment: diff --git a/src/deploy.py b/src/deploy.py index 891829e..1b79c65 100644 --- a/src/deploy.py +++ b/src/deploy.py @@ -1,6 +1,6 @@ -from src.exceptions import SqlAgentOperatorException -from src.model import SsisDeployment -from src.sql.db import Database +from exceptions import SqlAgentOperatorException +from model import SsisDeployment +from sql.db import Database def deploy_ssis( diff --git a/src/model.py b/src/model.py index 3d93b9f..9398918 100644 --- a/src/model.py +++ b/src/model.py @@ -4,7 +4,7 @@ from dataclasses_json import config, dataclass_json -from src.exceptions import ConfigurationError +from exceptions import ConfigurationError class FrequencyType(Enum): diff --git a/src/sql-deployment-tools.py b/src/sql-deployment-tools.py index 6800b12..e7d4290 100644 --- a/src/sql-deployment-tools.py +++ b/src/sql-deployment-tools.py @@ -3,8 +3,8 @@ import os import sys -from src.config import ConfigurationError, load_configuration -from src.deploy import deploy_ssis +from config import ConfigurationError, load_configuration +from deploy import deploy_ssis if __name__ == "__main__": parent_parser = argparse.ArgumentParser(description="SSIS Deployment Helper") diff --git a/test/test_config.py b/test/test_config.py index b6885df..7c100d8 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -4,9 +4,9 @@ import toml from typing_extensions import assert_type -from src.config import load_configuration -from src.exceptions import ConfigurationError -from src.model import SsisDeployment +from config import load_configuration +from exceptions import ConfigurationError +from model import SsisDeployment class TestConfig: diff --git a/test/test_model.py b/test/test_model.py index 24616f6..df9da01 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -5,9 +5,9 @@ import toml from typing_extensions import assert_type -from src.config import load_configuration -from src.exceptions import ConfigurationError -from src.model import ( +from config import load_configuration +from exceptions import ConfigurationError +from model import ( DayOfWeekFrequencyInterval, FrequencyInterval, FrequencyType, From 2eb29e004340d9a04a5e937493e69c1c345a570b Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 21 Jul 2022 14:00:49 +0100 Subject: [PATCH 13/17] added src back in --- src/config.py | 4 ++-- src/deploy.py | 6 +++--- src/model.py | 2 +- src/sql-deployment-tools.py | 4 ++-- src/sql/db.py | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/config.py b/src/config.py index 7a09a4b..9eb7c45 100644 --- a/src/config.py +++ b/src/config.py @@ -1,7 +1,7 @@ import toml -from exceptions import ConfigurationError -from model import SsisDeployment +from src.exceptions import ConfigurationError +from src.model import SsisDeployment def load_configuration(configuration: str) -> SsisDeployment: diff --git a/src/deploy.py b/src/deploy.py index 1b79c65..891829e 100644 --- a/src/deploy.py +++ b/src/deploy.py @@ -1,6 +1,6 @@ -from exceptions import SqlAgentOperatorException -from model import SsisDeployment -from sql.db import Database +from src.exceptions import SqlAgentOperatorException +from src.model import SsisDeployment +from src.sql.db import Database def deploy_ssis( diff --git a/src/model.py b/src/model.py index 9398918..3d93b9f 100644 --- a/src/model.py +++ b/src/model.py @@ -4,7 +4,7 @@ from dataclasses_json import config, dataclass_json -from exceptions import ConfigurationError +from src.exceptions import ConfigurationError class FrequencyType(Enum): diff --git a/src/sql-deployment-tools.py b/src/sql-deployment-tools.py index e7d4290..6800b12 100644 --- a/src/sql-deployment-tools.py +++ b/src/sql-deployment-tools.py @@ -3,8 +3,8 @@ import os import sys -from config import ConfigurationError, load_configuration -from deploy import deploy_ssis +from src.config import ConfigurationError, load_configuration +from src.deploy import deploy_ssis if __name__ == "__main__": parent_parser = argparse.ArgumentParser(description="SSIS Deployment Helper") diff --git a/src/sql/db.py b/src/sql/db.py index 51b5a73..61ad9a4 100644 --- a/src/sql/db.py +++ b/src/sql/db.py @@ -3,9 +3,9 @@ import pyodbc import sqlparams -from exceptions import SqlAgentOperatorException -from model import NotifyLevelEmail -from sql import query +from src.exceptions import SqlAgentOperatorException +from src.model import NotifyLevelEmail +from src.sql import query class Database: From 70f3c3c0e99d30d8503d9a075bf3ae31281bb55e Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 21 Jul 2022 14:07:44 +0100 Subject: [PATCH 14/17] removed src. from imports --- src/config.py | 4 ++-- src/deploy.py | 6 +++--- src/model.py | 2 +- src/sql-deployment-tools.py | 4 ++-- src/sql/db.py | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/config.py b/src/config.py index 9eb7c45..7a09a4b 100644 --- a/src/config.py +++ b/src/config.py @@ -1,7 +1,7 @@ import toml -from src.exceptions import ConfigurationError -from src.model import SsisDeployment +from exceptions import ConfigurationError +from model import SsisDeployment def load_configuration(configuration: str) -> SsisDeployment: diff --git a/src/deploy.py b/src/deploy.py index 891829e..1b79c65 100644 --- a/src/deploy.py +++ b/src/deploy.py @@ -1,6 +1,6 @@ -from src.exceptions import SqlAgentOperatorException -from src.model import SsisDeployment -from src.sql.db import Database +from exceptions import SqlAgentOperatorException +from model import SsisDeployment +from sql.db import Database def deploy_ssis( diff --git a/src/model.py b/src/model.py index 3d93b9f..9398918 100644 --- a/src/model.py +++ b/src/model.py @@ -4,7 +4,7 @@ from dataclasses_json import config, dataclass_json -from src.exceptions import ConfigurationError +from exceptions import ConfigurationError class FrequencyType(Enum): diff --git a/src/sql-deployment-tools.py b/src/sql-deployment-tools.py index 6800b12..e7d4290 100644 --- a/src/sql-deployment-tools.py +++ b/src/sql-deployment-tools.py @@ -3,8 +3,8 @@ import os import sys -from src.config import ConfigurationError, load_configuration -from src.deploy import deploy_ssis +from config import ConfigurationError, load_configuration +from deploy import deploy_ssis if __name__ == "__main__": parent_parser = argparse.ArgumentParser(description="SSIS Deployment Helper") diff --git a/src/sql/db.py b/src/sql/db.py index 61ad9a4..51b5a73 100644 --- a/src/sql/db.py +++ b/src/sql/db.py @@ -3,9 +3,9 @@ import pyodbc import sqlparams -from src.exceptions import SqlAgentOperatorException -from src.model import NotifyLevelEmail -from src.sql import query +from exceptions import SqlAgentOperatorException +from model import NotifyLevelEmail +from sql import query class Database: From 3b986930fc664cfc92c04d738c2b7637cc11ac0a Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 21 Jul 2022 14:17:50 +0100 Subject: [PATCH 15/17] added src. to imports --- pytest.ini | 1 - src/config.py | 4 ++-- src/deploy.py | 6 +++--- src/model.py | 2 +- src/sql-deployment-tools.py | 4 ++-- src/sql/db.py | 6 +++--- test/test_config.py | 6 +++--- test/test_model.py | 6 +++--- 8 files changed, 17 insertions(+), 18 deletions(-) diff --git a/pytest.ini b/pytest.ini index 490c27f..c2a30b0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,2 @@ [pytest] addopts = --cov=./src/ --cov-config=./.coveragerc --junitxml=./test-results/unit-test-results.xml -v -rF -rP -pythonpath = src diff --git a/src/config.py b/src/config.py index 7a09a4b..9eb7c45 100644 --- a/src/config.py +++ b/src/config.py @@ -1,7 +1,7 @@ import toml -from exceptions import ConfigurationError -from model import SsisDeployment +from src.exceptions import ConfigurationError +from src.model import SsisDeployment def load_configuration(configuration: str) -> SsisDeployment: diff --git a/src/deploy.py b/src/deploy.py index 1b79c65..891829e 100644 --- a/src/deploy.py +++ b/src/deploy.py @@ -1,6 +1,6 @@ -from exceptions import SqlAgentOperatorException -from model import SsisDeployment -from sql.db import Database +from src.exceptions import SqlAgentOperatorException +from src.model import SsisDeployment +from src.sql.db import Database def deploy_ssis( diff --git a/src/model.py b/src/model.py index 9398918..3d93b9f 100644 --- a/src/model.py +++ b/src/model.py @@ -4,7 +4,7 @@ from dataclasses_json import config, dataclass_json -from exceptions import ConfigurationError +from src.exceptions import ConfigurationError class FrequencyType(Enum): diff --git a/src/sql-deployment-tools.py b/src/sql-deployment-tools.py index e7d4290..6800b12 100644 --- a/src/sql-deployment-tools.py +++ b/src/sql-deployment-tools.py @@ -3,8 +3,8 @@ import os import sys -from config import ConfigurationError, load_configuration -from deploy import deploy_ssis +from src.config import ConfigurationError, load_configuration +from src.deploy import deploy_ssis if __name__ == "__main__": parent_parser = argparse.ArgumentParser(description="SSIS Deployment Helper") diff --git a/src/sql/db.py b/src/sql/db.py index 51b5a73..61ad9a4 100644 --- a/src/sql/db.py +++ b/src/sql/db.py @@ -3,9 +3,9 @@ import pyodbc import sqlparams -from exceptions import SqlAgentOperatorException -from model import NotifyLevelEmail -from sql import query +from src.exceptions import SqlAgentOperatorException +from src.model import NotifyLevelEmail +from src.sql import query class Database: diff --git a/test/test_config.py b/test/test_config.py index 7c100d8..b6885df 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -4,9 +4,9 @@ import toml from typing_extensions import assert_type -from config import load_configuration -from exceptions import ConfigurationError -from model import SsisDeployment +from src.config import load_configuration +from src.exceptions import ConfigurationError +from src.model import SsisDeployment class TestConfig: diff --git a/test/test_model.py b/test/test_model.py index df9da01..24616f6 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -5,9 +5,9 @@ import toml from typing_extensions import assert_type -from config import load_configuration -from exceptions import ConfigurationError -from model import ( +from src.config import load_configuration +from src.exceptions import ConfigurationError +from src.model import ( DayOfWeekFrequencyInterval, FrequencyInterval, FrequencyType, From b2cb368384d303d6f76c771ffae714818e031d7e Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Thu, 21 Jul 2022 14:22:49 +0100 Subject: [PATCH 16/17] imports as main --- src/sql/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/db.py b/src/sql/db.py index 61ad9a4..7c35716 100644 --- a/src/sql/db.py +++ b/src/sql/db.py @@ -3,9 +3,9 @@ import pyodbc import sqlparams +from sql import query from src.exceptions import SqlAgentOperatorException from src.model import NotifyLevelEmail -from src.sql import query class Database: From c1ec61c95a5b2d9b363da825da7c23851be50252 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Fri, 22 Jul 2022 15:53:33 +0100 Subject: [PATCH 17/17] added time window --- README.md | 18 +++++----- src/deploy.py | 1 + src/model.py | 9 ++++- src/sql/agent/create_job_schedule.sql | 1 + src/sql/db.py | 2 ++ test/conftest.py | 8 ++++- test/test_model.py | 47 +++++++++++++++++++++++---- 7 files changed, 69 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 8caee31..dff5d19 100644 --- a/README.md +++ b/README.md @@ -166,11 +166,13 @@ Driver={SQL Server Native Client 11.0};Server=.;Database=SSISDB;Trusted_Connecti ### Schedules TOML keys -| key | type | required | allowable values | description | -|-----------------|-----------|---------------------|---------------------------------------------------------------------------------------|-------------------------------------------------------------------| -| name | str | yes | any string | schedule name | -| unit | str | yes | "MINUTE" "DAY" "WEEK" "MONTH" | time unit | -| every\_n\_units | int | yes | any integer | x value in: repeats every _x unit_s | -| schedule\_time | int | no _default:0_ | integer time value HHMMSS in 24h clock e.g 0 is midnight, 70000 is 7am, 190000 is 7pm | job scheduled start time in 24h clock | -| run\_days | List[str] | when unit = "WEEK" | str values: "SUNDAY" "MONDAY" "TUESDAY" "WEDNESDAY" "THURSDAY" "FRIDAY" "SATURDAY" | days of the week that job should run | -| day\_of\_month | int | when unit = "MONTH" | any integer | day of the month to run scheduled job. 1 being first day of month | +| key | type | required | allowable values | description | +|-----|------|----------|------------------|-------------| +| name | str | yes | any string | schedule name | +| unit | str | yes | "MINUTE" "DAY" "WEEK" "MONTH" | time unit | +| every\_n\_units | int | yes | any integer | x value in: repeats every _x unit_s | +| schedule\_time | int | no _default:0_ | integer time value HHMMSS in 24h clock e.g 0 is midnight, 70000 is 7am, 190000 is 7pm | job scheduled start time in 24h clock | +| run\_days | List[str] | when unit = "WEEK" | str values: "SUNDAY" "MONDAY" "TUESDAY" "WEDNESDAY" "THURSDAY" "FRIDAY" "SATURDAY" | days of the week that job should run | +| day\_of\_month | int | when unit = "MONTH" | any integer | day of the month to run scheduled job. 1 being first day of month | +| window_start | int _default:0_| optional when unit = "MINUTE", ignored otherwise | integer time value HHMMSS in 24h clock e.g 0 is midnight, 70000 is 7am, 190000 is 7pm | start time of window in which to run job | +| window_end | int _default:235959_| optional when unit = "MINUTE", ignored otherwise | integer time value HHMMSS in 24h clock e.g 0 is midnight, 70000 is 7am, 190000 is 7pm | end time of window in which to run job | diff --git a/src/deploy.py b/src/deploy.py index 891829e..7c0cbc3 100644 --- a/src/deploy.py +++ b/src/deploy.py @@ -78,6 +78,7 @@ def deploy_ssis( parameters.freq_subday_interval, parameters.freq_recurrence_factor, parameters.active_start_time, + parameters.active_end_time, ) if ssis_deployment.job.notification_email_address: diff --git a/src/model.py b/src/model.py index 3d93b9f..82bd880 100644 --- a/src/model.py +++ b/src/model.py @@ -59,6 +59,7 @@ class ScheduleQueryParameters: freq_subday_interval: int freq_recurrence_factor: int active_start_time: int + active_end_time: int @dataclass_json @@ -68,6 +69,8 @@ class Schedule: unit: str every_n_units: int schedule_time: int = 0 + window_start: int = 0 + window_end: int = 235959 run_days: typing.Optional[typing.List[str]] = None day_of_month: typing.Optional[int] = None @@ -79,7 +82,8 @@ def transform_for_query(self) -> ScheduleQueryParameters: 4, self.every_n_units, 0, - self.schedule_time, + self.window_start, + self.window_end, ) if self.unit == "DAY": return ScheduleQueryParameters( @@ -89,6 +93,7 @@ def transform_for_query(self) -> ScheduleQueryParameters: 0, 0, self.schedule_time, + 235959, ) if self.unit == "WEEK": return ScheduleQueryParameters( @@ -98,6 +103,7 @@ def transform_for_query(self) -> ScheduleQueryParameters: 0, self.every_n_units, self.schedule_time, + 235959, ) if self.unit == "MONTH": return ScheduleQueryParameters( @@ -107,6 +113,7 @@ def transform_for_query(self) -> ScheduleQueryParameters: 0, self.every_n_units, self.schedule_time, + 235959, ) def __post_init__(self): diff --git a/src/sql/agent/create_job_schedule.sql b/src/sql/agent/create_job_schedule.sql index 37dfaf8..354635e 100644 --- a/src/sql/agent/create_job_schedule.sql +++ b/src/sql/agent/create_job_schedule.sql @@ -11,6 +11,7 @@ EXEC msdb.dbo.sp_add_jobschedule , @freq_relative_interval = 0 , @freq_recurrence_factor = $freq_recurrence_factor , @active_start_time = $active_start_time + , @active_end_time = $active_end_time , @schedule_id = @schedule_id OUTPUT ; diff --git a/src/sql/db.py b/src/sql/db.py index 7c35716..979cf8e 100644 --- a/src/sql/db.py +++ b/src/sql/db.py @@ -244,6 +244,7 @@ def agent_create_job_schedule( freq_subday_interval: int, freq_recurrence_factor: int, active_start_time: int, + active_end_time: int, ): self._execute_sql( query.agent_create_job_schedule, @@ -256,6 +257,7 @@ def agent_create_job_schedule( "freq_subday_interval": freq_subday_interval, "freq_recurrence_factor": freq_recurrence_factor, "active_start_time": active_start_time, + "active_end_time": active_end_time, }, ) diff --git a/test/conftest.py b/test/conftest.py index 3bb65da..fbefa65 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -34,7 +34,13 @@ "every_n_units": 30, "schedule_time": 200000, }, - {"name": "Autumn Mountain", "unit": "MINUTE", "every_n_units": 1440}, + { + "name": "Autumn Mountain", + "unit": "MINUTE", + "every_n_units": 1440, + "window_start": 100000, + "window_end": 120000, + }, { "name": "Peaceful Valley", "unit": "WEEK", diff --git a/test/test_model.py b/test/test_model.py index 24616f6..efdb17d 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -296,9 +296,16 @@ def test_SsisDeployment_Job_Schedules_are_set_correctly(self): """ expected = [ Schedule("Winter Moon", "DAY", 30, 200000), - Schedule("Autumn Mountain", "MINUTE", 1440, 0), - Schedule("Peaceful Valley", "WEEK", 1, 0, ["MONDAY", "FRIDAY"]), - Schedule("Snowfall", "MONTH", 1, 0, None, 15), + Schedule( + "Autumn Mountain", + "MINUTE", + 1440, + 0, + 100000, + 120000, + ), + Schedule("Peaceful Valley", "WEEK", 1, 0, 0, 235959, ["MONDAY", "FRIDAY"]), + Schedule("Snowfall", "MONTH", 1, 0, 0, 235959, None, 15), ] actual = load_configuration(toml.dumps(TEST_CONFIG)) assert actual.job.schedules == expected @@ -308,10 +315,10 @@ def test_SsisDeployment_Job_Schedule_Transforms_are_correct(self): Test schedules transform into sp_schedule_job variables correctly """ expected = [ - ScheduleQueryParameters(4, 30, 1, 0, 0, 200000), - ScheduleQueryParameters(4, 1, 4, 1440, 0, 0), - ScheduleQueryParameters(8, 34, 1, 0, 1, 0), - ScheduleQueryParameters(16, 15, 1, 0, 1, 0), + ScheduleQueryParameters(4, 30, 1, 0, 0, 200000, 235959), + ScheduleQueryParameters(4, 1, 4, 1440, 0, 100000, 120000), + ScheduleQueryParameters(8, 34, 1, 0, 1, 0, 235959), + ScheduleQueryParameters(16, 15, 1, 0, 1, 0, 235959), ] actual = load_configuration(toml.dumps(TEST_CONFIG)) assert [x.transform_for_query() for x in actual.job.schedules] == expected @@ -484,6 +491,32 @@ def test_SsisDeployment_throws_an_exception_when_schedule_minutes_is_not_an_inte with pytest.raises(ConfigurationError): load_configuration(toml.dumps(config)) + def test_SsisDeployment_throws_an_exception_when_unit_is_day_and_run_days_missing( + self, + ): + """ + Test that if unit = 'WEEK' and run_days not present there is an error. + """ + + config = copy.deepcopy(TEST_CONFIG) + del config["job"]["schedules"][2]["run_days"] + + with pytest.raises(ConfigurationError): + load_configuration(toml.dumps(config)) + + def test_SsisDeployment_throws_exception_if_unit_is_month_and_day_of_month_missing( + self, + ): + """ + Test that if unit = 'MONTH' and day_of_month not present there is an error. + """ + + config = copy.deepcopy(TEST_CONFIG) + del config["job"]["schedules"][3]["day_of_month"] + + with pytest.raises(ConfigurationError): + load_configuration(toml.dumps(config)) + def test_SsisDeployment_throws_exception_step_type_is_ssis_tsql_command_set( self, ):