From e947e427150eddd77f7c2ab1c742336776554172 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 25 Oct 2024 13:43:16 -0400 Subject: [PATCH 1/6] Updating Ownership hierarchy to allow schemachange to create the schema if it does not exist. --- CHANGELOG.md | 8 ++++++++ demo/provision/setup_schemachange_schema.sql | 1 + demo/setup/basics_demo/A__setup_basics_demo.sql | 1 + demo/setup/citibike_demo/A__setup_citibike_demo.sql | 1 + .../citibike_demo_jinja/A__setup_citibike_demo_jinja.sql | 1 + schemachange/cli.py | 2 +- tests/test_cli_misc.py | 2 +- 7 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 656504ce..f409859f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file. *The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).* +## [4.0.0] - TBD +### Added +- use of structlog for standard log outputs + +### Changed +- Refactored the main cli.py to be more modular to aide in future development and testing. + + ## [3.7.0] - 2024-07-22 ### Added - Improved unit test coverage diff --git a/demo/provision/setup_schemachange_schema.sql b/demo/provision/setup_schemachange_schema.sql index 0c4322ce..6be6ad30 100644 --- a/demo/provision/setup_schemachange_schema.sql +++ b/demo/provision/setup_schemachange_schema.sql @@ -38,6 +38,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C); CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS; -- USE SCHEMA INFORMATION_SCHEMA; -- DROP SCHEMA IF EXISTS PUBLIC; +GRANT OWNERSHIP ON IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE); -- SCHEMA diff --git a/demo/setup/basics_demo/A__setup_basics_demo.sql b/demo/setup/basics_demo/A__setup_basics_demo.sql index a39e16fc..31f48df3 100644 --- a/demo/setup/basics_demo/A__setup_basics_demo.sql +++ b/demo/setup/basics_demo/A__setup_basics_demo.sql @@ -31,6 +31,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C); CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS; -- USE SCHEMA INFORMATION_SCHEMA; -- DROP SCHEMA IF EXISTS PUBLIC; +GRANT OWNERSHIP ON IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE); -- SCHEMA diff --git a/demo/setup/citibike_demo/A__setup_citibike_demo.sql b/demo/setup/citibike_demo/A__setup_citibike_demo.sql index a39e16fc..31f48df3 100644 --- a/demo/setup/citibike_demo/A__setup_citibike_demo.sql +++ b/demo/setup/citibike_demo/A__setup_citibike_demo.sql @@ -31,6 +31,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C); CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS; -- USE SCHEMA INFORMATION_SCHEMA; -- DROP SCHEMA IF EXISTS PUBLIC; +GRANT OWNERSHIP ON IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE); -- SCHEMA diff --git a/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql b/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql index a39e16fc..31f48df3 100644 --- a/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql +++ b/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql @@ -31,6 +31,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C); CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS; -- USE SCHEMA INFORMATION_SCHEMA; -- DROP SCHEMA IF EXISTS PUBLIC; +GRANT OWNERSHIP ON IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE); -- SCHEMA diff --git a/schemachange/cli.py b/schemachange/cli.py index d6fe54f0..f896d096 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -13,7 +13,7 @@ # region Global Variables # metadata -SCHEMACHANGE_VERSION = "3.7.0" +SCHEMACHANGE_VERSION = "4.0.0" SNOWFLAKE_APPLICATION_NAME = "schemachange" module_logger = structlog.getLogger(__name__) diff --git a/tests/test_cli_misc.py b/tests/test_cli_misc.py index e5877b17..97bbc686 100644 --- a/tests/test_cli_misc.py +++ b/tests/test_cli_misc.py @@ -10,7 +10,7 @@ def test_cli_given__schemachange_version_change_updated_in_setup_config_file(): - assert SCHEMACHANGE_VERSION == "3.7.0" + assert SCHEMACHANGE_VERSION == "4.0.0" def test_cli_given__constants_exist(): From 8d15e52dcc6385b809fc75c42721e4f1da336b69 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 25 Oct 2024 13:50:22 -0400 Subject: [PATCH 2/6] fixing syntax issue --- demo/provision/setup_schemachange_schema.sql | 2 +- demo/setup/basics_demo/A__setup_basics_demo.sql | 2 +- demo/setup/citibike_demo/A__setup_citibike_demo.sql | 2 +- demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/demo/provision/setup_schemachange_schema.sql b/demo/provision/setup_schemachange_schema.sql index 6be6ad30..906c1326 100644 --- a/demo/provision/setup_schemachange_schema.sql +++ b/demo/provision/setup_schemachange_schema.sql @@ -38,7 +38,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C); CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS; -- USE SCHEMA INFORMATION_SCHEMA; -- DROP SCHEMA IF EXISTS PUBLIC; -GRANT OWNERSHIP ON IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); +GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE); -- SCHEMA diff --git a/demo/setup/basics_demo/A__setup_basics_demo.sql b/demo/setup/basics_demo/A__setup_basics_demo.sql index 31f48df3..2d7edf78 100644 --- a/demo/setup/basics_demo/A__setup_basics_demo.sql +++ b/demo/setup/basics_demo/A__setup_basics_demo.sql @@ -31,7 +31,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C); CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS; -- USE SCHEMA INFORMATION_SCHEMA; -- DROP SCHEMA IF EXISTS PUBLIC; -GRANT OWNERSHIP ON IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); +GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE); -- SCHEMA diff --git a/demo/setup/citibike_demo/A__setup_citibike_demo.sql b/demo/setup/citibike_demo/A__setup_citibike_demo.sql index 31f48df3..2d7edf78 100644 --- a/demo/setup/citibike_demo/A__setup_citibike_demo.sql +++ b/demo/setup/citibike_demo/A__setup_citibike_demo.sql @@ -31,7 +31,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C); CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS; -- USE SCHEMA INFORMATION_SCHEMA; -- DROP SCHEMA IF EXISTS PUBLIC; -GRANT OWNERSHIP ON IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); +GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE); -- SCHEMA diff --git a/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql b/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql index 31f48df3..2d7edf78 100644 --- a/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql +++ b/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql @@ -31,7 +31,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C); CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS; -- USE SCHEMA INFORMATION_SCHEMA; -- DROP SCHEMA IF EXISTS PUBLIC; -GRANT OWNERSHIP ON IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); +GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE); -- SCHEMA From 2537bb9ee323e770c9891921fd98eaee57a73cbd Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 25 Oct 2024 15:29:33 -0400 Subject: [PATCH 3/6] checking the default schema creation step --- demo/basics_demo/schemachange-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/basics_demo/schemachange-config.yml b/demo/basics_demo/schemachange-config.yml index 18680db4..0e23a72e 100644 --- a/demo/basics_demo/schemachange-config.yml +++ b/demo/basics_demo/schemachange-config.yml @@ -7,7 +7,7 @@ snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}" snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}" snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}" snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}" -snowflake-schema: "{{ env_var('MY_TARGET_SCHEMA')}}" +# snowflake-schema: "{{ env_var('MY_TARGET_SCHEMA')}}" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true From fc480ae9fecadd4b60db34d744dfacb6baf9f514 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 25 Oct 2024 19:39:19 -0400 Subject: [PATCH 4/6] adding missing schema reference in demo script --- demo/basics_demo/V1.0.2__StoredProc.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/demo/basics_demo/V1.0.2__StoredProc.sql b/demo/basics_demo/V1.0.2__StoredProc.sql index bea83a0c..dca2436d 100644 --- a/demo/basics_demo/V1.0.2__StoredProc.sql +++ b/demo/basics_demo/V1.0.2__StoredProc.sql @@ -1,3 +1,5 @@ +use database {{ database_name }}; +use schema {{ schema_name }}; -- This block of code executes in Visual Studio Code but fails in Schemachange. -- Use the $$ ... $$ to mark the block and execute the code block successfully. -- The comment from a community user help find the root cause. From 1a2f1e76f5b1b8bd819e0e15e7347032dfa266bb Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 25 Oct 2024 19:40:02 -0400 Subject: [PATCH 5/6] Added create schema privilege to deploy role because schemachange tries to create the default schema if it does not exist. --- demo/provision/setup_schemachange_schema.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/demo/provision/setup_schemachange_schema.sql b/demo/provision/setup_schemachange_schema.sql index 906c1326..e1eef074 100644 --- a/demo/provision/setup_schemachange_schema.sql +++ b/demo/provision/setup_schemachange_schema.sql @@ -20,6 +20,8 @@ CREATE DATABASE ROLE IF NOT EXISTS DB_R; CREATE DATABASE ROLE IF NOT EXISTS DB_W; CREATE DATABASE ROLE IF NOT EXISTS DB_C; +GRANT CREATE SCHEMA ON DATABASE IDENTIFIER($TARGET_DB_NAME) TO DATABASE ROLE DB_C; + GRANT DATABASE ROLE DB_C TO ROLE IDENTIFIER($DEPLOY_ROLE); CREATE DATABASE ROLE IF NOT EXISTS IDENTIFIER($SC_M); From 4a261ecb9973ac9466910e84f3572b362dcce8d2 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Fri, 25 Oct 2024 19:45:10 -0400 Subject: [PATCH 6/6] Restore the schema reference in demo config --- demo/basics_demo/schemachange-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/basics_demo/schemachange-config.yml b/demo/basics_demo/schemachange-config.yml index 0e23a72e..18680db4 100644 --- a/demo/basics_demo/schemachange-config.yml +++ b/demo/basics_demo/schemachange-config.yml @@ -7,7 +7,7 @@ snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}" snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}" snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}" snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}" -# snowflake-schema: "{{ env_var('MY_TARGET_SCHEMA')}}" +snowflake-schema: "{{ env_var('MY_TARGET_SCHEMA')}}" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true