Merge pull request #32 from socialpoint-labs/fixies-and-improvements

integrate several fixies and improvements from separated branches

.github/workflows/publish.yml

@@ -0,0 +1,27 @@
+name: Upload Python Package
+on:
+  release:
+    types: [published]
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.6", "3.7", "3.8"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+      - name: Build package
+        run: python -m build
+      - name: Publish package
+        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

Makefile

@@ -13,12 +13,12 @@ clean-pyc:
 	find . -name '*~' -exec rm -f {} +
 
 deploy-loc:
-	python setup.py build
-	python setup.py install
+	python3 -m pip install --upgrade build
+	python3 -m build
 
 release:
-	python setup.py sdist bdist_wheel
-	twine upload dist/*
+	python3 -m pip install --upgrade twine
+	python3 -m twine upload dist/*
 
 empty-dist:
 	rm dist/*

pyproject.toml

@@ -0,0 +1,45 @@
+[project]
+name = "sqlbucket"
+version = "0.4.2"
+authors = [
+  { name="Philippe Oger", email="phil.oger@gmail.com" },
+]
+description = "SQLBucket - Write your SQL ETL flow and ETL integrity tool."
+license = {file = "LICENSE"}
+readme = "README.md"
+requires-python = ">=3.7"
+keywords = ['sql', 'etl', 'data-integrity']
+classifiers=[
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: End Users/Desktop",
+    "Intended Audience :: Science/Research",
+    "Topic :: Database :: Front-Ends",
+    "Topic :: Software Development :: Libraries :: Python Modules"
+    ]
+dependencies = [
+    'click>=6.0',
+    'jinja2==2.10.1',
+    'markupsafe==1.1.1',
+    'pyyaml>=5.0',
+    'sqlalchemy~=1.3',
+    'tabulate>=0.7.5'
+]
+
+[project.optional-dependencies]
+dev = [
+    'pytest>=4',
+    'coverage'
+]
+
+[project.urls]
+"Homepage" = "https://github.com/socialpoint-labs/sqlbucket"
+
+
+[build-system]
+requires = ["setuptools>=45"]
+build-backend = "setuptools.build_meta"

sqlbucket/__init__.py

@@ -1,8 +1,2 @@
-
-
-__version__ = "0.4.1"
-
-
 from sqlbucket.core import SQLBucket
 from sqlbucket.project import Project
-

sqlbucket/cli.py

@@ -35,11 +35,13 @@ def create_project(sqlbucket, name):
     @click.option('--rendering', '-r', is_flag=True, help="Only render queries")
     @click.option('--all', '-all', is_flag=True, help="All dbs")
     @click.option('--edb', '-x', required=False, type=str, help="Excluded dbs")
+    @click.option('--silent', '-s', required=False, is_flag=True, default=False,
+                  help="Do not notify execution status.")
     @click.pass_obj
     @click.argument('args', nargs=-1)
     def run_job(sqlbucket, name, db, fstep, tstep, to_date, from_date,
                 from_days, to_days, group, isolation, verbose, rendering,
-                all, edb, args):
+                all, edb, silent, args):
 
         submitted_variables = cli_variables_parser(args)
 
@@ -90,7 +92,8 @@ def run_job(sqlbucket, name, db, fstep, tstep, to_date, from_date,
                     to_step=tstep,
                     group=group,
                     verbose=verbose,
-                    isolation_level=isolation
+                    isolation_level=isolation,
+                    silent=silent
                 )
 
     @cli.command(context_settings=dict(ignore_unknown_options=True))

sqlbucket/integrity.py

@@ -1,4 +1,4 @@
-from sqlbucket.runners import create_connection
+from sqlbucket.runners import create_connection, connection_query
 from sqlbucket.runners import logger
 from tabulate import tabulate
 from sqlbucket.utils import integrity_logo, success
@@ -42,6 +42,7 @@ def run_integrity(configuration: dict, prefix: str = '', verbose: bool = False):
             errors += 1
             logger.info(f'Query {query_name} encountered an error:')
             logger.error(e)
+            connection_query(configuration, connection)
             continue
 
     # logging summary

sqlbucket/macros/common.jinja2

@@ -66,7 +66,7 @@ from {{ table }}
 
 {% macro are_within_threshold(scalar_a, scalar_b, threshold, integrity_check_name=None) %}
 select
-    {% if not integrity_check_name %}'source_to_target_integrity_check{% else %}{{ integrity_check_name }}{% endif %} as integrity_check
+    {% if not integrity_check_name %}'source_to_target_integrity_check'{% else %}{{ integrity_check_name }}{% endif %} as integrity_check,
     ({{ scalar_a }}) as expected,
     ({{ scalar_b }}) as calculated,
     abs((({{ scalar_a }}) - ({{ scalar_b }})) / nullif(({{ scalar_b }}), 0)) <= {{ threshold }}

sqlbucket/project.py

@@ -89,6 +89,35 @@ def configure_integrity(self) -> dict:
             "connection_query": self.get_connection_query()
         }
 
+    def send_msg(run_func):
+
+        def wrapper(self, *args, **kwargs):
+
+            silent = kwargs.pop("silent", False)
+
+            if silent:
+                run_func(self, *args, **kwargs)
+            else:
+                group = kwargs.get("group")
+                config = self.configure(group)
+
+                if 'context' in config and 'f' in config['context']:
+                    funcs_reg = config['context']['f']
+                    start_msg = funcs_reg.get('start_msg')
+                    end_msg = funcs_reg.get('end_msg')
+
+                    try:
+                        if start_msg: start_msg(**config)
+                        run_func(self, *args, **kwargs)
+                        if end_msg: end_msg(**config)
+                    except:
+                        exception_msg = funcs_reg.get('exception_msg')
+                        if exception_msg: exception_msg(**config)
+                        raise
+
+        return wrapper
+
+    @send_msg
     def run(self, group: str = None, from_step: int = 1, to_step: int = None,
             verbose: bool = False, isolation_level: str = None) -> None:
         configuration = self.configure(group)

sqlbucket/runners.py

@@ -109,9 +109,13 @@ def create_connection(
             isolation_level=isolation_level
         )
     connection = engine.connect()
+    connection_query(configuration, connection)
+
+    return connection
+
+
+def connection_query(configuration: dict, connection: Connection):
     if configuration.get('connection_query') is not None:
         logger.info(f'Running connection query: '
                     f'{configuration["connection_query"]}')
         connection.execute(text(configuration['connection_query']))
-
-    return connection