diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..6f8eb9f --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,27 @@ +name: Upload Python Package +on: + release: + types: [published] +jobs: + deploy: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.6", "3.7", "3.8"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/Makefile b/Makefile index a483759..dd4b2f2 100644 --- a/Makefile +++ b/Makefile @@ -13,12 +13,12 @@ clean-pyc: find . -name '*~' -exec rm -f {} + deploy-loc: - python setup.py build - python setup.py install + python3 -m pip install --upgrade build + python3 -m build release: - python setup.py sdist bdist_wheel - twine upload dist/* + python3 -m pip install --upgrade twine + python3 -m twine upload dist/* empty-dist: rm dist/* diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ddd8b00 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,45 @@ +[project] +name = "sqlbucket" +version = "0.4.2" +authors = [ + { name="Philippe Oger", email="phil.oger@gmail.com" }, +] +description = "SQLBucket - Write your SQL ETL flow and ETL integrity tool." +license = {file = "LICENSE"} +readme = "README.md" +requires-python = ">=3.7" +keywords = ['sql', 'etl', 'data-integrity'] +classifiers=[ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: End Users/Desktop", + "Intended Audience :: Science/Research", + "Topic :: Database :: Front-Ends", + "Topic :: Software Development :: Libraries :: Python Modules" + ] +dependencies = [ + 'click>=6.0', + 'jinja2==2.10.1', + 'markupsafe==1.1.1', + 'pyyaml>=5.0', + 'sqlalchemy~=1.3', + 'tabulate>=0.7.5' +] + +[project.optional-dependencies] +dev = [ + 'pytest>=4', + 'coverage' +] + +[project.urls] +"Homepage" = "https://github.com/socialpoint-labs/sqlbucket" + + +[build-system] +requires = ["setuptools>=45"] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py deleted file mode 100644 index c3811d6..0000000 --- a/setup.py +++ /dev/null @@ -1,58 +0,0 @@ -import io -import re -from setuptools import setup - - -with io.open('sqlbucket/__init__.py', 'rt', encoding='utf8') as f: - version = re.search( - r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', - f.read(), - re.MULTILINE - ).group(1) - - -setup( - name='sqlbucket', - packages=[ - 'sqlbucket', - 'sqlbucket.macros', - 'sqlbucket.template', - 'sqlbucket.template.queries', - 'sqlbucket.template.integrity' - ], - include_package_data=True, - description='SQLBucket - Write your SQL ETL flow and ETL integrity tool.', - long_description=open('README.rst', 'r').read(), - version=version, - author='Philippe Oger', - author_email='phil.oger@gmail.com', - url='https://github.com/socialpoint-labs/sqlbucket', - keywords=['sql', 'etl', 'data-integrity'], - install_requires=[ - 'click>=6.0', - 'jinja2==2.10.1', - 'markupsafe==1.1.1', - 'pyyaml>=5.0', - 'sqlalchemy~=1.3', - 'tabulate>=0.7.5' - ], - extras_require={ - 'dev': [ - 'pytest>=4', - 'coverage' - ] - }, - classifiers=[ - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "Intended Audience :: End Users/Desktop", - "Intended Audience :: Science/Research", - "Topic :: Database :: Front-Ends", - "Topic :: Software Development :: Libraries :: Python Modules" - ], - license='MIT' -) diff --git a/sqlbucket/__init__.py b/sqlbucket/__init__.py index 0a8ab84..df2d877 100644 --- a/sqlbucket/__init__.py +++ b/sqlbucket/__init__.py @@ -1,8 +1,2 @@ - - -__version__ = "0.4.1" - - from sqlbucket.core import SQLBucket from sqlbucket.project import Project - diff --git a/sqlbucket/cli.py b/sqlbucket/cli.py index 70d373a..c27c6ba 100644 --- a/sqlbucket/cli.py +++ b/sqlbucket/cli.py @@ -35,11 +35,13 @@ def create_project(sqlbucket, name): @click.option('--rendering', '-r', is_flag=True, help="Only render queries") @click.option('--all', '-all', is_flag=True, help="All dbs") @click.option('--edb', '-x', required=False, type=str, help="Excluded dbs") + @click.option('--silent', '-s', required=False, is_flag=True, default=False, + help="Do not notify execution status.") @click.pass_obj @click.argument('args', nargs=-1) def run_job(sqlbucket, name, db, fstep, tstep, to_date, from_date, from_days, to_days, group, isolation, verbose, rendering, - all, edb, args): + all, edb, silent, args): submitted_variables = cli_variables_parser(args) @@ -90,7 +92,8 @@ def run_job(sqlbucket, name, db, fstep, tstep, to_date, from_date, to_step=tstep, group=group, verbose=verbose, - isolation_level=isolation + isolation_level=isolation, + silent=silent ) @cli.command(context_settings=dict(ignore_unknown_options=True)) diff --git a/sqlbucket/integrity.py b/sqlbucket/integrity.py index 58bb330..8d160b6 100644 --- a/sqlbucket/integrity.py +++ b/sqlbucket/integrity.py @@ -1,4 +1,4 @@ -from sqlbucket.runners import create_connection +from sqlbucket.runners import create_connection, connection_query from sqlbucket.runners import logger from tabulate import tabulate from sqlbucket.utils import integrity_logo, success @@ -42,6 +42,7 @@ def run_integrity(configuration: dict, prefix: str = '', verbose: bool = False): errors += 1 logger.info(f'Query {query_name} encountered an error:') logger.error(e) + connection_query(configuration, connection) continue # logging summary diff --git a/sqlbucket/macros/common.jinja2 b/sqlbucket/macros/common.jinja2 index 3bebc37..ef1fdd3 100644 --- a/sqlbucket/macros/common.jinja2 +++ b/sqlbucket/macros/common.jinja2 @@ -66,7 +66,7 @@ from {{ table }} {% macro are_within_threshold(scalar_a, scalar_b, threshold, integrity_check_name=None) %} select - {% if not integrity_check_name %}'source_to_target_integrity_check{% else %}{{ integrity_check_name }}{% endif %} as integrity_check + {% if not integrity_check_name %}'source_to_target_integrity_check'{% else %}{{ integrity_check_name }}{% endif %} as integrity_check, ({{ scalar_a }}) as expected, ({{ scalar_b }}) as calculated, abs((({{ scalar_a }}) - ({{ scalar_b }})) / nullif(({{ scalar_b }}), 0)) <= {{ threshold }} diff --git a/sqlbucket/project.py b/sqlbucket/project.py index b2f8e21..1ab27f2 100644 --- a/sqlbucket/project.py +++ b/sqlbucket/project.py @@ -89,6 +89,35 @@ def configure_integrity(self) -> dict: "connection_query": self.get_connection_query() } + def send_msg(run_func): + + def wrapper(self, *args, **kwargs): + + silent = kwargs.pop("silent", False) + + if silent: + run_func(self, *args, **kwargs) + else: + group = kwargs.get("group") + config = self.configure(group) + + if 'context' in config and 'f' in config['context']: + funcs_reg = config['context']['f'] + start_msg = funcs_reg.get('start_msg') + end_msg = funcs_reg.get('end_msg') + + try: + if start_msg: start_msg(**config) + run_func(self, *args, **kwargs) + if end_msg: end_msg(**config) + except: + exception_msg = funcs_reg.get('exception_msg') + if exception_msg: exception_msg(**config) + raise + + return wrapper + + @send_msg def run(self, group: str = None, from_step: int = 1, to_step: int = None, verbose: bool = False, isolation_level: str = None) -> None: configuration = self.configure(group) diff --git a/sqlbucket/runners.py b/sqlbucket/runners.py index 4337d7b..94ae173 100644 --- a/sqlbucket/runners.py +++ b/sqlbucket/runners.py @@ -109,9 +109,13 @@ def create_connection( isolation_level=isolation_level ) connection = engine.connect() + connection_query(configuration, connection) + + return connection + + +def connection_query(configuration: dict, connection: Connection): if configuration.get('connection_query') is not None: logger.info(f'Running connection query: ' f'{configuration["connection_query"]}') connection.execute(text(configuration['connection_query'])) - - return connection