From 31c8883d07db5036887a96f4a7a31fa8b348c26b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Sun, 22 Aug 2021 16:51:51 +0200 Subject: [PATCH 01/10] Update CI badge --- .github/workflows/cartoframes-ci.yml | 2 +- README.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cartoframes-ci.yml b/.github/workflows/cartoframes-ci.yml index 03527e729..eb8b8ef71 100644 --- a/.github/workflows/cartoframes-ci.yml +++ b/.github/workflows/cartoframes-ci.yml @@ -1,4 +1,4 @@ -name: Run CARTOFrames tests +name: CI on: push: diff --git a/README.rst b/README.rst index 24af6b881..e09de0432 100644 --- a/README.rst +++ b/README.rst @@ -2,8 +2,8 @@ CARTOframes *********** -.. image:: https://travis-ci.org/CartoDB/cartoframes.svg?branch=develop - :target: https://travis-ci.org/CartoDB/cartoframes +.. image:: https://github.com/CartoDB/cartoframes/actions/workflows/cartoframes-ci.yml/badge.svg?branch=develop + :target: https://github.com/CartoDB/cartoframes/actions/workflows/cartoframes-ci.yml .. image:: https://img.shields.io/badge/pypi-v1.2.3-orange :target: https://pypi.org/project/cartoframes/1.2.3 From f99b402874ee9f703f69f55fa59e27aafbeed0c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Aug 2021 20:53:38 +0000 Subject: [PATCH 02/10] Bump notebook from 6.1.5 to 6.4.1 in /tests/notebooks Bumps [notebook](http://jupyter.org) from 6.1.5 to 6.4.1. --- updated-dependencies: - dependency-name: notebook dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/notebooks/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/notebooks/requirements.txt b/tests/notebooks/requirements.txt index 67d51022b..3e397b7b2 100644 --- a/tests/notebooks/requirements.txt +++ b/tests/notebooks/requirements.txt @@ -1,4 +1,4 @@ -notebook==6.1.5 +notebook==6.4.1 pytest==6.1.1 pytest-mock==3.3.1 nbconvert==6.0.7 From f2ccf872a516e2980e93c814dbf05dc6a1563c42 Mon Sep 17 00:00:00 2001 From: Mmoncadaisla Date: Fri, 27 Aug 2021 12:51:20 +0200 Subject: [PATCH 03/10] Add cartodbfy step for create_table_from_query and copy_table functions --- cartoframes/io/carto.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/cartoframes/io/carto.py b/cartoframes/io/carto.py index 25fe0d5c2..1f1801d30 100644 --- a/cartoframes/io/carto.py +++ b/cartoframes/io/carto.py @@ -6,7 +6,7 @@ from carto.exceptions import CartoException -from .managers.context_manager import ContextManager, _compute_copy_data, get_dataframe_columns_info +from .managers.context_manager import ContextManager, _compute_copy_data, get_dataframe_columns_info, _cartodbfy_query from ..utils.geom_utils import is_reprojection_needed, reproject, has_geometry, set_geometry from ..utils.logger import log from ..utils.utils import is_valid_str, is_sql_query @@ -275,7 +275,7 @@ def rename_table(table_name, new_table_name, credentials=None, if_exists='fail', log.info('Success! Table "{0}" renamed to table "{1}" correctly'.format(table_name, new_table_name)) -def copy_table(table_name, new_table_name, credentials=None, if_exists='fail', log_enabled=True): +def copy_table(table_name, new_table_name, credentials=None, if_exists='fail', log_enabled=True, cartodbfy=True): """Copy a table into a new table in the CARTO account. Args: @@ -285,7 +285,8 @@ def copy_table(table_name, new_table_name, credentials=None, if_exists='fail', l instance of Credentials (username, api_key, etc). if_exists (str, optional): 'fail', 'replace', 'append'. Default is 'fail'. log_enabled (bool, optional): enable the logging mechanism. Default is True. - + cartodbfy (bool, optional): convert the table to CARTO format. Default True. More info + `here `. Raises: ValueError: if the table names provided are wrong or the if_exists param is not valid. @@ -305,11 +306,22 @@ def copy_table(table_name, new_table_name, credentials=None, if_exists='fail', l context_manager = ContextManager(credentials) new_table_name = context_manager.create_table_from_query(query, new_table_name, if_exists) + if cartodbfy is True: + schema = context_manager.get_schema() + cartodbfy_query = _cartodbfy_query(new_table_name, schema) + context_manager.execute_long_running_query(cartodbfy_query) + if log_enabled: log.info('Success! Table "{0}" copied to table "{1}" correctly'.format(table_name, new_table_name)) -def create_table_from_query(query, new_table_name, credentials=None, if_exists='fail', log_enabled=True): +def create_table_from_query( + query, + new_table_name, + credentials=None, + if_exists='fail', + log_enabled=True, + cartodbfy=True): """Create a new table from an SQL query in the CARTO account. Args: @@ -319,7 +331,8 @@ def create_table_from_query(query, new_table_name, credentials=None, if_exists=' instance of Credentials (username, api_key, etc). if_exists (str, optional): 'fail', 'replace', 'append'. Default is 'fail'. log_enabled (bool, optional): enable the logging mechanism. Default is True. - + cartodbfy (bool, optional): convert the table to CARTO format. Default True. More info + `here `. Raises: ValueError: if the query or table name provided is wrong or the if_exists param is not valid. @@ -337,6 +350,11 @@ def create_table_from_query(query, new_table_name, credentials=None, if_exists=' context_manager = ContextManager(credentials) new_table_name = context_manager.create_table_from_query(query, new_table_name, if_exists) + if cartodbfy is True: + schema = context_manager.get_schema() + cartodbfy_query = _cartodbfy_query(new_table_name, schema) + context_manager.execute_long_running_query(cartodbfy_query) + if log_enabled: log.info('Success! Table "{0}" created correctly'.format(new_table_name)) From 0cd99761aeabe32f9baa20c2e30f81bd76b98333 Mon Sep 17 00:00:00 2001 From: Mmoncadaisla Date: Fri, 27 Aug 2021 14:06:37 +0200 Subject: [PATCH 04/10] CR: Move cartodbfy step to ContextManager create_table_from_query method --- cartoframes/io/carto.py | 14 ++------------ cartoframes/io/managers/context_manager.py | 6 +++++- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cartoframes/io/carto.py b/cartoframes/io/carto.py index 1f1801d30..5e66f7b0d 100644 --- a/cartoframes/io/carto.py +++ b/cartoframes/io/carto.py @@ -304,12 +304,7 @@ def copy_table(table_name, new_table_name, credentials=None, if_exists='fail', l query = 'SELECT * FROM {}'.format(table_name) context_manager = ContextManager(credentials) - new_table_name = context_manager.create_table_from_query(query, new_table_name, if_exists) - - if cartodbfy is True: - schema = context_manager.get_schema() - cartodbfy_query = _cartodbfy_query(new_table_name, schema) - context_manager.execute_long_running_query(cartodbfy_query) + new_table_name = context_manager.create_table_from_query(query, new_table_name, if_exists, cartodbfy) if log_enabled: log.info('Success! Table "{0}" copied to table "{1}" correctly'.format(table_name, new_table_name)) @@ -348,12 +343,7 @@ def create_table_from_query( ', '.join(IF_EXISTS_OPTIONS))) context_manager = ContextManager(credentials) - new_table_name = context_manager.create_table_from_query(query, new_table_name, if_exists) - - if cartodbfy is True: - schema = context_manager.get_schema() - cartodbfy_query = _cartodbfy_query(new_table_name, schema) - context_manager.execute_long_running_query(cartodbfy_query) + new_table_name = context_manager.create_table_from_query(query, new_table_name, if_exists, cartodbfy) if log_enabled: log.info('Success! Table "{0}" created correctly'.format(new_table_name)) diff --git a/cartoframes/io/managers/context_manager.py b/cartoframes/io/managers/context_manager.py index b975beb96..360998bd7 100644 --- a/cartoframes/io/managers/context_manager.py +++ b/cartoframes/io/managers/context_manager.py @@ -122,7 +122,7 @@ def copy_from(self, gdf, table_name, if_exists='fail', cartodbfy=True, return table_name - def create_table_from_query(self, query, table_name, if_exists): + def create_table_from_query(self, query, table_name, if_exists, cartodbfy=True): schema = self.get_schema() table_name = self.normalize_table_name(table_name) @@ -140,6 +140,10 @@ def create_table_from_query(self, query, table_name, if_exists): else: self._drop_create_table_from_query(table_name, schema, query) + if cartodbfy is True: + cartodbfy_query = _cartodbfy_query(table_name, schema) + self.execute_long_running_query(cartodbfy_query) + return table_name def list_tables(self, schema=None): From 50c796a32d0dbb3f5263169f8a73c035f99b6e8e Mon Sep 17 00:00:00 2001 From: Mmoncadaisla Date: Fri, 27 Aug 2021 14:12:20 +0200 Subject: [PATCH 05/10] Remove unused function --- cartoframes/io/carto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cartoframes/io/carto.py b/cartoframes/io/carto.py index 5e66f7b0d..30272b3de 100644 --- a/cartoframes/io/carto.py +++ b/cartoframes/io/carto.py @@ -6,7 +6,7 @@ from carto.exceptions import CartoException -from .managers.context_manager import ContextManager, _compute_copy_data, get_dataframe_columns_info, _cartodbfy_query +from .managers.context_manager import ContextManager, _compute_copy_data, get_dataframe_columns_info from ..utils.geom_utils import is_reprojection_needed, reproject, has_geometry, set_geometry from ..utils.logger import log from ..utils.utils import is_valid_str, is_sql_query From 88e49934ac1734ef3174cb11f07ecd4396008423 Mon Sep 17 00:00:00 2001 From: Mmoncadaisla Date: Fri, 27 Aug 2021 14:38:03 +0200 Subject: [PATCH 06/10] Add tests to check cartodbfication step on copy_table and create_table_from_query functions --- tests/unit/io/test_carto.py | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/unit/io/test_carto.py b/tests/unit/io/test_carto.py index 4acb40458..0857c0fce 100644 --- a/tests/unit/io/test_carto.py +++ b/tests/unit/io/test_carto.py @@ -500,6 +500,27 @@ def test_copy_table_wrong_if_exists(mocker): assert str(e.value) == 'Wrong option for the `if_exists` param. You should provide: fail, replace, append.' +def test_copy_table_no_cartodbfy(mocker): + # Given + cm_mock = mocker.patch.object(ContextManager, 'create_table_from_query') + + # When + copy_table('__table_name__', '__new_table_name__', CREDENTIALS, cartodbfy=False) + + # Then + assert cm_mock.call_args[0][3] is False + + +def test_copy_table_cartodbfy(mocker): + # Given + cm_mock = mocker.patch.object(ContextManager, 'create_table_from_query') + + # When + copy_table('__table_name__', '__new_table_name__', CREDENTIALS, cartodbfy=True) + + # Then + assert cm_mock.call_args[0][3] is True + def test_create_table_from_query_wrong_query(mocker): # When with pytest.raises(ValueError) as e: @@ -535,3 +556,25 @@ def test_create_table_from_query_wrong_if_exists(mocker): # Then assert str(e.value) == 'Wrong option for the `if_exists` param. You should provide: fail, replace, append.' + + +def test_create_table_from_query_no_cartodbfy(mocker): + # Given + cm_mock = mocker.patch.object(ContextManager, 'create_table_from_query') + + # When + create_table_from_query('SELECT * FROM table', '__new_table_name__', CREDENTIALS, cartodbfy=False) + + # Then + assert cm_mock.call_args[0][3] is False + + +def test_create_table_from_query_cartodbfy(mocker): + # Given + cm_mock = mocker.patch.object(ContextManager, 'create_table_from_query') + + # When + create_table_from_query('SELECT * FROM table', '__new_table_name__', CREDENTIALS, cartodbfy=True) + + # Then + assert cm_mock.call_args[0][3] is True From ddcbe1cff38abd9382bfb30ed0a5961ad00427ee Mon Sep 17 00:00:00 2001 From: Mmoncadaisla Date: Fri, 27 Aug 2021 14:40:48 +0200 Subject: [PATCH 07/10] Fix linter --- tests/unit/io/test_carto.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/io/test_carto.py b/tests/unit/io/test_carto.py index 0857c0fce..d65ff9774 100644 --- a/tests/unit/io/test_carto.py +++ b/tests/unit/io/test_carto.py @@ -521,6 +521,7 @@ def test_copy_table_cartodbfy(mocker): # Then assert cm_mock.call_args[0][3] is True + def test_create_table_from_query_wrong_query(mocker): # When with pytest.raises(ValueError) as e: From a3d2cc1cf49d5b438096b3390be6f63e213dcdcc Mon Sep 17 00:00:00 2001 From: Mmoncadaisla Date: Fri, 27 Aug 2021 17:30:19 +0200 Subject: [PATCH 08/10] Add test to verify that create_table_from query performs cartodbfication --- tests/unit/io/managers/test_context_manager.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/unit/io/managers/test_context_manager.py b/tests/unit/io/managers/test_context_manager.py index 54cf399bf..e2c684b86 100644 --- a/tests/unit/io/managers/test_context_manager.py +++ b/tests/unit/io/managers/test_context_manager.py @@ -278,3 +278,16 @@ def __init__(self): with pytest.raises(CartoRateLimitException): test_function(retry_times=0) + + def test_create_table_from_query_cartodbfy(self, mocker): + # Given + mocker.patch.object(ContextManager, 'has_table', return_value=False) + mocker.patch.object(ContextManager, 'get_schema', return_value='schema') + mock = mocker.patch.object(ContextManager, 'execute_long_running_query') + + # When + cm = ContextManager(self.credentials) + cm.create_table_from_query('SELECT * FROM table_name', '__new_table_name__', if_exists='fail', cartodbfy=True) + + # Then + mock.assert_called_with("SELECT CDB_CartodbfyTable('schema', '__new_table_name__')") From 040074552b8a55875833f7c313134ea10498391f Mon Sep 17 00:00:00 2001 From: Mmoncadaisla Date: Mon, 30 Aug 2021 09:49:27 +0200 Subject: [PATCH 09/10] Add tests for default cartodbfy parameter --- .../unit/io/managers/test_context_manager.py | 13 +++++++++++ tests/unit/io/test_carto.py | 22 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/tests/unit/io/managers/test_context_manager.py b/tests/unit/io/managers/test_context_manager.py index e2c684b86..f40cd5cf8 100644 --- a/tests/unit/io/managers/test_context_manager.py +++ b/tests/unit/io/managers/test_context_manager.py @@ -291,3 +291,16 @@ def test_create_table_from_query_cartodbfy(self, mocker): # Then mock.assert_called_with("SELECT CDB_CartodbfyTable('schema', '__new_table_name__')") + + def test_create_table_from_query_cartodbfy_default(self, mocker): + # Given + mocker.patch.object(ContextManager, 'has_table', return_value=False) + mocker.patch.object(ContextManager, 'get_schema', return_value='schema') + mock = mocker.patch.object(ContextManager, 'execute_long_running_query') + + # When + cm = ContextManager(self.credentials) + cm.create_table_from_query('SELECT * FROM table_name', '__new_table_name__', if_exists='fail') + + # Then + mock.assert_called_with("SELECT CDB_CartodbfyTable('schema', '__new_table_name__')") diff --git a/tests/unit/io/test_carto.py b/tests/unit/io/test_carto.py index d65ff9774..b3638f21f 100644 --- a/tests/unit/io/test_carto.py +++ b/tests/unit/io/test_carto.py @@ -522,6 +522,17 @@ def test_copy_table_cartodbfy(mocker): assert cm_mock.call_args[0][3] is True +def test_copy_table_cartodbfy_default(mocker): + # Given + cm_mock = mocker.patch.object(ContextManager, 'create_table_from_query') + + # When + copy_table('__table_name__', '__new_table_name__', CREDENTIALS) + + # Then + assert cm_mock.call_args[0][3] is True + + def test_create_table_from_query_wrong_query(mocker): # When with pytest.raises(ValueError) as e: @@ -579,3 +590,14 @@ def test_create_table_from_query_cartodbfy(mocker): # Then assert cm_mock.call_args[0][3] is True + + +def test_create_table_from_query_cartodbfy_default(mocker): + # Given + cm_mock = mocker.patch.object(ContextManager, 'create_table_from_query') + + # When + create_table_from_query('SELECT * FROM table', '__new_table_name__', CREDENTIALS) + + # Then + assert cm_mock.call_args[0][3] is True From 2bb68ee28a881430d055d0c4dac65f0b250e4421 Mon Sep 17 00:00:00 2001 From: Mmoncadaisla Date: Thu, 2 Sep 2021 11:15:59 +0200 Subject: [PATCH 10/10] Release 1.2.4 --- CHANGELOG.md | 6 ++++++ README.rst | 8 ++++---- cartoframes/_version.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 821e243bb..8e2e9ba7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.2.4] - 2021-09-02 + +### Changed + +- Add cartodbfication step for create_table_from_query and copy_table functions (#1744) + ## [1.2.3] - 2021-08-18 ### Changed diff --git a/README.rst b/README.rst index e09de0432..4db4a6c1c 100644 --- a/README.rst +++ b/README.rst @@ -4,8 +4,8 @@ CARTOframes .. image:: https://github.com/CartoDB/cartoframes/actions/workflows/cartoframes-ci.yml/badge.svg?branch=develop :target: https://github.com/CartoDB/cartoframes/actions/workflows/cartoframes-ci.yml -.. image:: https://img.shields.io/badge/pypi-v1.2.3-orange - :target: https://pypi.org/project/cartoframes/1.2.3 +.. image:: https://img.shields.io/badge/pypi-v1.2.4-orange + :target: https://pypi.org/project/cartoframes/1.2.4 A Python package for integrating `CARTO `__ maps, analysis, and data services into data science workflows. @@ -14,11 +14,11 @@ Python data analysis workflows often rely on the de facto standards `pandas