From 9d1a97731b5685fe2810a7b0cd3ca527e1f98a40 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Thu, 9 Mar 2023 15:19:07 +0100 Subject: [PATCH] feat: Use latest Resoto and bump version (#40) --- Makefile | 5 ++--- cloud2sql/__init__.py | 2 +- cloud2sql/arrow/writer.py | 1 - config-template.yaml | 6 +++--- requirements-parquet.txt | 4 ++-- requirements-snowflake.txt | 2 +- requirements-test.txt | 12 ++++++------ requirements.txt | 12 ++++++------ tests/arrow/model_test.py | 1 - 9 files changed, 21 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 583906d..6b318b8 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean clean-test clean-pyc clean-build clean-env docs help setup +.PHONY: clean clean-test clean-pyc clean-build clean-env docs help setup test test-all .DEFAULT_GOAL := help .SILENT: clean clean-build clean-pyc clean-test setup @@ -77,8 +77,7 @@ venv: python3 -m venv venv --prompt "cloud2sql" . ./venv/bin/activate && python3 -m pip install --upgrade pip . ./venv/bin/activate && pip install -r requirements-test.txt - . ./venv/bin/activate && pip install -r requirements.txt - . ./venv/bin/activate && pip install -e . + . ./venv/bin/activate && pip install -e ".[all]" . ./venv/bin/activate && mypy --install-types --non-interactive cloud2sql tests setup: clean clean-env venv diff --git a/cloud2sql/__init__.py b/cloud2sql/__init__.py index 5d37d50..862722c 100644 --- a/cloud2sql/__init__.py +++ b/cloud2sql/__init__.py @@ -11,4 +11,4 @@ __author__ = "Some Engineering Inc." __license__ = "Apache 2.0" __copyright__ = "Copyright © 2022 Some Engineering Inc." -__version__ = "0.7.2" +__version__ = "0.8.0" diff --git a/cloud2sql/arrow/writer.py b/cloud2sql/arrow/writer.py index 3ee578d..056bde1 100644 --- a/cloud2sql/arrow/writer.py +++ b/cloud2sql/arrow/writer.py @@ -168,7 +168,6 @@ def value_to_string(v: Any) -> str: def write_batch_to_file(batch: ArrowBatch) -> ArrowBatch: - to_normalize = collect_normalization_paths(batch.schema) for row in batch.rows: diff --git a/config-template.yaml b/config-template.yaml index b923fd9..23642e8 100644 --- a/config-template.yaml +++ b/config-template.yaml @@ -25,9 +25,9 @@ sources: # Account thread/process pool size account_pool_size: 8 # Region thread pool size - region_pool_size: 128 + region_pool_size: 32 # Number of threads available shared for all regions - shared_pool_size: 128 + shared_pool_size: 32 # Number of threads to collect a single region region_resources_pool_size: 2 # List of AWS services to collect (default: all) @@ -97,7 +97,7 @@ sources: # Onelogin client secret client_secret: null destinations: - # Define the destination to sync. + # Define the destination to sync. # PostgreSQL is listed here as example - see README.md for a list of possible targets. posgresql: host: 127.0.0.1 diff --git a/requirements-parquet.txt b/requirements-parquet.txt index bdf3674..7e17c7d 100644 --- a/requirements-parquet.txt +++ b/requirements-parquet.txt @@ -1,3 +1,3 @@ -pyarrow==10.0.1 +pyarrow==11.0.0 google-cloud-storage==2.7.0 -boto3==1.26.10 \ No newline at end of file +boto3>=1.26.61 diff --git a/requirements-snowflake.txt b/requirements-snowflake.txt index 8ffb958..5e9d872 100644 --- a/requirements-snowflake.txt +++ b/requirements-snowflake.txt @@ -1 +1 @@ -snowflake-sqlalchemy>=1.4.4 +snowflake-sqlalchemy==1.4.6 diff --git a/requirements-test.txt b/requirements-test.txt index 6ef63f2..32ac659 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,9 +1,9 @@ # test dependencies -pytest==7.2.0 -black==22.10.0 +pytest==7.2.2 +black==23.1.0 flake8>=6.0.0 -mypy>=0.991 -tox>=4.0.3 +mypy==1.1.1 +tox==4.4.6 wheel>=0.38.4 -coverage>=6.5.0 -resoto-plugin-example-collector>=3.0.0 +coverage==7.2.1 +resoto-plugin-example-collector==3.2.5 diff --git a/requirements.txt b/requirements.txt index 7a88fc1..1c24a5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,11 +6,11 @@ PyYAML>=6.0 rich>=12.6.0 resotoclient>=1.2.1 posthog>=2.2.0 -requests>=2.28.1 +requests>=2.28.2 -resotolib>=3.1.0 +resotolib>=3.2.5, <3.3 # all collector plugins -resoto-plugin-aws>=3.1.0 -resoto-plugin-digitalocean>=3.1.0 -resoto-plugin-gcp>=3.1.0 -resoto-plugin-k8s>=3.1.0 +resoto-plugin-aws>=3.2.5, <3.3 +resoto-plugin-digitalocean>=3.2.5, <3.3 +resoto-plugin-gcp>=3.2.5, <3.3 +resoto-plugin-k8s>=3.2.5, <3.3 diff --git a/tests/arrow/model_test.py b/tests/arrow/model_test.py index 7f7b370..905104c 100644 --- a/tests/arrow/model_test.py +++ b/tests/arrow/model_test.py @@ -34,7 +34,6 @@ def test_create_schema(model: Model) -> None: def test_update(parquet_writer: ArrowWriter) -> None: - parquet_writer.insert_node( # type: ignore { "type": "node",