From 3a7ca9fbaddac20b3ac0dbb5e90c7ea86096cda0 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Wed, 15 Nov 2023 11:11:20 +0100 Subject: [PATCH 1/2] [chore] Bump libs --- Makefile | 2 +- pyproject.toml | 7 +- requirements-all.txt | 168 +++++++++++++++++---------------- requirements.txt | 108 +++++++++++---------- resotodatalink/remote_graph.py | 2 +- tests/collect_plugins_test.py | 3 +- tox.ini | 2 +- 7 files changed, 154 insertions(+), 138 deletions(-) diff --git a/Makefile b/Makefile index 35e3bd8..420adf2 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ clean-env: ## remove environment lint: ## static code analysis black --line-length 120 --check resotodatalink tests flake8 resotodatalink - mypy --python-version 3.9 --strict --install-types --non-interactive resotodatalink tests + mypy --python-version 3.9 --strict resotodatalink tests test: ## run tests quickly with the default Python pytest diff --git a/pyproject.toml b/pyproject.toml index a8373fb..7182df6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,12 @@ extra = [ ] # All dev dependencies -dev = [ "pip-tools" ] +dev = [ + "pip-tools", + "types_python_dateutil", + "types_requests", + "types_setuptools", +] # All test dependencies test = [ diff --git a/requirements-all.txt b/requirements-all.txt index b0ef9f7..59af86f 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -1,129 +1,135 @@ -aiodns==3.0.0 -aiohttp[speedups]==3.8.4 +aiodns==3.1.1 +aiohttp[speedups]==3.8.6 aiosignal==1.3.1 +annotated-types==0.6.0 asn1crypto==1.5.1 -astroid==2.15.5 -async-timeout==4.0.2 +astroid==3.0.1 +async-timeout==4.0.3 attrs==23.1.0 autocommand==2.2.2 backoff==2.2.1 -black==23.3.0 -boto3==1.26.161 -botocore==1.29.161 -brotli==1.0.9 -build==0.10.0 -cachetools==5.3.1 +black==23.11.0 +boto3==1.29.0 +botocore==1.32.0 +brotli==1.1.0 +build==1.0.3 +cachetools==5.3.2 cattrs==23.1.2 -certifi==2023.5.7 -cffi==1.15.1 -chardet==5.1.0 -charset-normalizer==3.1.0 +certifi==2023.7.22 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==2.0.12 cheroot==10.0.0 cherrypy==18.8.0 -click==8.1.3 +click==8.1.7 colorama==0.4.6 -coverage[toml]==7.2.7 -cryptography==40.0.2 -dill==0.3.6 -distlib==0.3.6 -filelock==3.12.2 -flake8==6.0.0 -frozenlist==1.3.3 -google-api-core==2.11.1 -google-auth==2.21.0 -google-cloud-core==2.3.2 -google-cloud-storage==2.10.0 +coverage[toml]==7.3.2 +cryptography==36.0.2 +dill==0.3.7 +distlib==0.3.7 +exceptiongroup==1.1.3 +filelock==3.13.1 +flake8==6.1.0 +frozendict==2.3.8 +frozenlist==1.4.0 +google-api-core==2.14.0 +google-auth==2.23.4 +google-cloud-core==2.3.3 +google-cloud-storage==2.13.0 google-crc32c==1.5.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.1 -hypothesis==6.79.3 +google-resumable-media==2.6.0 +googleapis-common-protos==1.61.0 +hypothesis==6.88.4 idna==3.4 -inflect==6.0.4 +inflect==7.0.0 iniconfig==2.0.0 isort==5.12.0 jaraco-collections==4.3.0 jaraco-context==4.3.0 -jaraco-functools==3.8.0 +jaraco-functools==4.0.0 jaraco-text==3.11.1 jmespath==1.0.1 jsons==1.6.3 -lazy-object-proxy==1.9.0 markdown-it-py==3.0.0 mccabe==0.7.0 mdurl==0.1.2 monotonic==1.6 -more-itertools==9.1.0 +more-itertools==10.1.0 multidict==6.0.4 -mypy==1.4.1 +mypy==1.7.0 mypy-extensions==1.0.0 -networkx==3.1 -numpy==1.25.0 +networkx==3.2.1 +numpy==1.26.2 oscrypto==1.3.0 -packaging==23.1 +packaging==23.2 parsy==2.1 -pathspec==0.11.1 +pathspec==0.11.2 pep8-naming==0.13.3 pint==0.22 -pip-tools==6.13.0 -platformdirs==3.8.0 -pluggy==1.2.0 -portend==3.1.0 -posthog==3.0.1 -prometheus-client==0.17.0 -protobuf==4.23.3 -psutil==5.9.5 -psycopg2-binary==2.9.6 -pyarrow==12.0.1 +pip-tools==7.3.0 +platformdirs==3.11.0 +pluggy==1.3.0 +portend==3.2.0 +posthog==3.0.2 +prometheus-client==0.18.0 +protobuf==4.25.0 +psutil==5.9.6 +psycopg2-binary==2.9.9 +pyarrow==14.0.1 pyasn1==0.5.0 pyasn1-modules==0.3.0 -pycares==4.3.0 -pycodestyle==2.10.0 +pycares==4.4.0 +pycodestyle==2.11.1 pycparser==2.21 -pycryptodomex==3.18.0 -pydantic==1.10.9 -pyflakes==3.0.1 -pygments==2.15.1 -pyjwt==2.7.0 -pylint==2.17.4 +pycryptodomex==3.19.0 +pydantic==2.5.0 +pydantic-core==2.14.1 +pyflakes==3.1.0 +pygments==2.16.1 +pyjwt==2.8.0 +pylint==3.0.2 pymysql==1.1.0 -pyopenssl==23.2.0 -pyproject-api==1.5.2 +pyopenssl==22.0.0 +pyproject-api==1.6.1 pyproject-hooks==1.0.0 -pytest==7.4.0 -pytest-asyncio==0.21.0 +pytest==7.4.3 +pytest-asyncio==0.21.1 pytest-cov==4.1.0 pytest-runner==6.0.0 python-dateutil==2.8.2 -pytz==2023.3 -pyyaml==6.0 +pytz==2023.3.post1 +pyyaml==6.0.1 requests==2.31.0 -resoto-plugin-example-collector==3.5.3 +resoto-plugin-example-collector==3.8.0 resotoclient==1.5.0 -resotolib==3.5.3 -rich==13.4.2 +resotolib==3.8.0 +rich==13.6.0 rsa==4.9 -s3transfer==0.6.1 +s3transfer==0.7.0 six==1.16.0 -snowflake-connector-python==3.0.4 -snowflake-sqlalchemy==1.4.7 +snowflake-connector-python==2.7.9 +snowflake-sqlalchemy==1.5.1 sortedcontainers==2.4.0 -sqlalchemy==1.4.48 +sqlalchemy==1.4.50 tempora==5.5.0 -tomlkit==0.11.8 -tox==4.6.3 -typeguard==4.0.0 -typing-extensions==4.6.3 +tomli==2.0.1 +tomlkit==0.12.3 +tox==4.11.3 +typeguard==4.1.5 +types-python-dateutil==2.8.19.14 +types-requests==2.31.0.10 +types-setuptools==68.2.0.1 +typing-extensions==4.8.0 typish==1.9.3 tzdata==2023.3 -tzlocal==5.0.1 -urllib3==1.26.16 -virtualenv==20.23.1 -websocket-client==1.6.1 -wheel==0.40.0 -wrapt==1.15.0 +tzlocal==5.2 +urllib3==2.0.7 +virtualenv==20.24.6 +websocket-client==1.6.4 +wheel==0.41.3 yarl==1.9.2 zc-lockfile==3.0.post1 # The following packages are considered to be unsafe in a requirements file: -pip==23.1.2 -setuptools==68.0.0 +pip==23.3.1 +setuptools==68.2.2 diff --git a/requirements.txt b/requirements.txt index 741ce70..ae7ca7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,88 +1,92 @@ -aiodns==3.0.0 -aiohttp[speedups]==3.8.4 +aiodns==3.1.1 +aiohttp[speedups]==3.8.6 aiosignal==1.3.1 +annotated-types==0.6.0 asn1crypto==1.5.1 -async-timeout==4.0.2 +async-timeout==4.0.3 attrs==23.1.0 autocommand==2.2.2 -boto3==1.26.161 -botocore==1.29.161 -brotli==1.0.9 -cachetools==5.3.1 +boto3==1.29.0 +botocore==1.32.0 +brotli==1.1.0 +cachetools==5.3.2 cattrs==23.1.2 -certifi==2023.5.7 -cffi==1.15.1 -charset-normalizer==3.1.0 +certifi==2023.7.22 +cffi==1.16.0 +charset-normalizer==3.3.2 cheroot==10.0.0 cherrypy==18.8.0 -cryptography==40.0.2 -filelock==3.12.2 -frozenlist==1.3.3 -google-api-core==2.11.1 -google-auth==2.21.0 -google-cloud-core==2.3.2 -google-cloud-storage==2.10.0 +cryptography==41.0.5 +exceptiongroup==1.1.3 +filelock==3.13.1 +frozendict==2.3.8 +frozenlist==1.4.0 +google-api-core==2.14.0 +google-auth==2.23.4 +google-cloud-core==2.3.3 +google-cloud-storage==2.13.0 google-crc32c==1.5.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.1 +google-resumable-media==2.6.0 +googleapis-common-protos==1.61.0 idna==3.4 -inflect==6.0.4 +inflect==7.0.0 jaraco-collections==4.3.0 jaraco-context==4.3.0 -jaraco-functools==3.8.0 +jaraco-functools==4.0.0 jaraco-text==3.11.1 jmespath==1.0.1 jsons==1.6.3 markdown-it-py==3.0.0 mdurl==0.1.2 -more-itertools==9.1.0 +more-itertools==10.1.0 multidict==6.0.4 -networkx==3.1 -numpy==1.25.0 -oscrypto==1.3.0 -packaging==23.1 +networkx==3.2.1 +numpy==1.26.2 +packaging==23.2 parsy==2.1 pint==0.22 -portend==3.1.0 -prometheus-client==0.17.0 -protobuf==4.23.3 -psutil==5.9.5 -psycopg2-binary==2.9.6 -pyarrow==12.0.1 +platformdirs==3.11.0 +portend==3.2.0 +prometheus-client==0.18.0 +protobuf==4.25.0 +psutil==5.9.6 +psycopg2-binary==2.9.9 +pyarrow==14.0.1 pyasn1==0.5.0 pyasn1-modules==0.3.0 -pycares==4.3.0 +pycares==4.4.0 pycparser==2.21 -pycryptodomex==3.18.0 -pydantic==1.10.9 -pygments==2.15.1 -pyjwt==2.7.0 +pydantic==2.5.0 +pydantic-core==2.14.1 +pygments==2.16.1 +pyjwt==2.8.0 pymysql==1.1.0 -pyopenssl==23.2.0 +pyopenssl==23.3.0 python-dateutil==2.8.2 -pytz==2023.3 -pyyaml==6.0 +pytz==2023.3.post1 +pyyaml==6.0.1 requests==2.31.0 resotoclient==1.5.0 -resotolib==3.5.3 -rich==13.4.2 +resotolib==3.8.0 +rich==13.6.0 rsa==4.9 -s3transfer==0.6.1 +s3transfer==0.7.0 six==1.16.0 -snowflake-connector-python==3.0.4 -snowflake-sqlalchemy==1.4.7 +snowflake-connector-python==3.5.0 +snowflake-sqlalchemy==1.5.1 sortedcontainers==2.4.0 -sqlalchemy==1.4.48 +sqlalchemy==1.4.50 tempora==5.5.0 -typeguard==4.0.0 -typing-extensions==4.6.3 +tomlkit==0.12.3 +typeguard==4.1.5 +typing-extensions==4.8.0 typish==1.9.3 tzdata==2023.3 -tzlocal==5.0.1 -urllib3==1.26.16 -websocket-client==1.6.1 +tzlocal==5.2 +urllib3==1.26.18 +websocket-client==1.6.4 yarl==1.9.2 zc-lockfile==3.0.post1 # The following packages are considered to be unsafe in a requirements file: -setuptools==68.0.0 +setuptools==68.2.2 diff --git a/resotodatalink/remote_graph.py b/resotodatalink/remote_graph.py index 9ebc1b1..e7ba1ae 100644 --- a/resotodatalink/remote_graph.py +++ b/resotodatalink/remote_graph.py @@ -72,7 +72,7 @@ def set_carz(jsc: Json, rs: BaseResource) -> None: for ancestor, clazz in carz.items(): path = ["ancestors", ancestor, "reported"] if (cv := value_in_path(jsc, path)) and (cid := cv.get("id")) and (cname := cv.get("name")): - resource = lookup.get(cid, clazz(id=cid, name=cname)) # type: ignore + resource = lookup.get(cid, clazz(id=cid, name=cname)) lookup[cid] = resource setattr(rs, f"_{ancestor}", resource) # xxx is defined by _xxx property diff --git a/tests/collect_plugins_test.py b/tests/collect_plugins_test.py index d3dbc45..9dc1991 100644 --- a/tests/collect_plugins_test.py +++ b/tests/collect_plugins_test.py @@ -38,8 +38,9 @@ async def test_collect_sql(core_feedback: CoreFeedback) -> None: "link_example_region_example_instance": 2, "link_example_region_example_network": 2, "link_example_region_example_volume": 2, + "resource_short_property_access": 0, } - assert set(metadata.tables.keys()) == expected_counts.keys() # check that there are entries in the tables + assert set(metadata.tables.keys()) == set(expected_counts.keys()) # check that there are entries in the tables with Session(engine) as session: for table in metadata.tables.values(): assert session.query(table).count() == expected_counts[table.name] diff --git a/tox.ini b/tox.ini index 0a22673..fddbede 100644 --- a/tox.ini +++ b/tox.ini @@ -23,4 +23,4 @@ commands = black --line-length 120 --check --diff --target-version py39 . [testenv:mypy] commands= - python -m mypy --install-types --non-interactive --python-version 3.9 resotodatalink tests + python -m mypy --python-version 3.9 resotodatalink tests From 77604816d0b114d1d2998b526e56b745baf7cbe6 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Wed, 15 Nov 2023 11:22:34 +0100 Subject: [PATCH 2/2] Define urllib dependency explicitly --- Makefile | 2 +- pyproject.toml | 1 + requirements-all.txt | 15 +++++++-------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 420adf2..e7cce30 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ update: requirements: pip-compile -q --no-annotate --resolver=backtracking --upgrade --allow-unsafe --no-header -o requirements.txt --extra=extra - pip-compile -q --all-extras --no-annotate --resolver=backtracking --upgrade --allow-unsafe --no-header -o requirements-all.txt + pip-compile -q --no-annotate --resolver=backtracking --upgrade --allow-unsafe --no-header -o requirements-all.txt --all-extras list-outdated: pip list --outdated diff --git a/pyproject.toml b/pyproject.toml index 7182df6..fc2e8be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ postgres = [ "psycopg2-binary" ] # All extra dependencies extra = [ "boto3", + "urllib3 < 2", # boto3 requires urllib3 < 2 "google-cloud-storage", "pymysql", "psycopg2-binary", diff --git a/requirements-all.txt b/requirements-all.txt index 59af86f..be430a7 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -18,13 +18,13 @@ cattrs==23.1.2 certifi==2023.7.22 cffi==1.16.0 chardet==5.2.0 -charset-normalizer==2.0.12 +charset-normalizer==3.3.2 cheroot==10.0.0 cherrypy==18.8.0 click==8.1.7 colorama==0.4.6 coverage[toml]==7.3.2 -cryptography==36.0.2 +cryptography==41.0.5 dill==0.3.7 distlib==0.3.7 exceptiongroup==1.1.3 @@ -60,7 +60,6 @@ mypy==1.7.0 mypy-extensions==1.0.0 networkx==3.2.1 numpy==1.26.2 -oscrypto==1.3.0 packaging==23.2 parsy==2.1 pathspec==0.11.2 @@ -81,7 +80,6 @@ pyasn1-modules==0.3.0 pycares==4.4.0 pycodestyle==2.11.1 pycparser==2.21 -pycryptodomex==3.19.0 pydantic==2.5.0 pydantic-core==2.14.1 pyflakes==3.1.0 @@ -89,7 +87,7 @@ pygments==2.16.1 pyjwt==2.8.0 pylint==3.0.2 pymysql==1.1.0 -pyopenssl==22.0.0 +pyopenssl==23.3.0 pyproject-api==1.6.1 pyproject-hooks==1.0.0 pytest==7.4.3 @@ -107,7 +105,7 @@ rich==13.6.0 rsa==4.9 s3transfer==0.7.0 six==1.16.0 -snowflake-connector-python==2.7.9 +snowflake-connector-python==3.5.0 snowflake-sqlalchemy==1.5.1 sortedcontainers==2.4.0 sqlalchemy==1.4.50 @@ -117,13 +115,14 @@ tomlkit==0.12.3 tox==4.11.3 typeguard==4.1.5 types-python-dateutil==2.8.19.14 -types-requests==2.31.0.10 +types-requests==2.31.0.6 types-setuptools==68.2.0.1 +types-urllib3==1.26.25.14 typing-extensions==4.8.0 typish==1.9.3 tzdata==2023.3 tzlocal==5.2 -urllib3==2.0.7 +urllib3==1.26.18 virtualenv==20.24.6 websocket-client==1.6.4 wheel==0.41.3