From 2af71a5d042a78f9f60b15f2638446a1292e5443 Mon Sep 17 00:00:00 2001 From: Qiao Qiao <68757394+qiaouchicago@users.noreply.github.com> Date: Mon, 12 Aug 2024 13:19:39 -0500 Subject: [PATCH] DEV-2882 update deps (#293) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * upgrade pytest * add pytest config in pyproject.toml * add github action for prcess * upgrade moto， httmock, requests-mock, flask, jsonschema, lxml, requests, pyyaml * remove dev-requiremetns.txt * add markupsafe==2.0.1 * remove deps, ndg-httpsclient~=0.5.0, pyasn1 pyopenssl, cryptography, python-jose * update pytest config --- dev-requirements.txt | 219 ------------------------------------ pyproject.toml | 21 ++++ requirements.txt | 52 +++++---- setup.cfg | 27 ++--- src/gdc_client/cli.py | 12 -- tests/conftest.py | 5 +- tests/mock_server.py | 10 +- tests/test_upload_client.py | 2 + tox.ini | 4 +- 9 files changed, 68 insertions(+), 284 deletions(-) delete mode 100644 dev-requirements.txt diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 1dd76aac..00000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,219 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# pip-compile --extra=dev --output-file=dev-requirements.txt -# -atomicwrites==1.4.1 - # via pytest -attrs==22.1.0 - # via pytest -aws-sam-translator==1.11.0 - # via - # cfn-lint - # gdc-client (pyproject.toml) -aws-xray-sdk==2.11.0 - # via moto -boto==2.49.0 - # via moto -boto3==1.16.63 - # via - # aws-sam-translator - # moto -botocore==1.19.63 - # via - # aws-xray-sdk - # boto3 - # moto - # s3transfer -certifi==2024.7.4 - # via requests -cffi==1.15.1 - # via cryptography -cfn-lint==0.21.6 - # via - # gdc-client (pyproject.toml) - # moto -chardet==3.0.4 - # via requests -click==8.1.7 - # via - # flask - # gdc-client (pyproject.toml) -coverage[toml]==7.6.0 - # via pytest-cov -cryptography==43.0.0 - # via - # gdc-client (pyproject.toml) - # moto - # pyopenssl - # python-jose - # sshpubkeys -docker==4.4.4 - # via moto -ecdsa==0.14.1 - # via - # moto - # python-jose - # sshpubkeys -flask==1.0.4 - # via gdc-client (pyproject.toml) -httmock==1.3.0 - # via gdc-client (pyproject.toml) -idna==2.8 - # via - # moto - # requests -importlib-metadata==7.0.1 - # via - # gdc-client (pyproject.toml) - # pytest -intervaltree==3.0.2 - # via gdc-client (pyproject.toml) -itsdangerous==1.1.0 - # via flask -jinja2==2.11.3 - # via - # flask - # moto -jmespath==0.10.0 - # via - # boto3 - # botocore -jsondiff==2.0.0 - # via moto -jsonpatch==1.32 - # via cfn-lint -jsonpointer==2.3 - # via jsonpatch -jsonschema==2.6.0 - # via - # aws-sam-translator - # cfn-lint - # gdc-client (pyproject.toml) -lxml==4.4.3 - # via gdc-client (pyproject.toml) -markupsafe==1.1.1 - # via - # jinja2 - # moto -mock==3.0.5 - # via moto -more-itertools==8.14.0 - # via - # moto - # pytest -moto==1.3.16 - # via gdc-client (pyproject.toml) -ndg-httpsclient==0.5.1 - # via gdc-client (pyproject.toml) -packaging==20.9 - # via pytest -pluggy==0.13.1 - # via pytest -progressbar2==4.4.2 - # via gdc-client (pyproject.toml) -py==1.11.0 - # via pytest -pyasn1==0.4.8 - # via - # gdc-client (pyproject.toml) - # ndg-httpsclient - # python-jose - # rsa -pycparser==2.21 - # via cffi -pyopenssl==24.2.1 - # via - # gdc-client (pyproject.toml) - # ndg-httpsclient -pyparsing==2.4.7 - # via packaging -pytest==4.6.2 - # via - # gdc-client (pyproject.toml) - # pytest-cov -pytest-cov==5.0.0 - # via gdc-client (pyproject.toml) -python-dateutil==2.8.2 - # via - # botocore - # moto -python-jose[cryptography]==3.2.0 - # via - # gdc-client (pyproject.toml) - # moto -python-utils==3.8.2 - # via progressbar2 -pytz==2022.7.1 - # via moto -pyyaml==5.3.1 - # via - # cfn-lint - # gdc-client (pyproject.toml) - # moto -requests==2.22.0 - # via - # cfn-lint - # docker - # gdc-client (pyproject.toml) - # httmock - # moto - # requests-mock - # responses -requests-mock==1.5.2 - # via gdc-client (pyproject.toml) -responses==0.17.0 - # via moto -rsa==4.7.2 - # via python-jose -s3transfer==0.3.7 - # via boto3 -six==1.16.0 - # via - # aws-sam-translator - # cfn-lint - # docker - # ecdsa - # mock - # moto - # pytest - # python-dateutil - # python-jose - # requests-mock - # responses - # websocket-client -sortedcontainers==2.4.0 - # via intervaltree -sshpubkeys==3.3.1 - # via moto -termcolor==1.1.0 - # via gdc-client (pyproject.toml) -tomli==2.0.1 - # via coverage -typing-extensions==4.12.2 - # via python-utils -urllib3==1.25.11 - # via - # botocore - # requests - # responses -wcwidth==0.2.6 - # via pytest -websocket-client==0.59.0 - # via docker -werkzeug==1.0.1 - # via - # flask - # moto -wrapt==1.14.1 - # via aws-xray-sdk -xmltodict==0.13.0 - # via moto -zipp==3.17.0 - # via - # importlib-metadata - # moto - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/pyproject.toml b/pyproject.toml index 107ba5f5..e9d2d91e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,3 +40,24 @@ skip = [".gitignore", ".dockerignore"] [tool.setuptools_scm] local_scheme = "dirty-tag" + + +[tool.pytest.ini_options] +minversion = 6.0 +python_files = "test_*.py" +python_functions = "test_*" +junit_family = "xunit2" +junit_logging = "system-out" +console_output_style = "classic" +log_level = "ERROR" +log_cli = "true" +addopts = "-rfE --color=yes -p no:ddtrace -p no:ddtrace.pytest_bdd --cov=gdc_client --cov-report term" +norecursedirs = [ + ".git", + ".tox", + "dist", + "build" +] +testpaths = [ + "tests", +] diff --git a/requirements.txt b/requirements.txt index cd0220d2..1c0fbb81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,46 +4,48 @@ # # pip-compile # +attrs==23.2.0 + # via + # jsonschema + # referencing certifi==2024.7.4 # via requests -cffi==1.15.1 - # via cryptography -chardet==3.0.4 +charset-normalizer==3.3.2 # via requests -cryptography==43.0.0 - # via - # gdc-client (pyproject.toml) - # pyopenssl idna==2.8 # via requests importlib-metadata==7.0.1 # via gdc-client (pyproject.toml) +importlib-resources==6.4.0 + # via + # jsonschema + # jsonschema-specifications intervaltree==3.0.2 # via gdc-client (pyproject.toml) -jsonschema==2.6.0 +jsonschema==4.23.0 # via gdc-client (pyproject.toml) -lxml==4.4.2 - # via gdc-client (pyproject.toml) -ndg-httpsclient==0.5.0 +jsonschema-specifications==2023.12.1 + # via jsonschema +lxml==5.2.2 # via gdc-client (pyproject.toml) +pkgutil-resolve-name==1.3.10 + # via jsonschema progressbar2==3.55.0 # via gdc-client (pyproject.toml) -pyasn1==0.4.8 - # via - # gdc-client (pyproject.toml) - # ndg-httpsclient -pycparser==2.21 - # via cffi -pyopenssl==24.2.1 - # via - # gdc-client (pyproject.toml) - # ndg-httpsclient python-utils==2.7.1 # via progressbar2 -pyyaml==5.3.1 +pyyaml==6.0.1 # via gdc-client (pyproject.toml) -requests==2.22.0 +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.3 # via gdc-client (pyproject.toml) +rpds-py==0.19.1 + # via + # jsonschema + # referencing six==1.16.0 # via # progressbar2 @@ -55,4 +57,6 @@ termcolor==1.1.0 urllib3==1.25.11 # via requests zipp==3.17.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources diff --git a/setup.cfg b/setup.cfg index 92bbd495..12152842 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ classifiers = Programming Language :: Python Programming Language :: Python :: 3 Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.9 License :: OSI Approved :: Apache Software License Topic :: Internet :: WWW/HTTP @@ -25,17 +25,13 @@ package_dir = =src python_requires = >=3.8 install_requires = - cryptography - jsonschema~=2.6.0 - lxml~=4.4.2 - ndg-httpsclient~=0.5.0 - pyasn1 - pyOpenSSL - PyYAML>=5.1 + jsonschema + lxml + PyYAML>=6.0.1 intervaltree~=3.0.2 importlib_metadata termcolor~=1.1.0 - requests~=2.22.0 + requests progressbar2 [options.packages.find] @@ -46,15 +42,12 @@ where = src [options.extras_require] dev = click>= 8 - flask~=1.0.2 - pytest~=4.6.2 + flask + pytest pytest-cov - requests-mock~=1.5.2 - moto~=1.3.14 - python-jose<3.3.0 # lower the requirements for cryptography, so python3.5 could work - httmock~=1.3.0 - cfn-lint<0.22.0 # required by moto, limit version for jsonschema - aws-sam-translator<1.12.0a + requests-mock + moto + httmock [options.entry_points] console_scripts = diff --git a/src/gdc_client/cli.py b/src/gdc_client/cli.py index c907c643..aa68043b 100755 --- a/src/gdc_client/cli.py +++ b/src/gdc_client/cli.py @@ -12,18 +12,6 @@ from gdc_client.common.config import GDCClientConfigShared, GDCClientArgumentParser -#################################################### -# These packages are important but PyInstaller won't -# bundle them unless they are imported -import cffi -import cryptography -import OpenSSL -import ndg.httpsclient -import pyasn1 - -#################################################### - - DESCRIPTION = """ The Genomic Data Commons Command Line Client """ diff --git a/tests/conftest.py b/tests/conftest.py index 3d0bf553..482bc341 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,4 @@ import hashlib -import platform from io import BytesIO from multiprocessing import Process import tarfile @@ -7,7 +6,7 @@ from typing import Iterable, List, Mapping, Union import boto3 -from moto import mock_s3 +from moto import mock_aws import pytest from gdc_client.parcel.const import HTTP_CHUNK_SIZE @@ -160,7 +159,7 @@ def mock_response(url: str) -> None: @pytest.fixture def mock_s3_conn(): - with mock_s3(): + with mock_aws(): conn = boto3.resource("s3") yield conn diff --git a/tests/mock_server.py b/tests/mock_server.py index 1d67d1da..4ac05a34 100644 --- a/tests/mock_server.py +++ b/tests/mock_server.py @@ -147,18 +147,14 @@ def download(ids=""): data = "" filename = "test_file.txt" + headers = request.headers ids = ids.split(",") - args = request.json - headers = request.headers - - if args: + if request.content_type == "application/json": + args = request.json ids = args.get("ids") - if isinstance(ids, str): - ids = [ids] - for i in ids: if i not in uuids.keys(): return ( diff --git a/tests/test_upload_client.py b/tests/test_upload_client.py index 1c734ad7..51d6e85e 100644 --- a/tests/test_upload_client.py +++ b/tests/test_upload_client.py @@ -1,4 +1,5 @@ import json +import logging import os import re from collections import namedtuple @@ -525,6 +526,7 @@ def test_multipart_upload__complete_call_failed( def test_multipart_upload__upload_multipart_call_failed( s3_client, mock_multipart_upload_client, caplog ): + caplog.set_level(logging.WARNING) mock_multipart_upload_client.upload() assert_multipart_unsuccessful_scenario(s3_client, mock_multipart_upload_client) diff --git a/tox.ini b/tox.ini index d161a801..64a08869 100644 --- a/tox.ini +++ b/tox.ini @@ -6,8 +6,8 @@ allowlist_externals = pytest setenv= PYTHONHASHSEED=0 deps= - -rdev-requirements.txt - . + -crequirements.txt + .[dev] commands= pip freeze pytest -lvv tests/ --cov=gdc_client --junitxml=report.xml --cov-report html:docs/htmlcov --cov-report xml:coverage.xml {posargs}