diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 0000000..c48ea42 --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,44 @@ +name: Publish to Prod PyPi + +on: + release: + types: [created] + +jobs: + publish-to-pypi: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: [ "3.10" ] + poetry-version: [ 1.3.1 ] + + steps: + - uses: actions/checkout@v3 + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + shell: bash + run: sudo apt install libcurl4-openssl-dev libssl-dev + + - name: Install poetry ${{ matrix.poetry-version }} + run: python -m pip install poetry==${{ matrix.poetry-version }} + + - name: Install dependencies + shell: bash + run: poetry install + + - name: Configure PyPi + run: | + python -m poetry config pypi-token.pypi ${{ secrets.PYPI_PROD }} + + - name: Build package + run: poetry build + + - name: Publish package + run: poetry publish + + diff --git a/.github/workflows/test-and-lint.yml b/.github/workflows/test-and-lint.yml new file mode 100644 index 0000000..e1e580c --- /dev/null +++ b/.github/workflows/test-and-lint.yml @@ -0,0 +1,40 @@ +name: Test, Lint + +on: [push] + +jobs: + test-and-lint: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + poetry-version: [ 1.3.1 ] + + steps: + - uses: actions/checkout@v3 + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + shell: bash + run: sudo apt install libcurl4-openssl-dev libssl-dev + + - name: Install poetry ${{ matrix.poetry-version }} + run: python -m pip install poetry==${{ matrix.poetry-version }} + + - name: Install dependencies + shell: bash + run: poetry install + + - name: Lint + shell: bash + run: poetry run black --check request_curl tests + + - name: Test + shell: bash + run: make test + + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1081cd9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Ennis Blank, Mauritz Uphoff + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f4f8962 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +lint: + poetry run black request_curl tests + +test: + poetry run pytest tests \ No newline at end of file diff --git a/README.md b/README.md index 873661e..bd7a205 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Request Curl -User-friendly wrapper for pycurl +A user-friendly wrapper for pycurl that simplifies HTTP requests. ## Installation Use the package manager @@ -11,48 +11,61 @@ to install request_curl. pip install request_curl ``` -## HTTP2 -HTTP2 is disabled by default. +# Quickstart +A request_curl session manages cookies, connection pooling, and configurations. +Basic Usage: ```python import request_curl -s = request_curl.Session(http2=True) -r = s.get("https://www.example.com") +s = request_curl.Session() +s.get('https://httpbin.org/get') # returns +s.request('GET', 'https://httpbin.org/get') # returns ``` -## Proxy Support -Proxy has to be formatted as a string. +Using a Context Manager +```python +import request_curl +with request_curl.Session() as session: + session.get('https://httpbin.org/get') # returns +``` + +# Features + +## Response Object + +The response object is similar to that of the [requests](https://pypi.org/project/requests/) library. ```python import request_curl s = request_curl.Session() -r = s.get("https://www.example.com", proxies="ip:port:user:password") +r = s.get("https://httpbin.org/get") + +print(r) # prints response object +print(r.status_code) # prints status code +print(r.content) # prints response content in bytes +print(r.text) # prints response content as text +print(r.json) # prints response content as JSON +print(r.url) # prints response URL +print(r.headers) # prints response URL ``` -## Content Decoding +## Proxy Support +Format the proxy as a string. + ```python import request_curl -s = request_curl.Session(accept_encoding="br, gzip, deflate") -r = s.get("https://www.example.com", debug=True) +s = request_curl.Session() +# supports authentication: r = s.get("https://httpbin.org/get", proxies="ip:port:user:password") +r = s.get("https://httpbin.org/get", proxies="ip:port") ``` -## Response Object - -The response object behaves -similar to the one of the requests library. +## HTTP2 +HTTP2 is disabled by default. ```python import request_curl -s = request_curl.Session() -r = s.get("https://www.example.com") - -print(r) -print(r.status_code) -print(r.content) -print(r.text) -print(r.json) -print(r.url) -print(r.history) +s = request_curl.Session(http2=True) +r = s.get("https://httpbin.org/get") ``` ## Cipher Suites @@ -68,22 +81,20 @@ cipher_suite = [ "AES256-GCM-SHA384" ] s = request_curl.Session(cipher_suite=cipher_suite) -r = s.get("https://www.example.com") +r = s.get("https://httpbin.org/get") ``` ## Debug Request -If debug is set to True the raw input -and output headers will bre printed out. +Set debug to True to print raw input and output headers. ```python import request_curl s = request_curl.Session() -r = s.get("https://www.example.com", debug=True) +r = s.get("https://httpbin.org/get", debug=True) ``` -## Custom Header -You can specify custom a customer header -as a dictionary. +## Custom Headers +Specify custom headers as a dictionary. ```python import request_curl @@ -91,19 +102,31 @@ s = request_curl.Session() headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36" } -r = s.get("https://www.example.com", headers=headers) +r = s.get("https://httpbin.org/get", headers=headers) ``` -## Install with Curl-Impersonate -- https://github.com/lwthiker/curl-impersonate/blob/main/INSTALL.md -- sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool -- ``sudo apt install -y libbrotli-dev golang build-essential libnghttp2-dev cmake libunwind-dev libssl-dev git python3-dev`` -- git clone https://github.com/pycurl/pycurl.git -- sudo python3 setup.py install --curl-config=/usr/local/bin/curl-impersonate-chrome-config +## Data ```python -import pycurl -pycurl.version_info() -# (9, '7.84.0', 480256, 'x86_64-pc-linux-gnu', 1370063517, 'BoringSSL', 0, '1.2.11', ('dict', 'file', 'ftp', 'ftps', 'gopher', 'gophers', 'http', 'https', 'imap', 'imaps', 'mqtt', 'pop3', 'pop3s', 'rtsp', 'smb', 'smbs', 'smtp', 'smtps', 'telnet', 'tftp'), None, 0, None) -quit() -``` \ No newline at end of file +import request_curl +s = request_curl.Session() + +# sending form data +form_data = {"key": "value"} +response = s.post("https://httpbin.org/post", data=form_data) + +# sending json data +json_data = {"key": "value"} +response = s.post("https://httpbin.org/post", json=json_data) +``` + +# Contributing + +We welcome contributions through pull requests. +Before making major changes, please open an issue to discuss your intended changes. +Also, ensure to update relevant tests. + +# License +Ennis Blank , Mauritz Uphoff + +[MIT](LICENSE) \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index e537e50..3650c97 100644 --- a/poetry.lock +++ b/poetry.lock @@ -19,6 +19,43 @@ docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib- tests = ["attrs[tests-no-zope]", "zope.interface"] tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] +[[package]] +name = "black" +version = "22.12.0" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, + {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, + {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, + {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, + {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, + {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, + {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, + {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, + {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, + {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, + {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, + {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""} +typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""} +typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + [[package]] name = "brotli" version = "1.0.9" @@ -221,6 +258,22 @@ files = [ {file = "charset_normalizer-3.0.1-py3-none-any.whl", hash = "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24"}, ] +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + [[package]] name = "colorama" version = "0.4.6" @@ -260,6 +313,27 @@ files = [ {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] +[[package]] +name = "importlib-metadata" +version = "6.0.0" +description = "Read metadata from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "importlib_metadata-6.0.0-py3-none-any.whl", hash = "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad"}, + {file = "importlib_metadata-6.0.0.tar.gz", hash = "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"}, +] + +[package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] + [[package]] name = "iniconfig" version = "2.0.0" @@ -272,6 +346,18 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] + [[package]] name = "packaging" version = "23.0" @@ -284,6 +370,37 @@ files = [ {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, ] +[[package]] +name = "pathspec" +version = "0.11.0" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.0-py3-none-any.whl", hash = "sha256:3a66eb970cbac598f9e5ccb5b2cf58930cd8e3ed86d393d541eaf2d8b1705229"}, + {file = "pathspec-0.11.0.tar.gz", hash = "sha256:64d338d4e0914e91c1792321e6907b5a593f1ab1851de7fc269557a21b30ebbc"}, +] + +[[package]] +name = "platformdirs" +version = "2.6.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"}, + {file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.4", markers = "python_version < \"3.8\""} + +[package.extras] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] + [[package]] name = "pluggy" version = "1.0.0" @@ -296,6 +413,9 @@ files = [ {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] +[package.dependencies] +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} + [package.extras] dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] @@ -327,6 +447,7 @@ files = [ attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" @@ -369,6 +490,52 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "typed-ast" +version = "1.5.4" +description = "a fork of Python 2 and 3 ast modules with type comment support" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"}, + {file = "typed_ast-1.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62"}, + {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac"}, + {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe"}, + {file = "typed_ast-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72"}, + {file = "typed_ast-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec"}, + {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47"}, + {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6"}, + {file = "typed_ast-1.5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1"}, + {file = "typed_ast-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6"}, + {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66"}, + {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c"}, + {file = "typed_ast-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2"}, + {file = "typed_ast-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d"}, + {file = "typed_ast-1.5.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f"}, + {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc"}, + {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6"}, + {file = "typed_ast-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e"}, + {file = "typed_ast-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35"}, + {file = "typed_ast-1.5.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97"}, + {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3"}, + {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72"}, + {file = "typed_ast-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1"}, + {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, +] + +[[package]] +name = "typing-extensions" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, +] + [[package]] name = "urllib3" version = "1.26.14" @@ -386,7 +553,23 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "zipp" +version = "3.12.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "zipp-3.12.0-py3-none-any.whl", hash = "sha256:9eb0a4c5feab9b08871db0d672745b53450d7f26992fd1e4653aa43345e97b86"}, + {file = "zipp-3.12.0.tar.gz", hash = "sha256:73efd63936398aac78fd92b6f4865190119d6c91b531532e798977ea8dd402eb"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + [metadata] lock-version = "2.0" -python-versions = "^3.10" -content-hash = "7e3424d07fda6f9e686afc3b5a9d933cdfa56ef8ffe92fae85b822cf0ee051fc" +python-versions = "^3.7" +content-hash = "e62bdd473d58918fe15f1b54d99c6802a95f550fd04e1d5133e53a69f4e183d1" diff --git a/pyproject.toml b/pyproject.toml index 2213229..a2e2d37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,24 @@ [tool.poetry] name = "request_curl" version = "0.0.1" -description = "" -authors = ["Notifysolutions "] +description = "A user-friendly wrapper for pycurl that simplifies HTTP requests" +authors = ["Mauritz Uphoff ", "Ennis Blank "] +readme = "README.md" +license = "MIT" +maintainers = ["Mauritz Uphoff", "Ennis Blank"] +homepage = "https://github.com/Notifysolutions/request_curl" +repository = "https://github.com/Notifysolutions/request_curl" +documentation = "https://github.com/Notifysolutions/request_curl/blob/main/README.md" [tool.poetry.dependencies] -python = "^3.10" +python = "^3.7" requests = "^2.28.1" Brotli = "^1.0.9" pycurl = "^7.45.2" [tool.poetry.dev-dependencies] pytest = "^7.1.3" +black = "^22.12.0" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/request_curl/__init__.py b/request_curl/__init__.py index 6b495ad..ed3cde9 100644 --- a/request_curl/__init__.py +++ b/request_curl/__init__.py @@ -1,5 +1,4 @@ -# GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD -from .api import get, options, post, put, patch, delete, head, request from .sessions import Session +from .defaults import CHROME_UA, CHROME_HEADERS, CHROME_CIPHER_SUITE version = "0.0.1" diff --git a/request_curl/api.py b/request_curl/api.py deleted file mode 100644 index aa5de4b..0000000 --- a/request_curl/api.py +++ /dev/null @@ -1,152 +0,0 @@ -""" -:copyright: (c) 2022 by Mauritz Uphoff. -""" -from .sessions import Session - - -def request(method, url, **kwargs): - """Constructs and sends a :class:`Request `. - :param method: method for the new :class:`Request` object: - ``GET``, ``OPTIONS``, ``HEAD``, ``POST``, ``PUT``, ``PATCH``, or ``DELETE``. - :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary, list of tuples or bytes to send - in the query string for the :class:`Request`. - :param data: (optional) Dictionary, list of tuples, bytes, or file-like - object to send in the body of the :class:`Request`. - :param json: (optional) A JSON serializable Python object - to send in the body of the :class:`Request`. - :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. - :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. - ``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')`` - or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string - defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers - to add for the file. - :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. - :param timeout: (optional) How many seconds to wait for the server to send data - before giving up, as a float, or a :ref:`(connect timeout, read - timeout) ` tuple. - :type timeout: float or tuple - :param allow_redirects: (optional) Boolean. Enable/disable - GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``True``. - :type allow_redirects: bool - :param proxies: (optional) String to the URL of the proxy. - :param verify: (optional) Either a boolean, in which case it controls whether we verify - the server's TLS certificate, or a string, in which case it must be a path - to a CA bundle to use. Defaults to ``True``. - :return: :class:`Response ` object - :rtype: requests.Response - - Usage:: - - >>> import request_curl - >>> req = request_curl.request('GET', 'https://httpbin.org/get') - >>> req - - """ - - # By using the 'with' statement we are sure the session is closed, thus we - # avoid leaving sockets open which can trigger a ResourceWarning in some - # cases, and look like a memory leak in others. - with Session() as session: - return session.request(method, url, **kwargs) - - -def get(url, params=None, **kwargs): - r"""Sends a GET request. - - :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary, list of tuples or bytes to send - in the query string for the :class:`Request`. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :return: :class:`Response ` object - :rtype: requests.Response - """ - - return request('get', url, params=params, **kwargs) - - -def options(url, **kwargs): - r"""Sends an OPTIONS request. - - :param url: URL for the new :class:`Request` object. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :return: :class:`Response ` object - :rtype: requests.Response - """ - - return request('options', url, **kwargs) - - -def head(url, **kwargs): - r"""Sends a HEAD request. - - :param url: URL for the new :class:`Request` object. - :param \*\*kwargs: Optional arguments that ``request`` takes. If - `allow_redirects` is not provided, it will be set to `False` (as - opposed to the default :meth:`request` behavior). - :return: :class:`Response ` object - :rtype: requests.Response - """ - - kwargs.setdefault('allow_redirects', False) - return request('head', url, **kwargs) - - -def post(url, data=None, json=None, **kwargs): - r"""Sends a POST request. - - :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, list of tuples, bytes, or file-like - object to send in the body of the :class:`Request`. - :param json: (optional) json data to send in the body of the :class:`Request`. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :return: :class:`Response ` object - :rtype: requests.Response - """ - - return request('post', url, data=data, json=json, **kwargs) - - -def put(url, data=None, **kwargs): - r"""Sends a PUT request. - - :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, list of tuples, bytes, or file-like - object to send in the body of the :class:`Request`. - :param json: (optional) json data to send in the body of the :class:`Request`. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :return: :class:`Response ` object - :rtype: requests.Response - """ - - return request('put', url, data=data, **kwargs) - - -def patch(url, data=None, **kwargs): - r"""Sends a PATCH request. - - :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, list of tuples, bytes, or file-like - object to send in the body of the :class:`Request`. - :param json: (optional) json data to send in the body of the :class:`Request`. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :return: :class:`Response ` object - :rtype: requests.Response - """ - - return request('patch', url, data=data, **kwargs) - - -def delete(url, **kwargs): - r"""Sends a DELETE request. - - :param url: URL for the new :class:`Request` object. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :return: :class:`Response ` object - :rtype: requests.Response - """ - - return request('delete', url, **kwargs) - - diff --git a/request_curl/defaults.py b/request_curl/defaults.py new file mode 100644 index 0000000..7c3f24d --- /dev/null +++ b/request_curl/defaults.py @@ -0,0 +1,47 @@ +from typing import Dict, List + +CHROME_UA: str = "".join( + [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_1)", + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36", + ] +) + +CHROME_HEADERS: Dict[str, str] = { + "sec-ch-ua": ' Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"Windows"', + "Upgrade-Insecure-Requests": "1", + "User-Agent": CHROME_UA, + "Accept": "".join( + [ + "text/html,application/xhtml+xml,application/xml;q=0.9,", + "image/avif,image/webp,image/apng,*/*;q=0.8,", + "application/signed-exchange;v=b3;q=0.9", + ] + ), + "Sec-Fetch-Site": "none", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-User": "?1", + "Sec-Fetch-Dest": "document", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "en-US,en;q=0.9", +} + +CHROME_CIPHER_SUITE: List[str] = [ + "TLS_AES_128_GCM_SHA256", + "TLS_AES_256_GCM_SHA384", + "TLS_CHACHA20_POLY1305_SHA256", + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-CHACHA20-POLY1305", + "ECDHE-RSA-CHACHA20-POLY1305", + "ECDHE-RSA-AES128-SHA", + "ECDHE-RSA-AES256-SHA", + "AES128-GCM-SHA256", + "AES256-GCM-SHA384", + "AES128-SHA", + "AES256-SHA", +] diff --git a/request_curl/dict.py b/request_curl/dict.py new file mode 100644 index 0000000..1ec4f60 --- /dev/null +++ b/request_curl/dict.py @@ -0,0 +1,41 @@ +from collections import OrderedDict +from typing import MutableMapping, Mapping + + +class CaseInsensitiveDict(MutableMapping): + def __init__(self, data=None, **kwargs): + self._store = OrderedDict() + if data is None: + data = {} + self.update(data, **kwargs) + + def __setitem__(self, key, value): + self._store[key.lower()] = (key, value) + + def __getitem__(self, key): + return self._store[key.lower()][1] + + def __delitem__(self, key): + del self._store[key.lower()] + + def __iter__(self): + return (casedkey for casedkey, mappedvalue in self._store.values()) + + def __len__(self): + return len(self._store) + + def lower_items(self): + return ((lowerkey, keyval[1]) for (lowerkey, keyval) in self._store.items()) + + def __eq__(self, other): + if isinstance(other, Mapping): + other = CaseInsensitiveDict(other) + else: + return NotImplemented + return dict(self.lower_items()) == dict(other.lower_items()) + + def copy(self): + return CaseInsensitiveDict(self._store.values()) + + def __repr__(self): + return str(dict(self.items())) diff --git a/request_curl/helper.py b/request_curl/helper.py index b5ce5d8..dc2d99b 100644 --- a/request_curl/helper.py +++ b/request_curl/helper.py @@ -3,16 +3,12 @@ from http.cookies import SimpleCookie -def to_cookiejar( - cookies: Union[str, CookieJar], - headers: List[Dict[str, str]] -) -> CookieJar: +def to_cookiejar(cookies: Union[str, CookieJar], headers: Dict[str, str]) -> CookieJar: if isinstance(cookies, CookieJar): return cookies _cookies: List[Tuple[str, str, str]] = [ - (get_cookie_name(cookie), get_cookie_value(cookie), "") - for cookie in cookies + (get_cookie_name(cookie), get_cookie_value(cookie), "") for cookie in cookies ] cookie_jar: CookieJar = CookieJar() @@ -20,16 +16,15 @@ def to_cookiejar( for name, value, domain in _cookies: cookie_jar.set_cookie(get_cookie(name, value, domain)) - for header in headers: - for key, value in header.items(): - if key.lower() != "set-cookie": - continue + for key, value in headers.items(): + if key.lower() != "set-cookie": + continue - header_set_cookie = SimpleCookie() - header_set_cookie.load(value) + header_set_cookie = SimpleCookie() + header_set_cookie.load(value) - for n, v in header_set_cookie.items(): - cookie_jar.set_cookie(get_cookie(n, v.value, "")) + for n, v in header_set_cookie.items(): + cookie_jar.set_cookie(get_cookie(n, v.value, "")) return cookie_jar @@ -44,15 +39,15 @@ def get_cookie(name: str, value: str, domain: str): domain=domain, domain_specified=False, domain_initial_dot=False, - path='/', + path="/", path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, - rest={'HttpOnly': ""}, - rfc2109=False + rest={"HttpOnly": ""}, + rfc2109=False, ) diff --git a/request_curl/models.py b/request_curl/models.py index 21ea058..d2e7a2b 100644 --- a/request_curl/models.py +++ b/request_curl/models.py @@ -8,9 +8,7 @@ import brotli import pycurl -from collections import OrderedDict -from collections.abc import Mapping, MutableMapping - +from request_curl.dict import CaseInsensitiveDict from request_curl.helper import to_cookiejar CURL_INFO_MAPPING: Dict[str, Any] = { @@ -52,10 +50,7 @@ class Response: def __init__( - self, - curl: pycurl.Curl, - body_output: BytesIO, - headers_output: BytesIO + self, curl: pycurl.Curl, body_output: BytesIO, headers_output: BytesIO ): self._curl: pycurl.Curl = curl @@ -110,13 +105,17 @@ def text(self) -> str: def __set_text(self): try: if not self._text: - self._text = self._body_output.getvalue().decode('UTF-8') + self._text = self._body_output.getvalue().decode("UTF-8") if "gzip" in self.__get_header_value("Content-Encoding"): try: if "ISO-8859-1" in self.__get_header_value("Content-Type"): - self._text = str(self.__decode_gzip(self._body_output), "ISO-8859-1") + self._text = str( + self.__decode_gzip(self._body_output), "ISO-8859-1" + ) else: - self._text = str(self.__decode_gzip(self._body_output), "utf-8") + self._text = str( + self.__decode_gzip(self._body_output), "utf-8" + ) except zlib.error: pass elif "br" in self.__get_header_value("Content-Encoding"): @@ -126,7 +125,7 @@ def __set_text(self): pass except Exception as e: - self._text = '' + self._text = "" @property def cookies(self) -> CookieJar: @@ -140,14 +139,15 @@ def parse_header_block(header_raw_block: List[str]): for header in header_raw_block: if not header.startswith("HTTP"): key, value = map(lambda u: u.strip(), header.split(":", 1)) - block_headers.append({key: value}) + block_headers.append((key, value)) return block_headers - raw_headers = self._headers_output.getvalue().decode('UTF-8') + raw_headers = self._headers_output.getvalue().decode("UTF-8") - for raw_block in self.__split_headers_blocks(raw_headers): - self._headers = parse_header_block(raw_block) + self._headers = CaseInsensitiveDict( + parse_header_block(self.__split_headers_blocks(raw_headers)[-1]) + ) @staticmethod def __split_headers_blocks(raw_headers): @@ -181,52 +181,8 @@ def __get_curl_info(self) -> dict: return self._response_info def __get_header_value(self, key: str) -> str: - for item in self.headers: - for k, value in item.items(): - if k.lower() == key.lower(): - return value + for k, value in self.headers.items(): + if k.lower() == key.lower(): + return value return "" - - -class CaseInsensitiveDict(MutableMapping): - def __init__(self, data=None, **kwargs): - self._store = OrderedDict() - if data is None: - data = {} - self.update(data, **kwargs) - - def __setitem__(self, key, value): - self._store[key.lower()] = (key, value) - - def __getitem__(self, key): - return self._store[key.lower()][1] - - def __delitem__(self, key): - del self._store[key.lower()] - - def __iter__(self): - return (casedkey for casedkey, mappedvalue in self._store.values()) - - def __len__(self): - return len(self._store) - - def lower_items(self): - return ( - (lowerkey, keyval[1]) - for (lowerkey, keyval) - in self._store.items() - ) - - def __eq__(self, other): - if isinstance(other, Mapping): - other = CaseInsensitiveDict(other) - else: - return NotImplemented - return dict(self.lower_items()) == dict(other.lower_items()) - - def copy(self): - return CaseInsensitiveDict(self._store.values()) - - def __repr__(self): - return str(dict(self.items())) diff --git a/request_curl/sessions.py b/request_curl/sessions.py index 1d8a4db..82e456a 100644 --- a/request_curl/sessions.py +++ b/request_curl/sessions.py @@ -1,4 +1,4 @@ -from http.cookiejar import CookieJar, Cookie +from http.cookiejar import CookieJar from io import BytesIO from typing import Dict, Optional, List, Any, Union import json as _json @@ -10,48 +10,6 @@ from request_curl.helper import get_cookie from request_curl.models import Response -DEFAULT_UA: str = "".join([ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_1)" - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" -]) - -DEFAULT_HEADER: Dict[str, str] = { - "sec-ch-ua": ' Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98', - "sec-ch-ua-mobile": "?0", - "sec-ch-ua-platform": '"Windows"', - "Upgrade-Insecure-Requests": "1", - "User-Agent": DEFAULT_UA, - "Accept": "".join([ - "text/html,application/xhtml+xml,application/xml;q=0.9,", - "image/avif,image/webp,image/apng,*/*;q=0.8," - "application/signed-exchange;v=b3;q=0.9" - ]), - "Sec-Fetch-Site": "none", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-User": "?1", - "Sec-Fetch-Dest": "document", - "Accept-Encoding": "gzip, deflate, br", - "Accept-Language": "en-US,en;q=0.9", -} - -DEFAULT_CIPHER_SUITE: List[str] = [ - "TLS_AES_128_GCM_SHA256", - "TLS_AES_256_GCM_SHA384", - "TLS_CHACHA20_POLY1305_SHA256", - "ECDHE-ECDSA-AES128-GCM-SHA256", - "ECDHE-RSA-AES128-GCM-SHA256", - "ECDHE-ECDSA-AES256-GCM-SHA384", - "ECDHE-RSA-AES256-GCM-SHA384", - "ECDHE-ECDSA-CHACHA20-POLY1305", - "ECDHE-RSA-CHACHA20-POLY1305", - "ECDHE-RSA-AES128-SHA", - "ECDHE-RSA-AES256-SHA", - "AES128-GCM-SHA256", - "AES256-GCM-SHA384", - "AES128-SHA", - "AES256-SHA", -] - class Session: """A request_curl session. @@ -66,62 +24,56 @@ class Session: Or as a context manager:: - >>> with request_curl.Session() as session: + >>> with request_curl.Session() as s: ... s.get('https://httpbin.org/get') """ + def __init__( self, - headers: Dict[str, str] = DEFAULT_HEADER, - cipher_suite: List[str] = DEFAULT_CIPHER_SUITE, - proxies: str = "", + headers: Dict[str, str] = None, + cipher_suite: List[str] = None, http2: bool = False, - accept_encoding: str = "gzip, deflate" + proxies: str = "", ): - self._curl = pycurl.Curl() - self._curl.setopt(pycurl.ACCEPT_ENCODING, accept_encoding) - - if http2: - self._curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_2_0) - else: - self._curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_1_1) + self.curl = pycurl.Curl() + self.headers = headers if headers else {} + self.cipher_suite = cipher_suite if cipher_suite else [] + self.http2 = http2 + self.proxies = proxies - if proxies: - self.__set_proxies(proxies) - else: - self._curl.setopt(pycurl.PROXY, "") - self._curl.setopt(pycurl.PROXYUSERPWD, "") + self.__debug_entries = [] + self.cookies = cookiejar_from_dict({}) - self._curl.setopt(pycurl.SSL_CIPHER_LIST, ",".join(cipher_suite)) + def __enter__(self): + return self - if not headers.get("user-agent"): - headers['user-agent'] = DEFAULT_UA + def __exit__(self, *args): + self.curl.close() - if headers: - self._curl.setopt( - pycurl.HTTPHEADER, - [f"{k}: {v}" for k, v in headers.items()] - ) - else: - self._curl.setopt( - pycurl.HTTPHEADER, - [f"{k}: {v}" for k, v in DEFAULT_HEADER.items()] + def __set_settings(self): + if self.headers: + self.curl.setopt( + pycurl.HTTPHEADER, [f"{k}: {v}" for k, v in self.headers.items()] ) - self.cookies: CookieJar = cookiejar_from_dict({}) + if self.http2: + self.curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_2_0) + else: + self.curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_1_1) - def __enter__(self): - return self + if len(self.proxies) > 0: + self.__set_proxies() - def __exit__(self, *args): - self._curl.close() + if len(self.cipher_suite) > 0: + self.curl.setopt(pycurl.SSL_CIPHER_LIST, ",".join(self.cipher_suite)) - def __set_proxies(self, proxies: str) -> None: - proxy_split: List[str] = proxies.split(":") - self._curl.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_HTTP) - self._curl.setopt(pycurl.PROXY, f"{proxy_split[0]}:{proxy_split[1]}") + def __set_proxies(self) -> None: + proxy_split: List[str] = self.proxies.split(":") + self.curl.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_HTTP) + self.curl.setopt(pycurl.PROXY, f"{proxy_split[0]}:{proxy_split[1]}") if len(proxy_split) > 3: - self._curl.setopt(pycurl.PROXYUSERPWD, f"{proxy_split[2]}:{proxy_split[3]}") + self.curl.setopt(pycurl.PROXYUSERPWD, f"{proxy_split[2]}:{proxy_split[3]}") def __add_cookies_to_session(self, cookies: CookieJar) -> None: self.cookies = merge_cookies(self.cookies, cookies) @@ -136,15 +88,16 @@ def request( self, method: str, url: str, + headers: Optional[Dict[str, str]] = None, params: Optional[Dict[str, str]] = None, data: Optional[Dict[str, Any]] = None, json: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = DEFAULT_HEADER, proxies: Optional[str] = None, timeout: Union[float, int] = 60, allow_redirects: bool = True, - http2: bool = True, - verify: bool = True + http2: bool = False, + verify: bool = True, + debug: bool = False, ): """Constructs a :class:`Request `, prepares it and sends it. Returns :class:`Response ` object. @@ -164,7 +117,7 @@ def request( :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) ` tuple. - :type timeout: float or tuple + :type timeout: float :param allow_redirects: (optional) Set to True by default. :type allow_redirects: bool :param http2: (optional) Set http2 to True by default. @@ -177,47 +130,62 @@ def request( certificates, which will make your application vulnerable to man-in-the-middle (MitM) attacks. Setting verify to ``False`` may be useful during local development or testing. - :rtype: requests.Response + :param debug: (optional) Set debug mode. + :type debug: bool + :rtype: Response """ + self.curl.reset() + self.__set_settings() if method.upper() == "POST": - self._curl.setopt(pycurl.POST, 1) + self.curl.setopt(pycurl.POST, 1) elif method.upper() == "GET": - self._curl.setopt(pycurl.HTTPGET, 1) + self.curl.setopt(pycurl.HTTPGET, 1) elif method.upper() == "HEAD": - self._curl.setopt(pycurl.NOBODY, 1) + self.curl.setopt(pycurl.NOBODY, 1) else: - self._curl.setopt(pycurl.CUSTOMREQUEST, method.upper()) + self.curl.setopt(pycurl.CUSTOMREQUEST, method.upper()) if params: url = url + "?" + "&".join([f"{k}={v};" for k, v in params.items()]) - self._curl.setopt(pycurl.URL, url) + self.curl.setopt(pycurl.URL, url) + self.curl.setopt(pycurl.FOLLOWLOCATION, allow_redirects) + self.curl.setopt(pycurl.TIMEOUT, timeout) if not verify: - self._curl.setopt(pycurl.SSL_VERIFYPEER, 0) - self._curl.setopt(pycurl.SSL_VERIFYHOST, 0) + self.curl.setopt(pycurl.SSL_VERIFYPEER, 0) + self.curl.setopt(pycurl.SSL_VERIFYHOST, 0) if proxies: - self.__set_proxies(proxies) + self.proxies = proxies + self.__set_proxies() - if not http2: - self._curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_1_1) + if http2: + self.curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_2_0) - self._curl.setopt(pycurl.FOLLOWLOCATION, allow_redirects) - self._curl.setopt(pycurl.TIMEOUT, timeout) + if headers: + self.curl.setopt( + pycurl.HTTPHEADER, [f"{k}: {v}" for k, v in headers.items()] + ) if data: form: List[str] = [f"{k}={v}" for k, v in data.items()] - self._curl.setopt(pycurl.POSTFIELDS, "&".join(form).encode("utf-8")) + self.curl.setopt(pycurl.POSTFIELDS, "&".join(form).encode("utf-8")) if json: + headers = headers.copy() if headers else self.headers.copy() headers["Accept"] = "application/json" headers["Content-Type"] = "application/json" headers["charset"] = "utf-8" + + self.curl.setopt( + pycurl.HTTPHEADER, [f"{k}: {v}" for k, v in headers.items()] + ) + if isinstance(json, dict): json_data = _json.dumps(json) - self._curl.setopt(pycurl.POSTFIELDS, json_data) + self.curl.setopt(pycurl.POSTFIELDS, json_data) if self.cookies: chunks = [] @@ -225,45 +193,47 @@ def request( name, value = quote_plus(cookie.name), quote_plus(cookie.value) chunks.append(f"{name}={value};") if chunks: - self._curl.setopt(pycurl.COOKIE, "".join(chunks)) - else: - self._curl.setopt(pycurl.COOKIELIST, "") + self.curl.setopt(pycurl.COOKIE, "".join(chunks)) + + if debug: + self.__debug_entries = [] + self.curl.setopt(pycurl.VERBOSE, 1) body_output: BytesIO = BytesIO() headers_output: BytesIO = BytesIO() - self._curl.setopt(pycurl.HEADERFUNCTION, headers_output.write) - self._curl.setopt(pycurl.WRITEFUNCTION, body_output.write) + self.curl.setopt(pycurl.HEADERFUNCTION, headers_output.write) + self.curl.setopt(pycurl.WRITEFUNCTION, body_output.write) - self._curl.perform() + self.curl.perform() - response = Response(self._curl, body_output, headers_output) + if debug: + print("\n".join(self.__debug_entries)) + + response = Response(self.curl, body_output, headers_output) self.__add_cookies_to_session(response.cookies) return response def debug_function(self, t, b): if t in [1, 2, 5, 6]: - self.debug_entries.append(b.decode("utf-8")) + self.__debug_entries.append(b.decode("utf-8")) def get(self, url, **kwargs): - r"""Sends a GET request. Returns :class:`Response` object. - - :param url: URL for the new :class:`Request` object. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :rtype: requests.Response - """ - - return self.request('GET', url, **kwargs) + r"""Sends a GET request. Returns :class:`Response` object.""" + return self.request("GET", url, **kwargs) def post(self, url, **kwargs): - r"""Sends a POST request. Returns :class:`Response` object. + r"""Sends a POST request. Returns :class:`Response` object.""" + return self.request("POST", url, **kwargs) - :param url: URL for the new :class:`Request` object. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :rtype: requests.Response - """ + def options(self, url, **kwargs): + r"""Sends a OPTIONS request. Returns :class:`Response` object.""" + return self.request("OPTIONS", url, **kwargs) - return self.request('POST', url, **kwargs) + def delete(self, url, **kwargs): + r"""Sends a DELETE request. Returns :class:`Response` object.""" + return self.request("DELETE", url, **kwargs) - def options(self, url, **kwargs): - return self.request('OPTIONS', url, **kwargs) + def put(self, url, **kwargs): + r"""Sends a PUT request. Returns :class:`Response` object.""" + return self.request("PUT", url, **kwargs) diff --git a/tests/test_request_curl.py b/tests/test_request_curl.py new file mode 100644 index 0000000..8831c09 --- /dev/null +++ b/tests/test_request_curl.py @@ -0,0 +1,162 @@ +import request_curl +from request_curl import CHROME_CIPHER_SUITE, CHROME_HEADERS, CHROME_UA +from request_curl.dict import CaseInsensitiveDict + +TLS_API: str = "https://tls.peet.ws/api/all" +HTTP_BIN_API: str = "https://httpbin.org" + + +def test_context_manager(): + with request_curl.Session() as session: + r = session.get(HTTP_BIN_API + "/get") + assert r.status_code == 200 + + +def test_http_version(): + session = request_curl.Session(http2=True) + response = session.get(TLS_API).json + assert response.get("http_version") == "h2" + + session = request_curl.Session(http2=False) + response = session.get(TLS_API).json + assert response.get("http_version") == "HTTP/1.1" + + +def test_request_methods(): + session = request_curl.Session() + + assert session.get(TLS_API).json["method"] == "GET" + assert session.post(TLS_API).json["method"] == "POST" + assert session.options(TLS_API).json["method"] == "OPTIONS" + assert session.delete(TLS_API).json["method"] == "DELETE" + assert session.put(TLS_API).json["method"] == "PUT" + + +def test_response_object_props(): + session = request_curl.Session() + + for url in [TLS_API, HTTP_BIN_API + "/get"]: + response = session.get(url) + + assert isinstance(response.status_code, int) and response.status_code >= 0 + assert isinstance(response.content, bytes) and len(response.content) > 0 + assert isinstance(response.text, str) and len(response.text) > 0 + assert isinstance(response.json, dict) and len(response.json) > 0 + assert isinstance(response.url, str) and len(response.url) > 0 + + +def test_custom_cipher_suite(): + cipher_suite = [ + "AES128-SHA256", + "AES256-SHA256", + "AES128-GCM-SHA256", + "AES256-GCM-SHA384", + ] + session = request_curl.Session(cipher_suite=cipher_suite) + response = session.get(TLS_API) + + r_cipher_suite = [value for _, value in enumerate(response.json["tls"]["ciphers"])] + + assert "TLS_RSA_WITH_AES_128_CBC_SHA256" in r_cipher_suite + assert "TLS_RSA_WITH_AES_256_CBC_SHA256" in r_cipher_suite + assert "TLS_RSA_WITH_AES_128_GCM_SHA256" in r_cipher_suite + assert "TLS_RSA_WITH_AES_256_GCM_SHA384" in r_cipher_suite + + +def test_custom_request_header(): + session = request_curl.Session(headers={"user-agent": "request_curl"}) + response = session.get(TLS_API) + + assert "user-agent: request_curl" in "".join(response.json["http1"]["headers"]) + + +def test_request_form_data(): + data = {"key": "value"} + session = request_curl.Session() + response = session.post(HTTP_BIN_API + "/post", data=data) + + assert response.json["form"] == data + + +def test_request_json_body(): + _json = {"key": "value"} + session = request_curl.Session() + response = session.post(HTTP_BIN_API + "/post", json=_json) + + assert response.json["json"] == _json + + +def test_request_url_params(): + params = {"key": "value"} + session = request_curl.Session() + response = session.get(HTTP_BIN_API + "/get", params=params) + + assert "key=value" in response.url + + +def test_session_cookies_add(): + session = request_curl.Session(http2=True, headers={}) + session.add_cookie("a", "b", "c") + assert len(session.cookies) >= 0 + assert session.cookies["a"] == "b" + + +def test_session_cookies_request(): + session = request_curl.Session(http2=True, headers={}) + session.get("https://google.com") + assert len(session.cookies) >= 0 + session.get(HTTP_BIN_API) + assert len(session.cookies) >= 0 + session.get(TLS_API) + assert len(session.cookies) >= 0 + + +def test_curl_reset(): + session = request_curl.Session(http2=False, headers={"user-agent": "test_1"}) + response = session.get(TLS_API) + assert "user-agent: test_1" in "".join(response.json["http1"]["headers"]) + + response = session.get(TLS_API, headers={"user-agent": "test_2"}) + assert "user-agent: test_1" not in "".join(response.json["http1"]["headers"]) + assert "user-agent: test_2" in "".join(response.json["http1"]["headers"]) + + response = session.post(TLS_API, json={"key": "value"}) + assert "application/json" in "".join(response.json["http1"]["headers"]) + + response = session.post(TLS_API, data={"key": "value"}) + assert "application/json" not in "".join(response.json["http1"]["headers"]) + + response = session.post(TLS_API, data={"key": "value"}, http2=True) + assert response.json.get("http_version") == "h2" + + +def test_debug(): + session = request_curl.Session(http2=True, headers={"user-agent": "test_1"}) + response = session.get(TLS_API, debug=True) + + assert response.status_code == 200 + + +def test_response_header(): + session = request_curl.Session(http2=True, headers={"user-agent": "test_1"}) + response = session.get(HTTP_BIN_API) + + assert isinstance(response.headers, CaseInsensitiveDict) + assert isinstance(response.headers["content-type"], str) + + +def test_with_chrome_fp(): + session = request_curl.Session( + http2=True, cipher_suite=CHROME_CIPHER_SUITE, headers=CHROME_HEADERS + ) + + response = session.get(TLS_API) + + assert response.json["user_agent"] == CHROME_UA + + +def test_response_cookies(): + session = request_curl.Session() + response = session.get("https://google.com") + + assert len(response.cookies) > 0 diff --git a/tests/test_tls_requests.py b/tests/test_tls_requests.py deleted file mode 100644 index 35db9ea..0000000 --- a/tests/test_tls_requests.py +++ /dev/null @@ -1,14 +0,0 @@ -import request_curl - - -def test_request_curl(): - session = request_curl.Session(http2=True, headers={}) - response = session.get("https://google.com") - assert response.status_code == 200 - - -def test_session_cookies(): - session = request_curl.Session(http2=True, headers={}) - session.add_cookie("a", "b", "c") - assert len(session.cookies) >= 0 -