diff --git a/scrapy_poet/utils/__init__.py b/scrapy_poet/utils/__init__.py index 62e37012..0d86ba0c 100644 --- a/scrapy_poet/utils/__init__.py +++ b/scrapy_poet/utils/__init__.py @@ -17,6 +17,11 @@ default_registry, ) +try: + from scrapy.http.request import NO_CALLBACK # available on Scrapy >= 2.8 +except ImportError: + NO_CALLBACK = lambda: None # noqa: E731 + def get_scrapy_data_path(createdir: bool = True, default_dir: str = ".scrapy") -> str: """Return a path to a folder where Scrapy is storing data. @@ -37,6 +42,7 @@ def http_request_to_scrapy_request(request: HttpRequest, **kwargs) -> Request: method=request.method, headers=request.headers, body=request.body, + callback=NO_CALLBACK, **kwargs, ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 69b52a87..27fa4222 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,6 +8,7 @@ from web_poet import HttpRequest, HttpResponse from scrapy_poet.utils import ( + NO_CALLBACK, create_registry_instance, get_scrapy_data_path, http_request_to_scrapy_request, @@ -39,32 +40,32 @@ def test_get_scrapy_data_path(mock_inside_project, mock_makedirs, tmp_path): ( HttpRequest("https://example.com"), {}, - Request("https://example.com"), + Request("https://example.com", callback=NO_CALLBACK), ), ( HttpRequest("https://example.com"), {"dont_filter": True}, - Request("https://example.com", dont_filter=True), + Request("https://example.com", callback=NO_CALLBACK, dont_filter=True), ), ( HttpRequest("https://example.com", method="POST"), {}, - Request("https://example.com", method="POST"), + Request("https://example.com", callback=NO_CALLBACK, method="POST"), ), ( HttpRequest("https://example.com", headers={"a": "b"}), {}, - Request("https://example.com", headers={"a": "b"}), + Request("https://example.com", callback=NO_CALLBACK, headers={"a": "b"}), ), ( HttpRequest("https://example.com", headers={"a": "b"}), {}, - Request("https://example.com", headers=(("a", "b"),)), + Request("https://example.com", callback=NO_CALLBACK, headers=(("a", "b"),)), ), ( HttpRequest("https://example.com", headers=(("a", "b"),)), {}, - Request("https://example.com", headers=(("a", "b"),)), + Request("https://example.com", callback=NO_CALLBACK, headers=(("a", "b"),)), ), ( HttpRequest( @@ -74,13 +75,14 @@ def test_get_scrapy_data_path(mock_inside_project, mock_makedirs, tmp_path): {}, Request( "https://example.com", + callback=NO_CALLBACK, headers=(("a", "b"), ("a", "c")), ), ), ( HttpRequest("https://example.com", body=b"a"), {}, - Request("https://example.com", body=b"a"), + Request("https://example.com", callback=NO_CALLBACK, body=b"a"), ), ), )