Set NO_CALLBACK for HttpClientProvider requests.

scrapinghub · Oct 31, 2023 · 353f891 · 353f891
1 parent 7b03d71
commit 353f891
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 7 deletions.
diff --git a/scrapy_poet/utils/__init__.py b/scrapy_poet/utils/__init__.py
@@ -17,6 +17,11 @@
     default_registry,
 )
 
+try:
+    from scrapy.http.request import NO_CALLBACK  # available on Scrapy >= 2.8
+except ImportError:
+    NO_CALLBACK = lambda: None  # noqa: E731
+
 
 def get_scrapy_data_path(createdir: bool = True, default_dir: str = ".scrapy") -> str:
     """Return a path to a folder where Scrapy is storing data.
@@ -37,6 +42,7 @@ def http_request_to_scrapy_request(request: HttpRequest, **kwargs) -> Request:
         method=request.method,
         headers=request.headers,
         body=request.body,
+        callback=NO_CALLBACK,
         **kwargs,
     )
 

diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -8,6 +8,7 @@
 from web_poet import HttpRequest, HttpResponse
 
 from scrapy_poet.utils import (
+    NO_CALLBACK,
     create_registry_instance,
     get_scrapy_data_path,
     http_request_to_scrapy_request,
@@ -39,32 +40,32 @@ def test_get_scrapy_data_path(mock_inside_project, mock_makedirs, tmp_path):
         (
             HttpRequest("https://example.com"),
             {},
-            Request("https://example.com"),
+            Request("https://example.com", callback=NO_CALLBACK),
         ),
         (
             HttpRequest("https://example.com"),
             {"dont_filter": True},
-            Request("https://example.com", dont_filter=True),
+            Request("https://example.com", callback=NO_CALLBACK, dont_filter=True),
         ),
         (
             HttpRequest("https://example.com", method="POST"),
             {},
-            Request("https://example.com", method="POST"),
+            Request("https://example.com", callback=NO_CALLBACK, method="POST"),
         ),
         (
             HttpRequest("https://example.com", headers={"a": "b"}),
             {},
-            Request("https://example.com", headers={"a": "b"}),
+            Request("https://example.com", callback=NO_CALLBACK, headers={"a": "b"}),
         ),
         (
             HttpRequest("https://example.com", headers={"a": "b"}),
             {},
-            Request("https://example.com", headers=(("a", "b"),)),
+            Request("https://example.com", callback=NO_CALLBACK, headers=(("a", "b"),)),
         ),
         (
             HttpRequest("https://example.com", headers=(("a", "b"),)),
             {},
-            Request("https://example.com", headers=(("a", "b"),)),
+            Request("https://example.com", callback=NO_CALLBACK, headers=(("a", "b"),)),
         ),
         (
             HttpRequest(
@@ -74,13 +75,14 @@ def test_get_scrapy_data_path(mock_inside_project, mock_makedirs, tmp_path):
             {},
             Request(
                 "https://example.com",
+                callback=NO_CALLBACK,
                 headers=(("a", "b"), ("a", "c")),
             ),
         ),
         (
             HttpRequest("https://example.com", body=b"a"),
             {},
-            Request("https://example.com", body=b"a"),
+            Request("https://example.com", callback=NO_CALLBACK, body=b"a"),
         ),
     ),
 )