Fixes for newer Scrapy.

Author: wRAR

scrapy_poet/spidermiddlewares.py

@@ -21,6 +21,7 @@ def process_spider_exception(
 
         if not isinstance(exception, Retry):
             return None
+        assert response.request
         new_request_or_none = get_retry_request(
             response.request,
             spider=spider,

scrapy_poet/utils/__init__.py

@@ -20,8 +20,7 @@
 try:
     from scrapy.http.request import NO_CALLBACK  # available on Scrapy >= 2.8
 except ImportError:
-    # NO_CALLBACK = lambda: None  # noqa: E731
-    NO_CALLBACK = None
+    NO_CALLBACK = None  # type: ignore[assignment]
 
 
 def get_scrapy_data_path(createdir: bool = True, default_dir: str = ".scrapy") -> str:

tests/test_downloader.py

@@ -444,6 +444,7 @@ async def __call__(
             custom_request = Request(
                 request.url, body=request.body, callback=NO_CALLBACK
             )
+            assert crawler.engine
             scrapy_response: Response = await maybe_deferred_to_future(
                 crawler.engine.download(custom_request)
             )

tests/test_response_required_logic.py

@@ -22,13 +22,7 @@
     HttpResponseProvider,
     PageObjectInputProvider,
 )
-from scrapy_poet.utils import is_min_scrapy_version
-
-# See: https://github.com/scrapinghub/scrapy-poet/issues/118
-try:
-    from scrapy.http.request import NO_CALLBACK  # available on Scrapy >= 2.8
-except ImportError:
-    NO_CALLBACK = lambda: None  # type: ignore[misc]  # noqa: E731
+from scrapy_poet.utils import NO_CALLBACK, is_min_scrapy_version
 
 
 @attr.s(auto_attribs=True)