Add support for annotated deps in savefixture.

docs/providers.rst

@@ -344,7 +344,8 @@ To handle this you need the following changes in your provider:
 .. code-block:: python
 
     from andi.typeutils import strip_annotated
-    from scrapy_poet import AnnotatedResult, PageObjectInputProvider
+    from scrapy_poet import PageObjectInputProvider
+    from web_poet import AnnotatedResult
 
 
     class Provider(PageObjectInputProvider):
@@ -360,10 +361,7 @@ To handle this you need the following changes in your provider:
                 metadata = getattr(cls, "__metadata__", None)
                 obj = ...  # create the instance using cls and metadata
                 if metadata:
-                    # wrap the instance into a scrapy_poet.AnnotatedResult object
+                    # wrap the instance into a web_poet.AnnotatedResult object
                     obj = AnnotatedResult(obj, metadata)
                 result.append(obj)
             return result
-
-.. autoclass:: scrapy_poet.AnnotatedResult
-   :members:

scrapy_poet/__init__.py

@@ -1,4 +1,4 @@
-from .api import AnnotatedResult, DummyResponse, callback_for
+from .api import DummyResponse, callback_for
 from .downloadermiddlewares import DownloaderStatsMiddleware, InjectionMiddleware
 from .page_input_providers import HttpResponseProvider, PageObjectInputProvider
 from .spidermiddlewares import RetryMiddleware

scrapy_poet/api.py

@@ -1,6 +1,5 @@
-from dataclasses import dataclass
 from inspect import iscoroutinefunction
-from typing import Any, Callable, Optional, Tuple, Type
+from typing import Callable, Optional, Type
 
 from scrapy.http import Request, Response
 from web_poet.pages import ItemPage
@@ -134,28 +133,3 @@ def parse(*args, item: page_or_item_cls, **kwargs):  # type:ignore
 
     setattr(parse, _CALLBACK_FOR_MARKER, True)
     return parse
-
-
-@dataclass
-class AnnotatedResult:
-    """Wrapper for annotated dependencies.
-
-    When a provider gets a :data:`typing.Annotated` type as a dependency type,
-    it will return an ``AnnotatedResult`` instance for it so that the caller
-    can match the dependency to its annotation.
-
-    :param result: The wrapped dependency instance.
-    :type result: Any
-
-    :param metadata: The copy of the annotation.
-    :type metadata: Tuple[Any, ...]
-    """
-
-    result: Any
-    metadata: Tuple[Any, ...]
-
-    def get_annotated_cls(self):
-        """Returns a re-created :class:`typing.Annotated` type."""
-        from typing import Annotated
-
-        return Annotated[(type(self.result), *self.metadata)]

scrapy_poet/commands.py

@@ -13,7 +13,7 @@
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
 from twisted.internet.defer import inlineCallbacks
-from web_poet import ItemPage
+from web_poet import AnnotatedResult, ItemPage
 from web_poet.exceptions import PageObjectAction
 from web_poet.testing import Fixture
 from web_poet.utils import ensure_awaitable
@@ -43,7 +43,11 @@ def build_instances_from_providers(
             request, response, plan
         )
         if request.meta.get("savefixture", False):
-            saved_dependencies.extend(instances.values())
+            for cls, value in instances.items():
+                metadata = getattr(cls, "__metadata__", None)
+                if metadata:
+                    value = AnnotatedResult(value, metadata)
+                saved_dependencies.append(value)
         return instances
 
 

scrapy_poet/injection.py

@@ -29,13 +29,13 @@
 from scrapy.utils.defer import deferred_from_coro, maybeDeferred_coro
 from scrapy.utils.misc import load_object
 from twisted.internet.defer import inlineCallbacks
-from web_poet import RulesRegistry
+from web_poet import AnnotatedResult, RulesRegistry
 from web_poet.page_inputs.http import request_fingerprint
 from web_poet.pages import ItemPage, is_injectable
 from web_poet.serialization.api import deserialize_leaf, load_class, serialize
 from web_poet.utils import get_fq_class_name
 
-from scrapy_poet.api import _CALLBACK_FOR_MARKER, AnnotatedResult, DummyResponse
+from scrapy_poet.api import _CALLBACK_FOR_MARKER, DummyResponse
 from scrapy_poet.cache import SerializedDataCache
 from scrapy_poet.injection_errors import (
     NonCallableProviderError,

tests/test_commands.py

@@ -5,6 +5,7 @@
 import tempfile
 from pathlib import Path
 
+import pytest
 from twisted.web.resource import Resource
 from web_poet.testing import Fixture
 
@@ -43,9 +44,8 @@ def test_savefixture(tmp_path) -> None:
     (cwd / project_name / "po.py").write_text(
         """
 import attrs
-from web_poet import HttpClient
+from web_poet import HttpClient, WebPage
 from web_poet.exceptions import HttpRequestError, HttpResponseError
-from web_poet.pages import WebPage
 
 
 @attrs.define
@@ -115,7 +115,7 @@ class MySpider(Spider):
     (cwd / project_name / "po.py").write_text(
         """
 import json
-from web_poet.pages import WebPage
+from web_poet import WebPage
 
 
 class HeadersPage(WebPage):
@@ -146,8 +146,8 @@ def test_savefixture_expected_exception(tmp_path) -> None:
     type_name = "foo.po.SamplePage"
     (cwd / project_name / "po.py").write_text(
         """
+from web_poet import WebPage
 from web_poet.exceptions import UseFallback
-from web_poet.pages import WebPage
 
 
 class SamplePage(WebPage):
@@ -222,3 +222,88 @@ class CustomItemAdapter(ItemAdapter):
     assert fixture.is_valid()
     item = json.loads(fixture.output_path.read_bytes())
     assert item == {"name": "chocolate"}
+
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
+)
+def test_savefixture_annotated(tmp_path) -> None:
+    project_name = "foo"
+    cwd = Path(tmp_path)
+    call_scrapy_command(str(cwd), "startproject", project_name)
+    cwd /= project_name
+    type_name = "foo.po.BTSBookPage"
+    (cwd / project_name / "providers.py").write_text(
+        """
+from andi.typeutils import strip_annotated
+from scrapy.http import Response
+from scrapy_poet import HttpResponseProvider
+from web_poet import (
+    AnnotatedResult,
+    HttpResponse,
+    HttpResponseHeaders,
+)
+
+
+class AnnotatedHttpResponseProvider(HttpResponseProvider):
+    def is_provided(self, type_) -> bool:
+        return super().is_provided(strip_annotated(type_))
+
+    def __call__(self, to_provide, response: Response):
+        result = []
+        for cls in to_provide:
+            obj = HttpResponse(
+                url=response.url,
+                body=response.body,
+                status=response.status,
+                headers=HttpResponseHeaders.from_bytes_dict(response.headers),
+            )
+            if metadata := getattr(cls, "__metadata__", None):
+                obj = AnnotatedResult(obj, metadata)
+            result.append(obj)
+        return result
+"""
+    )
+    (cwd / project_name / "po.py").write_text(
+        """
+from typing import Annotated
+
+import attrs
+from web_poet import HttpResponse, WebPage
+
+
+@attrs.define
+class BTSBookPage(WebPage):
+
+    response: Annotated[HttpResponse, "foo", 42]
+
+    async def to_item(self):
+        return {
+            'url': self.url,
+            'name': self.css("h1.name::text").get(),
+        }
+"""
+    )
+    with (cwd / project_name / "settings.py").open("a") as f:
+        f.write(
+            f"""
+SCRAPY_POET_PROVIDERS = {{"{project_name}.providers.AnnotatedHttpResponseProvider": 500}}
+"""
+        )
+
+    with MockServer(CustomResource) as server:
+        call_scrapy_command(
+            str(cwd),
+            "savefixture",
+            type_name,
+            f"{server.root_url}",
+        )
+    fixtures_dir = cwd / "fixtures"
+    fixture_dir = fixtures_dir / type_name / "test-1"
+    fixture = Fixture(fixture_dir)
+    assert fixture.is_valid()
+    assert (fixture.input_path / "AnnotatedResult HttpResponse-metadata.json").exists()
+    assert (
+        fixture.input_path / "AnnotatedResult HttpResponse-result-body.html"
+    ).exists()
+    assert fixture.meta_path.exists()

tests/test_injection.py

@@ -11,13 +11,12 @@
 from scrapy.http import Response
 from url_matcher import Patterns
 from url_matcher.util import get_domain
-from web_poet import Injectable, ItemPage, RulesRegistry, field
+from web_poet import AnnotatedResult, Injectable, ItemPage, RulesRegistry, field
 from web_poet.mixins import ResponseShortcutsMixin
 from web_poet.rules import ApplyRule
 
 from scrapy_poet import DummyResponse, HttpResponseProvider, PageObjectInputProvider
 from scrapy_poet.injection import (
-    AnnotatedResult,
     Injector,
     check_all_providers_are_callable,
     get_injector_for_testing,