From d322e1488c5c92a6943d81d3b66d6fd5e03b4d2a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 13 Jun 2024 19:13:07 +0500 Subject: [PATCH 01/10] Initial DynamicDeps support. --- scrapy_poet/__init__.py | 1 + scrapy_poet/injection.py | 45 +++++++++++++++++++++++++++++++++++----- tests/test_injection.py | 36 +++++++++++++++++++++++++++++--- 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/scrapy_poet/__init__.py b/scrapy_poet/__init__.py index 47756b50..393fd303 100644 --- a/scrapy_poet/__init__.py +++ b/scrapy_poet/__init__.py @@ -1,5 +1,6 @@ from .api import DummyResponse, callback_for from .downloadermiddlewares import DownloaderStatsMiddleware, InjectionMiddleware +from .injection import DynamicDeps from .page_input_providers import HttpResponseProvider, PageObjectInputProvider from .spidermiddlewares import RetryMiddleware from ._request_fingerprinter import ScrapyPoetRequestFingerprinter diff --git a/scrapy_poet/injection.py b/scrapy_poet/injection.py index 02f5f7ab..6988155e 100644 --- a/scrapy_poet/injection.py +++ b/scrapy_poet/injection.py @@ -4,6 +4,7 @@ import os import pprint import warnings +from collections import UserDict from typing import ( Any, Callable, @@ -54,6 +55,10 @@ class _UNDEFINED: pass +class DynamicDeps(UserDict): + pass + + class Injector: """ Keep all the logic required to do dependency injection in Scrapy callbacks. @@ -170,20 +175,48 @@ def build_plan(self, request: Request) -> andi.Plan: # Callable[[Callable], Optional[Callable]] but the registry # returns the typing for ``dict.get()`` method. overrides=self.registry.overrides_for(request.url).get, # type: ignore[arg-type] - custom_builder_fn=self._get_item_builder(request), + custom_builder_fn=self._get_custom_builder(request), ) - def _get_item_builder( + def _get_custom_builder( self, request: Request ) -> Callable[[Callable], Optional[Callable]]: """Return a function suitable for passing as ``custom_builder_fn`` to ``andi.plan``. The returned function can map an item to a factory for that item based - on the registry. + on the registry and also supports filling :class:`.DynamicDeps`. """ @functools.lru_cache(maxsize=None) # to minimize the registry queries def mapping_fn(item_cls: Callable) -> Optional[Callable]: + # building DynamicDeps + if item_cls is DynamicDeps: + dynamic_types = request.meta.get("inject", []) + if not dynamic_types: + return lambda: {} + + # inspired by dataclasses._create_fn() + args = [ + f"{type_.__name__}_arg: {type_.__name__}" for type_ in dynamic_types + ] + args_str = ", ".join(args) + result_args = [ + f"{type_.__name__}: {type_.__name__}_arg" for type_ in dynamic_types + ] + result_args_str = ", ".join(result_args) + ns = {type_.__name__: type_ for type_ in dynamic_types} + create_args = ns.keys() + create_args_str = ", ".join(create_args) + txt = ( + f"def __create_fn__({create_args_str}):\n" + f" def dynamic_deps_factory({args_str}) -> DynamicDeps:\n" + f" return DynamicDeps({{{result_args_str}}})\n" + f" return dynamic_deps_factory" + ) + exec(txt, globals(), ns) + return ns["__create_fn__"](*dynamic_types) + + # building items from pages page_object_cls: Optional[Type[ItemPage]] = self.registry.page_cls_for_item( request.url, cast(type, item_cls) ) @@ -480,7 +513,9 @@ class MySpider(Spider): return Injector(crawler, registry=registry) -def get_response_for_testing(callback: Callable) -> Response: +def get_response_for_testing( + callback: Callable, meta: Optional[Dict[str, Any]] = None +) -> Response: """ Return a :class:`scrapy.http.Response` with fake content with the configured callback. It is useful for testing providers. @@ -501,6 +536,6 @@ def get_response_for_testing(callback: Callable) -> Response: """.encode( "utf-8" ) - request = Request(url, callback=callback) + request = Request(url, callback=callback, meta=meta) response = Response(url, 200, None, html, request=request) return response diff --git a/tests/test_injection.py b/tests/test_injection.py index 8b6e27d4..9fcafec1 100644 --- a/tests/test_injection.py +++ b/tests/test_injection.py @@ -1,6 +1,6 @@ import shutil import sys -from typing import Any, Callable, Dict, Generator +from typing import Any, Callable, Dict, Generator, Optional import attr import parsel @@ -16,7 +16,12 @@ from web_poet.mixins import ResponseShortcutsMixin from web_poet.rules import ApplyRule -from scrapy_poet import DummyResponse, HttpResponseProvider, PageObjectInputProvider +from scrapy_poet import ( + DummyResponse, + DynamicDeps, + HttpResponseProvider, + PageObjectInputProvider, +) from scrapy_poet.injection import ( Injector, check_all_providers_are_callable, @@ -293,8 +298,9 @@ def _assert_instances( callback: Callable, expected_instances: Dict[type, Any], expected_kwargs: Dict[str, Any], + reqmeta: Optional[Dict[str, Any]] = None, ) -> Generator[Any, Any, None]: - response = get_response_for_testing(callback) + response = get_response_for_testing(callback, meta=reqmeta) request = response.request plan = injector.build_plan(response.request) @@ -535,6 +541,30 @@ def callback( # not injected at all. assert set(kwargs.keys()) == {"expensive", "item"} + @inlineCallbacks + def test_dynamic_deps(self): + def callback(dd: DynamicDeps): + pass + + provider = get_provider({Cls1, Cls2}) + injector = get_injector_for_testing({provider: 1}) + + expected_instances = { + DynamicDeps: DynamicDeps({Cls1: Cls1(), Cls2: Cls2()}), + Cls1: Cls1(), + Cls2: Cls2(), + } + expected_kwargs = { + "dd": DynamicDeps({Cls1: Cls1(), Cls2: Cls2()}), + } + yield self._assert_instances( + injector, + callback, + expected_instances, + expected_kwargs, + reqmeta={"inject": [Cls1, Cls2]}, + ) + class Html(Injectable): url = "http://example.com" From fff751c46b1f77b11338fc1755e87b72a5d55c5a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 20 Jun 2024 20:35:12 +0500 Subject: [PATCH 02/10] More DynamicDeps tests. --- tests/test_injection.py | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_injection.py b/tests/test_injection.py index 9fcafec1..91411db4 100644 --- a/tests/test_injection.py +++ b/tests/test_injection.py @@ -565,6 +565,52 @@ def callback(dd: DynamicDeps): reqmeta={"inject": [Cls1, Cls2]}, ) + @inlineCallbacks + def test_dynamic_deps_mix(self): + def callback(c1: Cls1, dd: DynamicDeps): + pass + + provider = get_provider({Cls1, Cls2}) + injector = get_injector_for_testing({provider: 1}) + + expected_instances = { + DynamicDeps: DynamicDeps({Cls1: Cls1(), Cls2: Cls2()}), + Cls1: Cls1(), + Cls2: Cls2(), + } + expected_kwargs = { + "c1": Cls1(), + "dd": DynamicDeps({Cls1: Cls1(), Cls2: Cls2()}), + } + yield self._assert_instances( + injector, + callback, + expected_instances, + expected_kwargs, + reqmeta={"inject": [Cls1, Cls2]}, + ) + + @inlineCallbacks + def test_dynamic_deps_no_meta(self): + def callback(dd: DynamicDeps): + pass + + provider = get_provider({Cls1, Cls2}) + injector = get_injector_for_testing({provider: 1}) + + expected_instances = { + DynamicDeps: DynamicDeps(), + } + expected_kwargs = { + "dd": DynamicDeps(), + } + yield self._assert_instances( + injector, + callback, + expected_instances, + expected_kwargs, + ) + class Html(Injectable): url = "http://example.com" From a9dba26768b2c296d74b1753442aa29ea0386dcd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 25 Jun 2024 20:35:48 +0500 Subject: [PATCH 03/10] Add docs for DynamicDeps. --- docs/dynamic-deps.rst | 54 ++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + scrapy_poet/injection.py | 6 +++++ 3 files changed, 61 insertions(+) create mode 100644 docs/dynamic-deps.rst diff --git a/docs/dynamic-deps.rst b/docs/dynamic-deps.rst new file mode 100644 index 00000000..ad6db135 --- /dev/null +++ b/docs/dynamic-deps.rst @@ -0,0 +1,54 @@ +.. _dynamic-deps: + +==================== +Dynamic dependencies +==================== + +Normally the dependencies for a callback are specified statically, as type +hints for its arguments: + +.. code-block:: python + + import scrapy + + + class BooksSpider(scrapy.Spider): + ... + + def start_requests(self): + yield scrapy.Request("http://books.toscrape.com/", self.parse_book) + + + def parse_book(self, response, book_page: BookPage, other_dep: OtherDep): + ... + +In some cases some or all of the dependencies need to be specified dynamically +instead, e.g. because they need to be different for different pages using the +same callback. You can use :class:`scrapy_poet.DynamicDeps +` for this. If you add a callback argument +with this type you can pass a list of additional dependency types in the +request meta dictionary using the "inject" key: + +.. code-block:: python + + import scrapy + + + class BooksSpider(scrapy.Spider): + ... + + def start_requests(self): + yield scrapy.Request( + "http://books.toscrape.com/", + self.parse_book, + meta={"inject": [OtherDep]}, + ) + + + def parse_book(self, response, book_page: BookPage, dynamic: DynamicDeps): + other_dep = dynamic[OtherDep] + ... + +The types passed this way will be used in the dependency resolution as usual, +and the created instances will be available in the +:class:`scrapy_poet.DynamicDeps ` instance. diff --git a/docs/index.rst b/docs/index.rst index f21cb4ca..1b022b83 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,6 +44,7 @@ To get started, see :ref:`intro-install` and :ref:`intro-tutorial`. :maxdepth: 1 rules-from-web-poet + dynamic-deps stats providers testing diff --git a/scrapy_poet/injection.py b/scrapy_poet/injection.py index 6988155e..37458e96 100644 --- a/scrapy_poet/injection.py +++ b/scrapy_poet/injection.py @@ -56,6 +56,12 @@ class _UNDEFINED: class DynamicDeps(UserDict): + """A container for dynamic dependencies provided via the ``"inject"`` request meta key. + + The dynamic dependency instances are available at the run time as dict + values with keys being dependency types. + """ + pass From 6f7e5d1d7d22f7369b36ec61bb74f79c35c08ebc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 27 Jun 2024 17:14:50 +0500 Subject: [PATCH 04/10] Extract and test _get_dynamic_deps_factory(). --- scrapy_poet/injection.py | 53 ++++++++++++++++++++++++---------------- tests/test_injection.py | 14 +++++++++++ 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/scrapy_poet/injection.py b/scrapy_poet/injection.py index 37458e96..bf3d24de 100644 --- a/scrapy_poet/injection.py +++ b/scrapy_poet/injection.py @@ -200,27 +200,7 @@ def mapping_fn(item_cls: Callable) -> Optional[Callable]: dynamic_types = request.meta.get("inject", []) if not dynamic_types: return lambda: {} - - # inspired by dataclasses._create_fn() - args = [ - f"{type_.__name__}_arg: {type_.__name__}" for type_ in dynamic_types - ] - args_str = ", ".join(args) - result_args = [ - f"{type_.__name__}: {type_.__name__}_arg" for type_ in dynamic_types - ] - result_args_str = ", ".join(result_args) - ns = {type_.__name__: type_ for type_ in dynamic_types} - create_args = ns.keys() - create_args_str = ", ".join(create_args) - txt = ( - f"def __create_fn__({create_args_str}):\n" - f" def dynamic_deps_factory({args_str}) -> DynamicDeps:\n" - f" return DynamicDeps({{{result_args_str}}})\n" - f" return dynamic_deps_factory" - ) - exec(txt, globals(), ns) - return ns["__create_fn__"](*dynamic_types) + return self._get_dynamic_deps_factory(dynamic_types) # building items from pages page_object_cls: Optional[Type[ItemPage]] = self.registry.page_cls_for_item( @@ -236,6 +216,37 @@ async def item_factory(page: page_object_cls) -> item_cls: # type: ignore[valid return mapping_fn + @staticmethod + def _get_dynamic_deps_factory( + dynamic_types: List[type], + ) -> Callable[..., DynamicDeps]: + """Return a function that creates a :class:`.DynamicDeps` instance from its args. + + It takes instances of types from ``dynamic_types`` as args and returns + a :class:`.DynamicDeps` instance where keys are types and values are + corresponding args. It has correct type hints so that it can be used as + an ``andi`` custom builder. + """ + + # inspired by dataclasses._create_fn() + args = [f"{type_.__name__}_arg: {type_.__name__}" for type_ in dynamic_types] + args_str = ", ".join(args) + result_args = [ + f"{type_.__name__}: {type_.__name__}_arg" for type_ in dynamic_types + ] + result_args_str = ", ".join(result_args) + ns = {type_.__name__: type_ for type_ in dynamic_types} + create_args = ns.keys() + create_args_str = ", ".join(create_args) + txt = ( + f"def __create_fn__({create_args_str}):\n" + f" def dynamic_deps_factory({args_str}) -> DynamicDeps:\n" + f" return DynamicDeps({{{result_args_str}}})\n" + f" return dynamic_deps_factory" + ) + exec(txt, globals(), ns) + return ns["__create_fn__"](*dynamic_types) + @inlineCallbacks def build_instances( self, diff --git a/tests/test_injection.py b/tests/test_injection.py index 91411db4..27e0dbb1 100644 --- a/tests/test_injection.py +++ b/tests/test_injection.py @@ -2,6 +2,7 @@ import sys from typing import Any, Callable, Dict, Generator, Optional +import andi import attr import parsel import pytest @@ -909,3 +910,16 @@ def callback(response: DummyResponse, arg_price: Price, arg_name: Name): response.request, response, plan ) assert injector.weak_cache.get(response.request) is None + + +def test_dynamic_deps_factory(): + factory = Injector._get_dynamic_deps_factory + fn = factory([int, Cls1]) + args = andi.inspect(fn) + assert args == { + "Cls1_arg": [Cls1], + "int_arg": [int], + } + c = Cls1() + dd = fn(int_arg=42, Cls1_arg=c) + assert dd == {int: 42, Cls1: c} From 9d6dde9cd2c657a42d33c1f82ee4150252339d72 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 27 Jun 2024 17:15:35 +0500 Subject: [PATCH 05/10] Cleanup. --- tests/test_injection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_injection.py b/tests/test_injection.py index 27e0dbb1..1f66c846 100644 --- a/tests/test_injection.py +++ b/tests/test_injection.py @@ -913,8 +913,7 @@ def callback(response: DummyResponse, arg_price: Price, arg_name: Name): def test_dynamic_deps_factory(): - factory = Injector._get_dynamic_deps_factory - fn = factory([int, Cls1]) + fn = Injector._get_dynamic_deps_factory([int, Cls1]) args = andi.inspect(fn) assert args == { "Cls1_arg": [Cls1], From efdbed13344b3920f04a106fbe1178d52ef77dfd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Jul 2024 13:00:35 +0500 Subject: [PATCH 06/10] Refactor _get_dynamic_deps_factory. --- scrapy_poet/injection.py | 36 ++++++++++++++++++++---------------- tests/test_injection.py | 11 +++++++++++ 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/scrapy_poet/injection.py b/scrapy_poet/injection.py index bf3d24de..4f3005be 100644 --- a/scrapy_poet/injection.py +++ b/scrapy_poet/injection.py @@ -9,6 +9,7 @@ Any, Callable, Dict, + Iterable, List, Mapping, Optional, @@ -216,6 +217,24 @@ async def item_factory(page: page_object_cls) -> item_cls: # type: ignore[valid return mapping_fn + @staticmethod + def _get_dynamic_deps_factory_text( + type_names: Iterable[str], + ) -> str: + # inspired by Python 3.11 dataclasses._create_fn() + # https://github.com/python/cpython/blob/v3.11.9/Lib/dataclasses.py#L413 + args = [f"{name}_arg: {name}" for name in type_names] + args_str = ", ".join(args) + result_args = [f"{name}: {name}_arg" for name in type_names] + result_args_str = ", ".join(result_args) + create_args_str = ", ".join(type_names) + return ( + f"def __create_fn__({create_args_str}):\n" + f" def dynamic_deps_factory({args_str}) -> DynamicDeps:\n" + f" return DynamicDeps({{{result_args_str}}})\n" + f" return dynamic_deps_factory" + ) + @staticmethod def _get_dynamic_deps_factory( dynamic_types: List[type], @@ -227,23 +246,8 @@ def _get_dynamic_deps_factory( corresponding args. It has correct type hints so that it can be used as an ``andi`` custom builder. """ - - # inspired by dataclasses._create_fn() - args = [f"{type_.__name__}_arg: {type_.__name__}" for type_ in dynamic_types] - args_str = ", ".join(args) - result_args = [ - f"{type_.__name__}: {type_.__name__}_arg" for type_ in dynamic_types - ] - result_args_str = ", ".join(result_args) ns = {type_.__name__: type_ for type_ in dynamic_types} - create_args = ns.keys() - create_args_str = ", ".join(create_args) - txt = ( - f"def __create_fn__({create_args_str}):\n" - f" def dynamic_deps_factory({args_str}) -> DynamicDeps:\n" - f" return DynamicDeps({{{result_args_str}}})\n" - f" return dynamic_deps_factory" - ) + txt = Injector._get_dynamic_deps_factory_text(ns.keys()) exec(txt, globals(), ns) return ns["__create_fn__"](*dynamic_types) diff --git a/tests/test_injection.py b/tests/test_injection.py index 1f66c846..094683ab 100644 --- a/tests/test_injection.py +++ b/tests/test_injection.py @@ -912,6 +912,17 @@ def callback(response: DummyResponse, arg_price: Price, arg_name: Name): assert injector.weak_cache.get(response.request) is None +def test_dynamic_deps_factory_text(): + txt = Injector._get_dynamic_deps_factory_text(["int", "Cls1"]) + assert ( + txt + == """def __create_fn__(int, Cls1): + def dynamic_deps_factory(int_arg: int, Cls1_arg: Cls1) -> DynamicDeps: + return DynamicDeps({int: int_arg, Cls1: Cls1_arg}) + return dynamic_deps_factory""" + ) + + def test_dynamic_deps_factory(): fn = Injector._get_dynamic_deps_factory([int, Cls1]) args = andi.inspect(fn) From f6461cc21ab221b27fa8f7bdf53ecf36e02fa09f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Jul 2024 20:48:04 +0500 Subject: [PATCH 07/10] More tests for dynamic deps. --- tests/test_injection.py | 50 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/test_injection.py b/tests/test_injection.py index 094683ab..535439f6 100644 --- a/tests/test_injection.py +++ b/tests/test_injection.py @@ -612,6 +612,56 @@ def callback(dd: DynamicDeps): expected_kwargs, ) + @inlineCallbacks + def test_dynamic_deps_page(self): + def callback(dd: DynamicDeps): + pass + + injector = get_injector_for_testing({}) + + response = get_response_for_testing(callback, meta={"inject": [PricePO]}) + request = response.request + + plan = injector.build_plan(response.request) + kwargs = yield from injector.build_callback_dependencies(request, response) + kwargs_types = {key: type(value) for key, value in kwargs.items()} + assert kwargs_types == { + "dd": DynamicDeps, + } + dd_types = {key: type(value) for key, value in kwargs["dd"].items()} + assert dd_types == { + PricePO: PricePO, + } + + instances = yield from injector.build_instances(request, response, plan) + assert set(instances) == {Html, PricePO, DynamicDeps} + + @inlineCallbacks + def test_dynamic_deps_item(self): + def callback(dd: DynamicDeps): + pass + + rules = [ApplyRule(Patterns(include=()), use=TestItemPage, to_return=TestItem)] + registry = RulesRegistry(rules=rules) + injector = get_injector_for_testing({}, registry=registry) + + response = get_response_for_testing(callback, meta={"inject": [TestItem]}) + request = response.request + + plan = injector.build_plan(response.request) + kwargs = yield from injector.build_callback_dependencies(request, response) + kwargs_types = {key: type(value) for key, value in kwargs.items()} + assert kwargs_types == { + "dd": DynamicDeps, + } + dd_types = {key: type(value) for key, value in kwargs["dd"].items()} + assert dd_types == { + TestItem: TestItem, + } + + instances = yield from injector.build_instances(request, response, plan) + assert set(instances) == {TestItemPage, TestItem, DynamicDeps} + class Html(Injectable): url = "http://example.com" From 1b9c00a1eaf0e5d6a70fb891eac7f512dee9e4e4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 12 Jul 2024 18:51:37 +0500 Subject: [PATCH 08/10] Small fixes. --- docs/dynamic-deps.rst | 10 +++++----- scrapy_poet/injection.py | 11 +++++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/dynamic-deps.rst b/docs/dynamic-deps.rst index ad6db135..242246d8 100644 --- a/docs/dynamic-deps.rst +++ b/docs/dynamic-deps.rst @@ -23,8 +23,8 @@ hints for its arguments: ... In some cases some or all of the dependencies need to be specified dynamically -instead, e.g. because they need to be different for different pages using the -same callback. You can use :class:`scrapy_poet.DynamicDeps +instead, e.g. because they need to be different for different requests using +the same callback. You can use :class:`scrapy_poet.DynamicDeps ` for this. If you add a callback argument with this type you can pass a list of additional dependency types in the request meta dictionary using the "inject" key: @@ -49,6 +49,6 @@ request meta dictionary using the "inject" key: other_dep = dynamic[OtherDep] ... -The types passed this way will be used in the dependency resolution as usual, -and the created instances will be available in the -:class:`scrapy_poet.DynamicDeps ` instance. +The types passed this way are used in the dependency resolution as usual, with +the created instances available in the :class:`scrapy_poet.DynamicDeps +` instance. diff --git a/scrapy_poet/injection.py b/scrapy_poet/injection.py index 4f3005be..3414741a 100644 --- a/scrapy_poet/injection.py +++ b/scrapy_poet/injection.py @@ -4,7 +4,6 @@ import os import pprint import warnings -from collections import UserDict from typing import ( Any, Callable, @@ -56,7 +55,7 @@ class _UNDEFINED: pass -class DynamicDeps(UserDict): +class DynamicDeps(dict): """A container for dynamic dependencies provided via the ``"inject"`` request meta key. The dynamic dependency instances are available at the run time as dict @@ -195,9 +194,9 @@ def _get_custom_builder( """ @functools.lru_cache(maxsize=None) # to minimize the registry queries - def mapping_fn(item_cls: Callable) -> Optional[Callable]: + def mapping_fn(dep_cls: Callable) -> Optional[Callable]: # building DynamicDeps - if item_cls is DynamicDeps: + if dep_cls is DynamicDeps: dynamic_types = request.meta.get("inject", []) if not dynamic_types: return lambda: {} @@ -205,12 +204,12 @@ def mapping_fn(item_cls: Callable) -> Optional[Callable]: # building items from pages page_object_cls: Optional[Type[ItemPage]] = self.registry.page_cls_for_item( - request.url, cast(type, item_cls) + request.url, cast(type, dep_cls) ) if not page_object_cls: return None - async def item_factory(page: page_object_cls) -> item_cls: # type: ignore[valid-type] + async def item_factory(page: page_object_cls) -> dep_cls: # type: ignore[valid-type] return await page.to_item() # type: ignore[attr-defined] return item_factory From 887e2fe6955be653ed6fc5eb47b7b0eeaf1c4801 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 17 Jul 2024 20:51:11 +0500 Subject: [PATCH 09/10] Improve test_dynamic_deps_mix(). --- tests/test_injection.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/test_injection.py b/tests/test_injection.py index 535439f6..fe68e5a9 100644 --- a/tests/test_injection.py +++ b/tests/test_injection.py @@ -574,22 +574,25 @@ def callback(c1: Cls1, dd: DynamicDeps): provider = get_provider({Cls1, Cls2}) injector = get_injector_for_testing({provider: 1}) - expected_instances = { + response = get_response_for_testing(callback, meta={"inject": [Cls1, Cls2]}) + request = response.request + + plan = injector.build_plan(response.request) + instances = yield from injector.build_instances(request, response, plan) + assert instances == { DynamicDeps: DynamicDeps({Cls1: Cls1(), Cls2: Cls2()}), Cls1: Cls1(), Cls2: Cls2(), } - expected_kwargs = { + assert instances[Cls1] is instances[DynamicDeps][Cls1] + assert instances[Cls2] is instances[DynamicDeps][Cls2] + + kwargs = yield from injector.build_callback_dependencies(request, response) + assert kwargs == { "c1": Cls1(), "dd": DynamicDeps({Cls1: Cls1(), Cls2: Cls2()}), } - yield self._assert_instances( - injector, - callback, - expected_instances, - expected_kwargs, - reqmeta={"inject": [Cls1, Cls2]}, - ) + assert kwargs["c1"] is kwargs["dd"][Cls1] @inlineCallbacks def test_dynamic_deps_no_meta(self): From 9016a7b613e4d02164ac9b564c7eea6108900d6f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 17 Jul 2024 21:41:08 +0500 Subject: [PATCH 10/10] Improve DynamicDeps docs. --- docs/dynamic-deps.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/dynamic-deps.rst b/docs/dynamic-deps.rst index 242246d8..529cf9f1 100644 --- a/docs/dynamic-deps.rst +++ b/docs/dynamic-deps.rst @@ -46,9 +46,15 @@ request meta dictionary using the "inject" key: def parse_book(self, response, book_page: BookPage, dynamic: DynamicDeps): + # access the dynamic dependency values by their type: other_dep = dynamic[OtherDep] ... + # or get them and their types at the run time: + for dep_type, dep in dynamic.items(): + if dep_type is OtherDep: + ... The types passed this way are used in the dependency resolution as usual, with the created instances available in the :class:`scrapy_poet.DynamicDeps -` instance. +` instance, which is a dictionary with +dependency types as keys and their instances as values.