Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Track in stats which fields from Zyte API automatic extraction are not overridden #202

Merged
merged 11 commits into from
Jul 25, 2024
Prev Previous commit
Next Next commit
Solve issues reported by CI
  • Loading branch information
Gallaecio committed Jun 11, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 3d79ac354a5ce065719ece18c19cd2ea0e7d4361
6 changes: 3 additions & 3 deletions scrapy_zyte_api/providers.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Type
from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Type, cast

import attrs
from andi.typeutils import is_typing_annotated, strip_annotated
@@ -54,7 +54,7 @@
ArticleNavigation: "articleNavigation",
JobPosting: "jobPosting",
}
_AUTO_PAGES: Set[Type] = {
_AUTO_PAGES: Set[type] = {
AutoArticlePage,
AutoArticleListPage,
AutoArticleNavigationPage,
@@ -151,7 +151,7 @@ async def __call__( # noqa: C901
http_response = None
screenshot_requested = Screenshot in to_provide
for cls in list(to_provide):
self._track_auto_fields(crawler, request, cls)
self._track_auto_fields(crawler, request, cast(type, cls))
item = self.injector.weak_cache.get(request, {}).get(cls)
if item:
results.append(item)
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
@@ -33,7 +33,7 @@ def get_version():
"andi>=0.6.0",
"scrapy-poet>=0.22.3",
"web-poet>=0.17.0",
"zyte-common-items>=0.8.0",
"zyte-common-items>=0.16.0",
]
},
classifiers=[
22 changes: 14 additions & 8 deletions tests/test_providers.py
Original file line number Diff line number Diff line change
@@ -1020,7 +1020,7 @@ def parse_(self, response: DummyResponse, page: ActionProductPage): # type: ign


def test_auto_pages_set():
assert set(_ITEM_KEYWORDS) == {get_item_cls(cls) for cls in _AUTO_PAGES}
assert set(_ITEM_KEYWORDS) == {get_item_cls(cls) for cls in _AUTO_PAGES} # type: ignore[call-overload]


@ensureDeferred
@@ -1108,7 +1108,8 @@ def parse(self, response: DummyResponse, product: Product):
),
}

default_registry.__init__() # Reset rules
# Reset rules
default_registry.__init__() # type: ignore[misc]


@ensureDeferred
@@ -1253,7 +1254,8 @@ def parse(self, response: DummyResponse, product: Product):
"scrapy-zyte-api/auto_fields/tests.test_providers.test_auto_field_stat_full_override.<locals>.MyProductPage": "",
}

default_registry.__init__() # Reset rules
# Reset rules
default_registry.__init__() # type: ignore[misc]


@ensureDeferred
@@ -1269,7 +1271,7 @@ def start_requests(self):
yield Request(self.url, callback=self.parse)

def parse(self, response: DummyResponse, product: Product):
product["name"] = "foo"
product.name = "foo"
yield product

settings = create_scrapy_settings()
@@ -1293,7 +1295,8 @@ def parse(self, response: DummyResponse, product: Product):
),
}

default_registry.__init__() # Reset rules
# Reset rules
default_registry.__init__() # type: ignore[misc]


@ensureDeferred
@@ -1345,7 +1348,8 @@ def parse(self, response: DummyResponse, page: MyProductPage):
),
}

default_registry.__init__() # Reset rules
# Reset rules
default_registry.__init__() # type: ignore[misc]


@ensureDeferred
@@ -1409,7 +1413,8 @@ def parse(self, response: DummyResponse, page: AltProductPage):
),
}

default_registry.__init__() # Reset rules
# Reset rules
default_registry.__init__() # type: ignore[misc]


@ensureDeferred
@@ -1453,4 +1458,5 @@ def parse(self, response: DummyResponse, product: Product):
"scrapy-zyte-api/auto_fields/tests.test_providers.test_auto_field_stat_non_auto_override.<locals>.MyProductPage": "",
}

default_registry.__init__() # Reset rules
# Reset rules
default_registry.__init__() # type: ignore[misc]
2 changes: 1 addition & 1 deletion tox.ini
Original file line number Diff line number Diff line change
@@ -90,7 +90,7 @@ deps =
andi==0.6.0
scrapy-poet==0.22.3
web-poet==0.17.0
zyte-common-items==0.8.0
zyte-common-items==0.16.0

[testenv:pinned-extra]
basepython=python3.8
Loading