Handle custom attributes received in the API response.

Author: wRAR

scrapy_zyte_api/providers.py

@@ -327,6 +327,9 @@ async def __call__(  # noqa: C901
                 continue
             assert issubclass(cls_stripped, Item)
             result = cls_stripped.from_dict(api_response.raw_api_response[kw])  # type: ignore[attr-defined]
+            custom_attrs = api_response.raw_api_response.get("customAttributes")
+            if custom_attrs:
+                result.customAttributes = custom_attrs
             if is_typing_annotated(cls):
                 result = AnnotatedInstance(result, cls.__metadata__)  # type: ignore[attr-defined]
             results.append(result)

setup.py

@@ -33,7 +33,8 @@ def get_version():
             "andi>=0.6.0",
             "scrapy-poet>=0.22.3",
             "web-poet>=0.17.0",
-            "zyte-common-items>=0.20.0",
+            # https://github.com/zytedata/zyte-common-items/pull/100
+            "zyte-common-items @ git+https://github.com/scrapinghub/zyte-common-items.git@custom-attrs",
         ]
     },
     classifiers=[

tests/mockserver.py

@@ -230,6 +230,12 @@ def render_POST(self, request):
                     "name"
                 ] += f" (country {request_data['geolocation']})"
 
+            if "customAttributes" in request_data:
+                response_data["customAttributes"] = {
+                    "attr1": "foo",
+                    "attr2": 42,
+                }
+
         return json.dumps(response_data).encode()
 
 

tests/test_providers.py

@@ -394,6 +394,33 @@ def parse_(self, response: DummyResponse, page: GeoProductPage):  # type: ignore
     assert "Geolocation dependencies must be annotated" in caplog.text
 
 
+@ensureDeferred
+async def test_provider_custom_attrs(mockserver):
+    settings = create_scrapy_settings()
+    settings["ZYTE_API_URL"] = mockserver.urljoin("/")
+    settings["SCRAPY_POET_PROVIDERS"] = {ZyteApiProvider: 0}
+    settings["ZYTE_API_PROVIDER_PARAMS"] = {
+        "customAttributes": {
+            "attr1": {"type": "string", "description": "descr1"},
+            "attr2": {"type": "number", "description": "descr2"},
+        }
+    }
+
+    item, url, _ = await crawl_single_item(ZyteAPISpider, HtmlResource, settings)
+    assert item["product"] == Product.from_dict(
+        dict(
+            url=url,
+            name="Product name",
+            price="10",
+            currency="USD",
+            customAttributes={
+                "attr1": "foo",
+                "attr2": 42,
+            },
+        )
+    )
+
+
 class RecordingHandler(ScrapyZyteAPIDownloadHandler):
     """Subclasses the original handler in order to record the Zyte API parameters
     used for each downloading request.