Fail on SPM + transparent mode

.github/workflows/test.yml

@@ -39,6 +39,11 @@ jobs:
         - python-version: '3.11'
           toxenv: provider
 
+        - python-version: '3.7'
+          toxenv: pinned-extra
+        - python-version: '3.11'
+          toxenv: extra
+
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}

scrapy_zyte_api/_downloader_middleware.py

@@ -1,5 +1,6 @@
 import logging
 
+from scrapy import signals
 from scrapy.exceptions import IgnoreRequest
 
 from ._params import _ParamParser
@@ -26,6 +27,58 @@ def __init__(self, crawler) -> None:
                 f"reached."
             )
 
+        crawler.signals.connect(self.open_spider, signal=signals.spider_opened)
+
+    def _get_spm_mw(self):
+        spm_mw_classes = []
+
+        try:
+            from scrapy_crawlera import CrawleraMiddleware
+        except ImportError:
+            pass
+        else:
+            spm_mw_classes.append(CrawleraMiddleware)
+
+        try:
+            from scrapy_zyte_smartproxy import ZyteSmartProxyMiddleware
+        except ImportError:
+            pass
+        else:
+            spm_mw_classes.append(ZyteSmartProxyMiddleware)
+
+        middlewares = self._crawler.engine.downloader.middleware.middlewares
+        for middleware in middlewares:
+            if isinstance(middleware, tuple(spm_mw_classes)):
+                return middleware
+        return None
+
+    def open_spider(self, spider):
+        settings = self._crawler.settings
+        in_transparent_mode = settings.getbool("ZYTE_API_TRANSPARENT_MODE", False)
+        spm_mw = self._get_spm_mw()
+        spm_is_enabled = spm_mw and spm_mw.is_enabled(spider)
+        if not in_transparent_mode or not spm_is_enabled:
+            return
+        logger.error(
+            "Both scrapy-zyte-smartproxy and the transparent mode of "
+            "scrapy-zyte-api are enabled. You should only enable one of "
+            "those at the same time.\n"
+            "\n"
+            "To combine requests that use scrapy-zyte-api and requests "
+            "that use scrapy-zyte-smartproxy in the same spider:\n"
+            "\n"
+            "1. Leave scrapy-zyte-smartproxy enabled.\n"
+            "2. Disable the transparent mode of scrapy-zyte-api.\n"
+            "3. To send a specific request through Zyte API, use "
+            "request.meta to set dont_proxy to True and zyte_api_automap "
+            "either to True or to a dictionary of extra request fields."
+        )
+        from twisted.internet import reactor
+
+        reactor.callLater(
+            0, self._crawler.engine.close_spider, spider, "plugin_conflict"
+        )
+
     def process_request(self, request, spider):
         if self._param_parser.parse(request) is None:
             return

tests/__init__.py

@@ -17,6 +17,9 @@
         "http": "scrapy_zyte_api.handler.ScrapyZyteAPIDownloadHandler",
         "https": "scrapy_zyte_api.handler.ScrapyZyteAPIDownloadHandler",
     },
+    "DOWNLOADER_MIDDLEWARES": {
+        "scrapy_zyte_api.ScrapyZyteAPIDownloaderMiddleware": 1000,
+    },
     "REQUEST_FINGERPRINTER_CLASS": "scrapy_zyte_api.ScrapyZyteAPIRequestFingerprinter",
     "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",  # Silence deprecation warning
     "ZYTE_API_KEY": _API_KEY,

tests/test_downloader_middleware.py

@@ -10,6 +10,10 @@
 from .mockserver import DelayedResource, MockServer
 
 
+class NamedSpider(Spider):
+    name = "named"
+
+
 @ensureDeferred
 async def test_autothrottle_handling():
     crawler = get_crawler()
@@ -132,3 +136,73 @@ def parse(self, response):
         )
         > 0
     )
+
+
+@ensureDeferred
+async def test_spm_conflict_smartproxy_setting(caplog):
+    settings = {
+        "ZYTE_API_TRANSPARENT_MODE": True,
+        "ZYTE_SMARTPROXY_APIKEY": "foo",
+        "ZYTE_SMARTPROXY_ENABLED": True,
+        **SETTINGS,
+    }
+    mws = dict(settings["DOWNLOADER_MIDDLEWARES"])
+    mws["scrapy_zyte_smartproxy.ZyteSmartProxyMiddleware"] = 610
+    settings["DOWNLOADER_MIDDLEWARES"] = mws
+    crawler = get_crawler(NamedSpider, settings_dict=settings)
+    await crawler.crawl()
+    assert crawler.stats.get_value("finish_reason") == "plugin_conflict"
+
+
+@ensureDeferred
+async def test_spm_conflict_smartproxy_spider_attr():
+    class SPMSpider(Spider):
+        name = "spm_spider"
+        zyte_smartproxy_enabled = True
+
+    settings = {
+        "ZYTE_API_TRANSPARENT_MODE": True,
+        "ZYTE_SMARTPROXY_APIKEY": "foo",
+        **SETTINGS,
+    }
+    mws = dict(settings["DOWNLOADER_MIDDLEWARES"])
+    mws["scrapy_zyte_smartproxy.ZyteSmartProxyMiddleware"] = 610
+    settings["DOWNLOADER_MIDDLEWARES"] = mws
+    crawler = get_crawler(SPMSpider, settings_dict=settings)
+    await crawler.crawl()
+    assert crawler.stats.get_value("finish_reason") == "plugin_conflict"
+
+
+@ensureDeferred
+async def test_spm_conflict_crawlera_setting():
+    settings = {
+        "ZYTE_API_TRANSPARENT_MODE": True,
+        "CRAWLERA_APIKEY": "foo",
+        "CRAWLERA_ENABLED": True,
+        **SETTINGS,
+    }
+    mws = dict(settings["DOWNLOADER_MIDDLEWARES"])
+    mws["scrapy_crawlera.CrawleraMiddleware"] = 610
+    settings["DOWNLOADER_MIDDLEWARES"] = mws
+    crawler = get_crawler(NamedSpider, settings_dict=settings)
+    await crawler.crawl()
+    assert crawler.stats.get_value("finish_reason") == "plugin_conflict"
+
+
+@ensureDeferred
+async def test_spm_conflict_crawlera_spider_attr():
+    class CrawleraSpider(Spider):
+        name = "crawlera_spider"
+        crawlera_enabled = True
+
+    settings = {
+        "ZYTE_API_TRANSPARENT_MODE": True,
+        "CRAWLERA_APIKEY": "foo",
+        **SETTINGS,
+    }
+    mws = dict(settings["DOWNLOADER_MIDDLEWARES"])
+    mws["scrapy_crawlera.CrawleraMiddleware"] = 610
+    settings["DOWNLOADER_MIDDLEWARES"] = mws
+    crawler = get_crawler(CrawleraSpider, settings_dict=settings)
+    await crawler.crawl()
+    assert crawler.stats.get_value("finish_reason") == "plugin_conflict"

tox.ini

@@ -91,6 +91,20 @@ deps =
     web-poet==0.13.0
     zyte-common-items==0.7.0
 
+[testenv:pinned-extra]
+basepython=python3.7
+deps =
+    {[testenv:pinned-scrapy-2x0]deps}
+    scrapy-crawlera==1.1.0
+    scrapy-zyte-smartproxy==2.0.0
+
+[testenv:extra]
+basepython=python3.11
+deps =
+    {[testenv]deps}
+    scrapy-crawlera
+    scrapy-zyte-smartproxy
+
 [testenv:mypy]
 deps =
     mypy==1.4.1