diff --git a/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py b/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py
index 259927c37..ca2f5a0e3 100644
--- a/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py
+++ b/napari-hub-commons/src/nhcommons/tests/utils/test_pypi_adapter.py
@@ -137,31 +137,9 @@ class TestPypiAdapter:
def setup_method(self, monkeypatch):
monkeypatch.setattr(requests, "get", self._mocked_requests_get)
- def _generate_html_data(self, plugin_version_list: List[Tuple[str, str]]):
- data = [
- f"""
-
- {plugin[0]}
- {plugin[1]}
-
- """
- for plugin in plugin_version_list
- ]
- return "
".join(data)
-
def _mocked_requests_get(self, *args, **kwargs):
- if args[0] == "https://pypi.org/search/":
- params = kwargs.get("params", {})
- page = params.get("page", 1000)
- if (
- params
- and len(params) == 3
- and params.get("o") == "-created"
- and params.get("c") == "Framework :: napari"
- and page < 3
- ):
- data = plugins()[:2] if page == 1 else plugins()[2:]
- return MockResponse(content=self._generate_html_data(data))
+ if args[0] == "https://api.napari.org/api/plugins":
+ return MockResponse(content=json.dumps({name: version for name, version in plugins()}))
elif args[0] == "https://pypi.org/pypi/napari-demo/json":
return MockResponse(content=valid_pypi_data())
elif args[0] == "https://pypi.org/pypi/default-demo/json":
@@ -171,15 +149,9 @@ def _mocked_requests_get(self, *args, **kwargs):
return MockResponse(status_code=requests.codes.not_found)
def test_get_all_plugins(self):
- self._version_field = "package-snippet__version"
expected = {plugin[0]: plugin[1] for plugin in plugins()}
assert expected == pypi_adapter.get_all_plugins()
- def test_get_all_plugins_invalid_response(self):
- self._version_field = "foo"
- with pytest.raises(ValueError):
- pypi_adapter.get_all_plugins()
-
@pytest.mark.parametrize(
"plugin, version, extra_fields, expected",
[
diff --git a/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py b/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py
index 3a765986b..921b61a00 100644
--- a/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py
+++ b/napari-hub-commons/src/nhcommons/utils/pypi_adapter.py
@@ -11,36 +11,24 @@
_NAME_PATTERN = re.compile('class="package-snippet__name">(.+)')
_VERSION_PATTERN = re.compile('class="package-snippet__version">(.+)')
_BASE_URL = "https://pypi.org"
-_SEARCH_URL = f"/search/"
_PLUGIN_DATA_URL = "/pypi/{plugin}/json"
+_NPE2API_URL = "https://api.napari.org/api"
logger = logging.getLogger(__name__)
def get_all_plugins() -> Dict[str, str]:
"""
- Query pypi to get all plugins.
+ Query npe2api to get all plugins.
+
+ Now we use the npe2api to get the list of plugins, which uses the public BigQuery pypi metadata
+ as a source of truth.
+
+ The previous implementation was broken by anti-scraping changes to PyPI.
:returns: all plugin names and latest version
"""
- logger.info("Getting all napari plugins from PYPI")
- packages = {}
- page = 1
- params = {"o": "-created", "c": "Framework :: napari"}
- while True:
- try:
- params["page"] = page
- response = _get_pypi_response(_SEARCH_URL, params=params)
- html = response.text
- names = _NAME_PATTERN.findall(html)
- versions = _VERSION_PATTERN.findall(html)
- logger.info(f"Count of plugins fetched for page={page} {len(packages)}")
- if len(names) != len(versions):
- raise ValueError("Count of plugin and version don't match")
- for name, version in zip(names, versions):
- packages[name] = version
- page += 1
- except HTTPError:
- break
+ logger.info("Getting all napari plugins from npe2api")
+ packages = get_request(_NPE2API_URL + "/plugins").json()
logger.info(f"Total number of napari plugins fetched={len(packages)}")
return packages