Fix error handling on HTTP 401 (#1904)

* Fix error handling on HTTP 401g

* add test

* fix test

* fix regex for resolve endpoints

Author: Wauplin

src/huggingface_hub/utils/_errors.py

@@ -1,11 +1,27 @@
+import re
 from typing import Optional
 
 from requests import HTTPError, Response
 
-from ..constants import INFERENCE_ENDPOINTS_ENDPOINT
 from ._fixes import JSONDecodeError
 
 
+REPO_API_REGEX = re.compile(
+    r"""
+        # staging or production endpoint
+        ^https://(hub-ci.)?huggingface.co
+        (
+            # on /api/repo_type/repo_id
+            /api/(models|datasets|spaces)/(.+)
+            |
+            # or /repo_id/resolve/revision/...
+            /(.+)/resolve/(.+)
+        )
+    """,
+    flags=re.VERBOSE,
+)
+
+
 class FileMetadataError(OSError):
     """Error triggered when the metadata of a file on the Hub cannot be retrieved (missing ETag or commit_hash).
 
@@ -285,25 +301,12 @@ def hf_raise_for_status(response: Response, endpoint_name: Optional[str] = None)
             )
             raise GatedRepoError(message, response) from e
 
-        elif (
-            response.status_code == 401
-            and response.request.url is not None
-            and "/api/collections" in response.request.url
-        ):
-            # Collection not found. We don't raise a custom error for this.
-            # This prevent from raising a misleading `RepositoryNotFoundError` (see below).
-            pass
-
-        elif (
+        elif error_code == "RepoNotFound" or (
             response.status_code == 401
+            and response.request is not None
             and response.request.url is not None
-            and INFERENCE_ENDPOINTS_ENDPOINT in response.request.url
+            and REPO_API_REGEX.search(response.request.url) is not None
         ):
-            # Not enough permission to list Inference Endpoints from this org. We don't raise a custom error for this.
-            # This prevent from raising a misleading `RepositoryNotFoundError` (see below).
-            pass
-
-        elif error_code == "RepoNotFound" or response.status_code == 401:
             # 401 is misleading as it is returned for:
             #    - private and gated repos if user is not authenticated
             #    - missing repos

tests/test_utils_errors.py

@@ -1,8 +1,10 @@
 import unittest
 
+import pytest
 from requests.models import PreparedRequest, Response
 
 from huggingface_hub.utils._errors import (
+    REPO_API_REGEX,
     BadRequestError,
     EntryNotFoundError,
     HfHubHTTPError,
@@ -23,17 +25,30 @@ def test_hf_raise_for_status_repo_not_found(self) -> None:
         self.assertEqual(context.exception.response.status_code, 404)
         self.assertIn("Request ID: 123", str(context.exception))
 
-    def test_hf_raise_for_status_repo_not_found_without_error_code(self) -> None:
+    def test_hf_raise_for_status_401_repo_url(self) -> None:
         response = Response()
         response.headers = {"X-Request-Id": 123}
         response.status_code = 401
         response.request = PreparedRequest()
+        response.request.url = "https://huggingface.co/api/models/username/reponame"
         with self.assertRaisesRegex(RepositoryNotFoundError, "Repository Not Found") as context:
             hf_raise_for_status(response)
 
         self.assertEqual(context.exception.response.status_code, 401)
         self.assertIn("Request ID: 123", str(context.exception))
 
+    def test_hf_raise_for_status_401_not_repo_url(self) -> None:
+        response = Response()
+        response.headers = {"X-Request-Id": 123}
+        response.status_code = 401
+        response.request = PreparedRequest()
+        response.request.url = "https://huggingface.co/api/collections"
+        with self.assertRaises(HfHubHTTPError) as context:
+            hf_raise_for_status(response)
+
+        self.assertEqual(context.exception.response.status_code, 401)
+        self.assertIn("Request ID: 123", str(context.exception))
+
     def test_hf_raise_for_status_revision_not_found(self) -> None:
         response = Response()
         response.headers = {"X-Error-Code": "RevisionNotFound", "X-Request-Id": 123}
@@ -239,3 +254,44 @@ def test_hf_hub_http_error_init_with_error_message_duplicated_in_header_and_body
             "this is a message\n\nError message duplicated in headers and body.",
         )
         self.assertEqual(error.server_message, "Error message duplicated in headers and body.")
+
+
+@pytest.mark.parametrize(
+    ("url", "should_match"),
+    [
+        # Listing endpoints => False
+        ("https://huggingface.co/api/models", False),
+        ("https://huggingface.co/api/datasets", False),
+        ("https://huggingface.co/api/spaces", False),
+        # Create repo endpoint => False
+        ("https://huggingface.co/api/repos/create", False),
+        # Collection endpoints => False
+        ("https://huggingface.co/api/collections", False),
+        ("https://huggingface.co/api/collections/foo/bar", False),
+        # Repo endpoints => True
+        ("https://huggingface.co/api/models/repo_id", True),
+        ("https://huggingface.co/api/datasets/repo_id", True),
+        ("https://huggingface.co/api/spaces/repo_id", True),
+        ("https://huggingface.co/api/models/username/repo_name/refs/main", True),
+        ("https://huggingface.co/api/datasets/username/repo_name/refs/main", True),
+        ("https://huggingface.co/api/spaces/username/repo_name/refs/main", True),
+        # Inference Endpoint => False
+        ("https://api.endpoints.huggingface.cloud/v2/endpoint/namespace", False),
+        # Staging Endpoint => True
+        ("https://hub-ci.huggingface.co/api/models/repo_id", True),
+        ("https://hub-ci.huggingface.co/api/datasets/repo_id", True),
+        ("https://hub-ci.huggingface.co/api/spaces/repo_id", True),
+        # /resolve Endpoint => True
+        ("https://huggingface.co/gpt2/resolve/main/README.md", True),
+        ("https://huggingface.co/datasets/google/fleurs/resolve/revision/README.md", True),
+        # Regression tests
+        ("https://huggingface.co/bert-base/resolve/main/pytorch_model.bin", True),
+        ("https://hub-ci.huggingface.co/__DUMMY_USER__/repo-1470b5/resolve/main/file.txt", True),
+    ],
+)
+def test_repo_api_regex(url: str, should_match: bool) -> None:
+    """Test the regex used to match repo API URLs."""
+    if should_match:
+        assert REPO_API_REGEX.match(url)
+    else:
+        assert REPO_API_REGEX.match(url) is None