Remove GitHub-specific token validation and generalize authentication

Co-authored-by: nicoragne <nicoragne@hotmail.fr>

Author: cursoragent

src/gitingest/clone.py

@@ -40,7 +40,7 @@ async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
     config : CloneConfig
         The configuration for cloning the repository.
     token : str | None
-        GitHub personal access token (PAT) for accessing private repositories.
+        Personal access token (PAT) for accessing private repositories.
 
     Raises
     ------
@@ -84,7 +84,7 @@ async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
     logger.debug("Resolved commit", extra={"commit": commit})
 
     clone_cmd = ["git"]
-    if token and is_github_host(url):
+    if token:
         clone_cmd += ["-c", create_git_auth_header(token, url=url)]
 
     clone_cmd += ["clone", "--single-branch", "--no-checkout", "--depth=1"]

src/gitingest/utils/auth.py

@@ -4,24 +4,20 @@
 
 import os
 
-from gitingest.utils.git_utils import validate_github_token
-
 
 def resolve_token(token: str | None) -> str | None:
     """Resolve the token to use for the query.
 
     Parameters
     ----------
     token : str | None
-        GitHub personal access token (PAT) for accessing private repositories.
+        Personal access token (PAT) for accessing private repositories.
 
     Returns
     -------
     str | None
         The resolved token.
 
     """
-    token = token or os.getenv("GITHUB_TOKEN")
-    if token:
-        validate_github_token(token)
+    token = token or os.getenv("GITHUB_TOKEN")  # Keep env var name for backward compatibility
     return token

src/gitingest/utils/git_utils.py

@@ -4,17 +4,16 @@
 
 import asyncio
 import base64
-import re
 import sys
 from pathlib import Path
-from typing import TYPE_CHECKING, Final, Iterable
+from typing import TYPE_CHECKING, Iterable
 from urllib.parse import urlparse
 
 import httpx
 from starlette.status import HTTP_200_OK, HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND
 
 from gitingest.utils.compat_func import removesuffix
-from gitingest.utils.exceptions import InvalidGitHubTokenError
+
 from gitingest.utils.logging_config import get_logger
 
 if TYPE_CHECKING:
@@ -23,10 +22,7 @@
 # Initialize logger for this module
 logger = get_logger(__name__)
 
-# GitHub Personal-Access tokens (classic + fine-grained).
-#   - ghp_ / gho_ / ghu_ / ghs_ / ghr_  → 36 alphanumerics
-#   - github_pat_                       → 22 alphanumerics + "_" + 59 alphanumerics
-_GITHUB_PAT_PATTERN: Final[str] = r"^(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9]{22}_[A-Za-z0-9]{59})$"
+
 
 
 def is_github_host(url: str) -> bool:
@@ -263,9 +259,9 @@ def create_git_command(base_cmd: list[str], local_path: str, url: str, token: st
     local_path : str
         The local path where the git command should be executed.
     url : str
-        The repository URL to check if it's a GitHub repository.
+        The repository URL for authentication.
     token : str | None
-        GitHub personal access token (PAT) for accessing private repositories.
+        Personal access token (PAT) for accessing private repositories.
 
     Returns
     -------
@@ -274,21 +270,20 @@ def create_git_command(base_cmd: list[str], local_path: str, url: str, token: st
 
     """
     cmd = [*base_cmd, "-C", local_path]
-    if token and is_github_host(url):
+    if token:
         cmd += ["-c", create_git_auth_header(token, url=url)]
     return cmd
 
 
-def create_git_auth_header(token: str, url: str = "https://github.com") -> str:
-    """Create a Basic authentication header for GitHub git operations.
+def create_git_auth_header(token: str, url: str) -> str:
+    """Create a Basic authentication header for git operations.
 
     Parameters
     ----------
     token : str
-        GitHub personal access token (PAT) for accessing private repositories.
+        Personal access token (PAT) for accessing private repositories.
     url : str
-        The GitHub URL to create the authentication header for.
-        Defaults to "https://github.com" if not provided.
+        The repository URL to create the authentication header for.
 
     Returns
     -------
@@ -298,35 +293,18 @@ def create_git_auth_header(token: str, url: str = "https://github.com") -> str:
     Raises
     ------
     ValueError
-        If the URL is not a valid GitHub repository URL.
+        If the URL is not a valid repository URL.
 
     """
     hostname = urlparse(url).hostname
     if not hostname:
-        msg = f"Invalid GitHub URL: {url!r}"
+        msg = f"Invalid repository URL: {url!r}"
         raise ValueError(msg)
 
     basic = base64.b64encode(f"x-oauth-basic:{token}".encode()).decode()
     return f"http.https://{hostname}/.extraheader=Authorization: Basic {basic}"
 
 
-def validate_github_token(token: str) -> None:
-    """Validate the format of a GitHub Personal Access Token.
-
-    Parameters
-    ----------
-    token : str
-        GitHub personal access token (PAT) for accessing private repositories.
-
-    Raises
-    ------
-    InvalidGitHubTokenError
-        If the token format is invalid.
-
-    """
-    if not re.fullmatch(_GITHUB_PAT_PATTERN, token):
-        raise InvalidGitHubTokenError
-
 
 async def checkout_partial_clone(config: CloneConfig, token: str | None) -> None:
     """Configure sparse-checkout for a partially cloned repository.

src/server/query_processor.py

@@ -9,7 +9,7 @@
 from gitingest.clone import clone_repo
 from gitingest.ingestion import ingest_query
 from gitingest.query_parser import parse_remote_repo
-from gitingest.utils.git_utils import resolve_commit, validate_github_token
+from gitingest.utils.git_utils import resolve_commit
 from gitingest.utils.logging_config import get_logger
 from gitingest.utils.pattern_utils import process_patterns
 from server.models import IngestErrorResponse, IngestResponse, IngestSuccessResponse, PatternType, S3Metadata
@@ -262,9 +262,6 @@ async def process_query(
         If the commit hash is not found (should never happen).
 
     """
-    if token:
-        validate_github_token(token)
-
     try:
         query = await parse_remote_repo(input_text, token=token)
     except Exception as exc:

tests/test_git_utils.py

@@ -1,58 +1,24 @@
 """Tests for the ``git_utils`` module.
 
-These tests validate the ``validate_github_token`` function, which ensures that
-GitHub personal access tokens (PATs) are properly formatted.
+These tests validate various git utility functions for repository operations.
 """
 
 from __future__ import annotations
 
 import base64
 from typing import TYPE_CHECKING
+from urllib.parse import urlparse
 
 import pytest
 
-from gitingest.utils.exceptions import InvalidGitHubTokenError
-from gitingest.utils.git_utils import create_git_auth_header, create_git_command, is_github_host, validate_github_token
+from gitingest.utils.git_utils import create_git_auth_header, create_git_command, is_github_host
 
 if TYPE_CHECKING:
     from pathlib import Path
 
     from pytest_mock import MockerFixture
 
 
-@pytest.mark.parametrize(
-    "token",
-    [
-        # Valid tokens: correct prefixes and at least 36 allowed characters afterwards
-        "github_pat_" + "a" * 22 + "_" + "b" * 59,
-        "ghp_" + "A" * 36,
-        "ghu_" + "B" * 36,
-        "ghs_" + "C" * 36,
-        "ghr_" + "D" * 36,
-        "gho_" + "E" * 36,
-    ],
-)
-def test_validate_github_token_valid(token: str) -> None:
-    """validate_github_token should accept properly-formatted tokens."""
-    # Should not raise any exception
-    validate_github_token(token)
-
-
-@pytest.mark.parametrize(
-    "token",
-    [
-        "github_pat_short",  # Too short after prefix
-        "ghp_" + "b" * 35,  # one character short
-        "invalidprefix_" + "c" * 36,  # Wrong prefix
-        "github_pat_" + "!" * 36,  # Disallowed characters
-        "github_pat_" + "a" * 36,  # Too short after 'github_pat_' prefix
-        "",  # Empty string
-    ],
-)
-def test_validate_github_token_invalid(token: str) -> None:
-    """Test that ``validate_github_token`` raises ``InvalidGitHubTokenError`` on malformed tokens."""
-    with pytest.raises(InvalidGitHubTokenError):
-        validate_github_token(token)
 
 
 @pytest.mark.parametrize(
@@ -72,15 +38,18 @@ def test_validate_github_token_invalid(token: str) -> None:
             "ghp_" + "d" * 36,
             [
                 "-c",
-                create_git_auth_header("ghp_" + "d" * 36),
-            ],  # Auth header expected for GitHub URL + token
+                create_git_auth_header("ghp_" + "d" * 36, "https://github.com/owner/repo.git"),
+            ],  # Auth header expected when token is provided
         ),
         (
             ["git", "clone"],
             "/some/path",
             "https://gitlab.com/owner/repo.git",
             "ghp_" + "e" * 36,
-            [],  # No auth header for non-GitHub URL even if token provided
+            [
+                "-c",
+                create_git_auth_header("ghp_" + "e" * 36, "https://gitlab.com/owner/repo.git"),
+            ],  # Auth header expected for any URL when token is provided
         ),
     ],
 )
@@ -103,17 +72,19 @@ def test_create_git_command(
 
 
 @pytest.mark.parametrize(
-    "token",
+    ("token", "url"),
     [
-        "ghp_abcdefghijklmnopqrstuvwxyz012345",  # typical ghp_ token
-        "github_pat_1234567890abcdef1234567890abcdef1234",
+        ("ghp_abcdefghijklmnopqrstuvwxyz012345", "https://github.com/owner/repo.git"),  # typical ghp_ token
+        ("github_pat_1234567890abcdef1234567890abcdef1234", "https://github.com/owner/repo.git"),
+        ("some_token", "https://gitlab.com/owner/repo.git"),  # non-GitHub URL
     ],
 )
-def test_create_git_auth_header(token: str) -> None:
+def test_create_git_auth_header(token: str, url: str) -> None:
     """Test that ``create_git_auth_header`` produces correct base64-encoded header."""
-    header = create_git_auth_header(token)
+    header = create_git_auth_header(token, url)
     expected_basic = base64.b64encode(f"x-oauth-basic:{token}".encode()).decode()
-    expected = f"http.https://github.com/.extraheader=Authorization: Basic {expected_basic}"
+    hostname = urlparse(url).hostname
+    expected = f"http.https://{hostname}/.extraheader=Authorization: Basic {expected_basic}"
     assert header == expected
 
 
@@ -122,7 +93,7 @@ def test_create_git_auth_header(token: str) -> None:
     [
         ("https://github.com/foo/bar.git", "ghp_" + "f" * 36, True),
         ("https://github.com/foo/bar.git", None, False),
-        ("https://gitlab.com/foo/bar.git", "ghp_" + "g" * 36, False),
+        ("https://gitlab.com/foo/bar.git", "ghp_" + "g" * 36, True),  # Now called for all URLs with token
     ],
 )
 def test_create_git_command_helper_calls(