From c14e989fdc8f78e0e82ad55bf563b212890af7e3 Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Sat, 7 Feb 2026 22:04:48 -0800 Subject: [PATCH] Revert "Add per-user submission rate limit (1 per hour) (#436)" This reverts commit 7d97a31266eae0a42d626e257d7fa238e8b3c1eb. --- src/kernelbot/api/api_utils.py | 17 +---- src/libkernelbot/leaderboard_db.py | 17 ----- tests/test_admin_api.py | 99 ------------------------------ tests/test_leaderboard_db.py | 45 -------------- 4 files changed, 1 insertion(+), 177 deletions(-) diff --git a/src/kernelbot/api/api_utils.py b/src/kernelbot/api/api_utils.py index f26ccd3b..ab1505ac 100644 --- a/src/kernelbot/api/api_utils.py +++ b/src/kernelbot/api/api_utils.py @@ -225,21 +225,6 @@ async def to_submit_info( try: with db_context as db: - # Per-user rate limit: max 1 submission per hour on Modal B200 for leaderboard 730 - if gpu_type == "B200": - lb_id = db.get_leaderboard_id(leaderboard_name) - if lb_id == 730: - last_submission_time = db.check_user_rate_limit(user_id) - if last_submission_time: - raise HTTPException( - status_code=429, - detail=( - f"Rate limit exceeded. You can submit once per hour. " - f"Last submission: {last_submission_time.isoformat()}. " - f"Consider using the NVIDIA runner instead of Modal for faster iteration." - ), - ) - leaderboard_item = db.get_leaderboard(leaderboard_name) gpus = leaderboard_item.get("gpu_types", []) if gpu_type not in gpus: @@ -254,7 +239,7 @@ async def to_submit_info( except Exception as e: raise HTTPException( status_code=500, - detail=f"Internal server error while validating submission: {e}", + detail=f"Internal server error while validating leaderboard/GPU: {e}", ) from e try: diff --git a/src/libkernelbot/leaderboard_db.py b/src/libkernelbot/leaderboard_db.py index f1d07cb8..334ad633 100644 --- a/src/libkernelbot/leaderboard_db.py +++ b/src/libkernelbot/leaderboard_db.py @@ -1172,23 +1172,6 @@ def cleanup_temp_users(self): logger.exception("Could not cleanup temp users", exc_info=e) raise KernelBotError("Database error while cleaning up temp users") from e - def check_user_rate_limit(self, user_id: str, hours: int = 1) -> Optional[datetime.datetime]: - """Check if user has submitted within the last `hours` hours. - Returns the most recent submission_time if rate-limited, None if allowed.""" - self.cursor.execute( - """ - SELECT submission_time - FROM leaderboard.submission - WHERE user_id = %s - AND submission_time > NOW() - INTERVAL '%s hours' - ORDER BY submission_time DESC - LIMIT 1 - """, - (str(user_id), hours), - ) - row = self.cursor.fetchone() - return row[0] if row else None - def validate_cli_id(self, cli_id: str) -> Optional[dict[str, str]]: """ Validates a CLI ID and returns the associated user ID if valid. diff --git a/tests/test_admin_api.py b/tests/test_admin_api.py index ada13823..ecd1b33c 100644 --- a/tests/test_admin_api.py +++ b/tests/test_admin_api.py @@ -1,6 +1,5 @@ """Tests for admin API endpoints.""" -import datetime from unittest.mock import MagicMock, patch import pytest @@ -406,101 +405,3 @@ def test_update_problems_with_errors(self, test_client, mock_backend): assert data["status"] == "ok" assert len(data["errors"]) == 1 assert data["errors"][0]["name"] == "bad-problem" - - -class TestSubmissionRateLimit: - """Test per-user submission rate limiting on Modal B200 for leaderboard 730.""" - - def test_rate_limit_blocks_b200_leaderboard_730(self, test_client, mock_backend): - """Second B200 submission to leaderboard 730 within 1 hour is rejected with 429.""" - mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db) - mock_backend.db.__exit__ = MagicMock(return_value=None) - - recent_time = datetime.datetime.now(tz=datetime.timezone.utc) - mock_backend.db.check_user_rate_limit = MagicMock(return_value=recent_time) - mock_backend.db.get_leaderboard_id = MagicMock(return_value=730) - mock_backend.db.validate_cli_id = MagicMock( - return_value={"user_id": "123", "user_name": "testuser"} - ) - - response = test_client.post( - "/test-lb/B200/test", - headers={"X-Popcorn-Cli-Id": "test-cli-id"}, - files={"file": ("solution.py", b"print('hello')", "text/plain")}, - ) - assert response.status_code == 429 - assert "Rate limit exceeded" in response.json()["detail"] - assert "NVIDIA runner" in response.json()["detail"] - - def test_rate_limit_skipped_for_non_b200(self, test_client, mock_backend): - """Rate limit is not enforced for non-B200 GPUs even on leaderboard 730.""" - mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db) - mock_backend.db.__exit__ = MagicMock(return_value=None) - mock_backend.accepts_jobs = True - - mock_backend.db.validate_cli_id = MagicMock( - return_value={"user_id": "123", "user_name": "testuser"} - ) - - mock_lb = MagicMock() - mock_lb.__getitem__ = lambda self, key: {"gpu_types": ["H100"]}[key] - mock_lb.get = lambda key, default=None: {"gpu_types": ["H100"]}.get(key, default) - mock_backend.db.get_leaderboard = MagicMock(return_value=mock_lb) - - response = test_client.post( - "/test-lb/H100/test", - headers={"X-Popcorn-Cli-Id": "test-cli-id"}, - files={"file": ("solution.py", b"print('hello')", "text/plain")}, - ) - # Should not hit rate limit at all — check_user_rate_limit should not be called - assert response.status_code != 429 - - def test_rate_limit_skipped_for_other_leaderboard(self, test_client, mock_backend): - """Rate limit is not enforced for B200 on a leaderboard other than 730.""" - mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db) - mock_backend.db.__exit__ = MagicMock(return_value=None) - mock_backend.accepts_jobs = True - - recent_time = datetime.datetime.now(tz=datetime.timezone.utc) - mock_backend.db.check_user_rate_limit = MagicMock(return_value=recent_time) - mock_backend.db.get_leaderboard_id = MagicMock(return_value=999) - mock_backend.db.validate_cli_id = MagicMock( - return_value={"user_id": "123", "user_name": "testuser"} - ) - - mock_lb = MagicMock() - mock_lb.__getitem__ = lambda self, key: {"gpu_types": ["B200"]}[key] - mock_lb.get = lambda key, default=None: {"gpu_types": ["B200"]}.get(key, default) - mock_backend.db.get_leaderboard = MagicMock(return_value=mock_lb) - - response = test_client.post( - "/other-lb/B200/test", - headers={"X-Popcorn-Cli-Id": "test-cli-id"}, - files={"file": ("solution.py", b"print('hello')", "text/plain")}, - ) - # Should not be rate limited since leaderboard ID is not 730 - assert response.status_code != 429 - - def test_rate_limit_allows_first_b200_submission(self, test_client, mock_backend): - """First B200 submission to leaderboard 730 passes the rate limit check.""" - mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db) - mock_backend.db.__exit__ = MagicMock(return_value=None) - mock_backend.accepts_jobs = True - - mock_backend.db.check_user_rate_limit = MagicMock(return_value=None) - mock_backend.db.get_leaderboard_id = MagicMock(return_value=730) - mock_backend.db.validate_cli_id = MagicMock( - return_value={"user_id": "123", "user_name": "testuser"} - ) - - mock_lb = MagicMock() - mock_lb.__getitem__ = lambda self, key: {"gpu_types": ["B200"]}[key] - mock_lb.get = lambda key, default=None: {"gpu_types": ["B200"]}.get(key, default) - mock_backend.db.get_leaderboard = MagicMock(return_value=mock_lb) - - response = test_client.post( - "/test-lb/B200/test", - headers={"X-Popcorn-Cli-Id": "test-cli-id"}, - files={"file": ("solution.py", b"print('hello')", "text/plain")}, - ) - assert response.status_code != 429 diff --git a/tests/test_leaderboard_db.py b/tests/test_leaderboard_db.py index 3d034cbf..1b349816 100644 --- a/tests/test_leaderboard_db.py +++ b/tests/test_leaderboard_db.py @@ -605,51 +605,6 @@ def test_generate_stats(database, submit_leaderboard): } -def test_check_user_rate_limit_no_submissions(database, submit_leaderboard): - """Test rate limit returns None when user has no submissions""" - with database as db: - result = db.check_user_rate_limit("999") - assert result is None - - -def test_check_user_rate_limit_recent_submission(database, submit_leaderboard): - """Test rate limit returns submission_time when user submitted recently""" - submit_time = datetime.datetime.now(tz=datetime.timezone.utc) - with database as db: - db.create_submission( - "submit-leaderboard", "file.py", 5, "code", submit_time, user_name="user" - ) - result = db.check_user_rate_limit("5") - assert result is not None - assert abs((result - submit_time).total_seconds()) < 2 - - -def test_check_user_rate_limit_old_submission(database, submit_leaderboard): - """Test rate limit returns None when submission is older than the window""" - old_time = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(hours=2) - with database as db: - db.create_submission( - "submit-leaderboard", "file.py", 5, "code", old_time, user_name="user" - ) - result = db.check_user_rate_limit("5") - assert result is None - - -def test_check_user_rate_limit_different_user(database, submit_leaderboard): - """Test rate limit only applies to the specific user""" - submit_time = datetime.datetime.now(tz=datetime.timezone.utc) - with database as db: - db.create_submission( - "submit-leaderboard", "file.py", 5, "code", submit_time, user_name="user5" - ) - # User 6 should not be rate limited - result = db.check_user_rate_limit("6") - assert result is None - # User 5 should be rate limited - result = db.check_user_rate_limit("5") - assert result is not None - - def test_get_user_submissions_empty(database, submit_leaderboard): """Test get_user_submissions returns empty list for user with no submissions""" with database as db: