diff --git a/src/kernelbot/api/api_utils.py b/src/kernelbot/api/api_utils.py index ab1505ac..f26ccd3b 100644 --- a/src/kernelbot/api/api_utils.py +++ b/src/kernelbot/api/api_utils.py @@ -225,6 +225,21 @@ async def to_submit_info( try: with db_context as db: + # Per-user rate limit: max 1 submission per hour on Modal B200 for leaderboard 730 + if gpu_type == "B200": + lb_id = db.get_leaderboard_id(leaderboard_name) + if lb_id == 730: + last_submission_time = db.check_user_rate_limit(user_id) + if last_submission_time: + raise HTTPException( + status_code=429, + detail=( + f"Rate limit exceeded. You can submit once per hour. " + f"Last submission: {last_submission_time.isoformat()}. " + f"Consider using the NVIDIA runner instead of Modal for faster iteration." + ), + ) + leaderboard_item = db.get_leaderboard(leaderboard_name) gpus = leaderboard_item.get("gpu_types", []) if gpu_type not in gpus: @@ -239,7 +254,7 @@ async def to_submit_info( except Exception as e: raise HTTPException( status_code=500, - detail=f"Internal server error while validating leaderboard/GPU: {e}", + detail=f"Internal server error while validating submission: {e}", ) from e try: diff --git a/src/libkernelbot/leaderboard_db.py b/src/libkernelbot/leaderboard_db.py index 334ad633..f1d07cb8 100644 --- a/src/libkernelbot/leaderboard_db.py +++ b/src/libkernelbot/leaderboard_db.py @@ -1172,6 +1172,23 @@ def cleanup_temp_users(self): logger.exception("Could not cleanup temp users", exc_info=e) raise KernelBotError("Database error while cleaning up temp users") from e + def check_user_rate_limit(self, user_id: str, hours: int = 1) -> Optional[datetime.datetime]: + """Check if user has submitted within the last `hours` hours. + Returns the most recent submission_time if rate-limited, None if allowed.""" + self.cursor.execute( + """ + SELECT submission_time + FROM leaderboard.submission + WHERE user_id = %s + AND submission_time > NOW() - INTERVAL '%s hours' + ORDER BY submission_time DESC + LIMIT 1 + """, + (str(user_id), hours), + ) + row = self.cursor.fetchone() + return row[0] if row else None + def validate_cli_id(self, cli_id: str) -> Optional[dict[str, str]]: """ Validates a CLI ID and returns the associated user ID if valid. diff --git a/tests/test_admin_api.py b/tests/test_admin_api.py index ecd1b33c..ada13823 100644 --- a/tests/test_admin_api.py +++ b/tests/test_admin_api.py @@ -1,5 +1,6 @@ """Tests for admin API endpoints.""" +import datetime from unittest.mock import MagicMock, patch import pytest @@ -405,3 +406,101 @@ def test_update_problems_with_errors(self, test_client, mock_backend): assert data["status"] == "ok" assert len(data["errors"]) == 1 assert data["errors"][0]["name"] == "bad-problem" + + +class TestSubmissionRateLimit: + """Test per-user submission rate limiting on Modal B200 for leaderboard 730.""" + + def test_rate_limit_blocks_b200_leaderboard_730(self, test_client, mock_backend): + """Second B200 submission to leaderboard 730 within 1 hour is rejected with 429.""" + mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db) + mock_backend.db.__exit__ = MagicMock(return_value=None) + + recent_time = datetime.datetime.now(tz=datetime.timezone.utc) + mock_backend.db.check_user_rate_limit = MagicMock(return_value=recent_time) + mock_backend.db.get_leaderboard_id = MagicMock(return_value=730) + mock_backend.db.validate_cli_id = MagicMock( + return_value={"user_id": "123", "user_name": "testuser"} + ) + + response = test_client.post( + "/test-lb/B200/test", + headers={"X-Popcorn-Cli-Id": "test-cli-id"}, + files={"file": ("solution.py", b"print('hello')", "text/plain")}, + ) + assert response.status_code == 429 + assert "Rate limit exceeded" in response.json()["detail"] + assert "NVIDIA runner" in response.json()["detail"] + + def test_rate_limit_skipped_for_non_b200(self, test_client, mock_backend): + """Rate limit is not enforced for non-B200 GPUs even on leaderboard 730.""" + mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db) + mock_backend.db.__exit__ = MagicMock(return_value=None) + mock_backend.accepts_jobs = True + + mock_backend.db.validate_cli_id = MagicMock( + return_value={"user_id": "123", "user_name": "testuser"} + ) + + mock_lb = MagicMock() + mock_lb.__getitem__ = lambda self, key: {"gpu_types": ["H100"]}[key] + mock_lb.get = lambda key, default=None: {"gpu_types": ["H100"]}.get(key, default) + mock_backend.db.get_leaderboard = MagicMock(return_value=mock_lb) + + response = test_client.post( + "/test-lb/H100/test", + headers={"X-Popcorn-Cli-Id": "test-cli-id"}, + files={"file": ("solution.py", b"print('hello')", "text/plain")}, + ) + # Should not hit rate limit at all — check_user_rate_limit should not be called + assert response.status_code != 429 + + def test_rate_limit_skipped_for_other_leaderboard(self, test_client, mock_backend): + """Rate limit is not enforced for B200 on a leaderboard other than 730.""" + mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db) + mock_backend.db.__exit__ = MagicMock(return_value=None) + mock_backend.accepts_jobs = True + + recent_time = datetime.datetime.now(tz=datetime.timezone.utc) + mock_backend.db.check_user_rate_limit = MagicMock(return_value=recent_time) + mock_backend.db.get_leaderboard_id = MagicMock(return_value=999) + mock_backend.db.validate_cli_id = MagicMock( + return_value={"user_id": "123", "user_name": "testuser"} + ) + + mock_lb = MagicMock() + mock_lb.__getitem__ = lambda self, key: {"gpu_types": ["B200"]}[key] + mock_lb.get = lambda key, default=None: {"gpu_types": ["B200"]}.get(key, default) + mock_backend.db.get_leaderboard = MagicMock(return_value=mock_lb) + + response = test_client.post( + "/other-lb/B200/test", + headers={"X-Popcorn-Cli-Id": "test-cli-id"}, + files={"file": ("solution.py", b"print('hello')", "text/plain")}, + ) + # Should not be rate limited since leaderboard ID is not 730 + assert response.status_code != 429 + + def test_rate_limit_allows_first_b200_submission(self, test_client, mock_backend): + """First B200 submission to leaderboard 730 passes the rate limit check.""" + mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db) + mock_backend.db.__exit__ = MagicMock(return_value=None) + mock_backend.accepts_jobs = True + + mock_backend.db.check_user_rate_limit = MagicMock(return_value=None) + mock_backend.db.get_leaderboard_id = MagicMock(return_value=730) + mock_backend.db.validate_cli_id = MagicMock( + return_value={"user_id": "123", "user_name": "testuser"} + ) + + mock_lb = MagicMock() + mock_lb.__getitem__ = lambda self, key: {"gpu_types": ["B200"]}[key] + mock_lb.get = lambda key, default=None: {"gpu_types": ["B200"]}.get(key, default) + mock_backend.db.get_leaderboard = MagicMock(return_value=mock_lb) + + response = test_client.post( + "/test-lb/B200/test", + headers={"X-Popcorn-Cli-Id": "test-cli-id"}, + files={"file": ("solution.py", b"print('hello')", "text/plain")}, + ) + assert response.status_code != 429 diff --git a/tests/test_leaderboard_db.py b/tests/test_leaderboard_db.py index 1b349816..3d034cbf 100644 --- a/tests/test_leaderboard_db.py +++ b/tests/test_leaderboard_db.py @@ -605,6 +605,51 @@ def test_generate_stats(database, submit_leaderboard): } +def test_check_user_rate_limit_no_submissions(database, submit_leaderboard): + """Test rate limit returns None when user has no submissions""" + with database as db: + result = db.check_user_rate_limit("999") + assert result is None + + +def test_check_user_rate_limit_recent_submission(database, submit_leaderboard): + """Test rate limit returns submission_time when user submitted recently""" + submit_time = datetime.datetime.now(tz=datetime.timezone.utc) + with database as db: + db.create_submission( + "submit-leaderboard", "file.py", 5, "code", submit_time, user_name="user" + ) + result = db.check_user_rate_limit("5") + assert result is not None + assert abs((result - submit_time).total_seconds()) < 2 + + +def test_check_user_rate_limit_old_submission(database, submit_leaderboard): + """Test rate limit returns None when submission is older than the window""" + old_time = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(hours=2) + with database as db: + db.create_submission( + "submit-leaderboard", "file.py", 5, "code", old_time, user_name="user" + ) + result = db.check_user_rate_limit("5") + assert result is None + + +def test_check_user_rate_limit_different_user(database, submit_leaderboard): + """Test rate limit only applies to the specific user""" + submit_time = datetime.datetime.now(tz=datetime.timezone.utc) + with database as db: + db.create_submission( + "submit-leaderboard", "file.py", 5, "code", submit_time, user_name="user5" + ) + # User 6 should not be rate limited + result = db.check_user_rate_limit("6") + assert result is None + # User 5 should be rate limited + result = db.check_user_rate_limit("5") + assert result is not None + + def test_get_user_submissions_empty(database, submit_leaderboard): """Test get_user_submissions returns empty list for user with no submissions""" with database as db: