From 1b99371f080095ee88d84a161e439ebb52f99577 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 26 Nov 2024 16:36:19 +0530 Subject: [PATCH] build(model_prices_and_context_window.json): add tpm/rpm limits for all gemini models Allows for ratelimit tracking for gemini models even with wildcard routing enabled Addresses https://github.com/BerriAI/litellm/issues/6914 --- ...odel_prices_and_context_window_backup.json | 58 ++++++++++++++++++- model_prices_and_context_window.json | 58 ++++++++++++++++++- tests/local_testing/test_get_model_info.py | 14 +++++ 3 files changed, 124 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 23eb59e0bb93..6ae1acfcf88c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3460,6 +3460,30 @@ "rpm": 2000, "source": "https://ai.google.dev/pricing" }, + "gemini/gemini-1.5-flash-8b": { + "max_tokens": 8192, + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_images_per_prompt": 3000, + "max_videos_per_prompt": 10, + "max_video_length": 1, + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_pdf_size_mb": 30, + "input_cost_per_token": 0, + "input_cost_per_token_above_128k_tokens": 0, + "output_cost_per_token": 0, + "output_cost_per_token_above_128k_tokens": 0, + "litellm_provider": "gemini", + "mode": "chat", + "supports_system_messages": true, + "supports_function_calling": true, + "supports_vision": true, + "supports_response_schema": true, + "tpm": 4000000, + "rpm": 4000, + "source": "https://ai.google.dev/pricing" + }, "gemini/gemini-1.5-flash-8b-exp-0924": { "max_tokens": 8192, "max_input_tokens": 1048576, @@ -3481,7 +3505,7 @@ "supports_vision": true, "supports_response_schema": true, "tpm": 4000000, - "rpm": 2000, + "rpm": 4000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-exp-1114": { @@ -3504,7 +3528,12 @@ "supports_function_calling": true, "supports_vision": true, "supports_response_schema": true, - "source": "https://ai.google.dev/pricing" + "tpm": 4000000, + "rpm": 1000, + "source": "https://ai.google.dev/pricing", + "metadata": { + "notes": "Rate limits not documented for gemini-exp-1114. Assuming same as gemini-1.5-pro." + } }, "gemini/gemini-1.5-flash-exp-0827": { "max_tokens": 8192, @@ -3526,6 +3555,8 @@ "supports_function_calling": true, "supports_vision": true, "supports_response_schema": true, + "tpm": 4000000, + "rpm": 2000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-1.5-flash-8b-exp-0827": { @@ -3547,6 +3578,9 @@ "supports_system_messages": true, "supports_function_calling": true, "supports_vision": true, + "supports_response_schema": true, + "tpm": 4000000, + "rpm": 4000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-pro": { @@ -3560,7 +3594,10 @@ "litellm_provider": "gemini", "mode": "chat", "supports_function_calling": true, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" + "rpd": 30000, + "tpm": 120000, + "rpm": 360, + "source": "https://ai.google.dev/gemini-api/docs/models/gemini" }, "gemini/gemini-1.5-pro": { "max_tokens": 8192, @@ -3577,6 +3614,8 @@ "supports_vision": true, "supports_tool_choice": true, "supports_response_schema": true, + "tpm": 4000000, + "rpm": 1000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-1.5-pro-002": { @@ -3595,6 +3634,8 @@ "supports_tool_choice": true, "supports_response_schema": true, "supports_prompt_caching": true, + "tpm": 4000000, + "rpm": 1000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-1.5-pro-001": { @@ -3613,6 +3654,8 @@ "supports_tool_choice": true, "supports_response_schema": true, "supports_prompt_caching": true, + "tpm": 4000000, + "rpm": 1000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-1.5-pro-exp-0801": { @@ -3630,6 +3673,8 @@ "supports_vision": true, "supports_tool_choice": true, "supports_response_schema": true, + "tpm": 4000000, + "rpm": 1000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-1.5-pro-exp-0827": { @@ -3647,6 +3692,8 @@ "supports_vision": true, "supports_tool_choice": true, "supports_response_schema": true, + "tpm": 4000000, + "rpm": 1000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-1.5-pro-latest": { @@ -3664,6 +3711,8 @@ "supports_vision": true, "supports_tool_choice": true, "supports_response_schema": true, + "tpm": 4000000, + "rpm": 1000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-pro-vision": { @@ -3678,6 +3727,9 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, + "rpd": 30000, + "tpm": 120000, + "rpm": 360, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, "gemini/gemini-gemma-2-27b-it": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 23eb59e0bb93..6ae1acfcf88c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3460,6 +3460,30 @@ "rpm": 2000, "source": "https://ai.google.dev/pricing" }, + "gemini/gemini-1.5-flash-8b": { + "max_tokens": 8192, + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_images_per_prompt": 3000, + "max_videos_per_prompt": 10, + "max_video_length": 1, + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_pdf_size_mb": 30, + "input_cost_per_token": 0, + "input_cost_per_token_above_128k_tokens": 0, + "output_cost_per_token": 0, + "output_cost_per_token_above_128k_tokens": 0, + "litellm_provider": "gemini", + "mode": "chat", + "supports_system_messages": true, + "supports_function_calling": true, + "supports_vision": true, + "supports_response_schema": true, + "tpm": 4000000, + "rpm": 4000, + "source": "https://ai.google.dev/pricing" + }, "gemini/gemini-1.5-flash-8b-exp-0924": { "max_tokens": 8192, "max_input_tokens": 1048576, @@ -3481,7 +3505,7 @@ "supports_vision": true, "supports_response_schema": true, "tpm": 4000000, - "rpm": 2000, + "rpm": 4000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-exp-1114": { @@ -3504,7 +3528,12 @@ "supports_function_calling": true, "supports_vision": true, "supports_response_schema": true, - "source": "https://ai.google.dev/pricing" + "tpm": 4000000, + "rpm": 1000, + "source": "https://ai.google.dev/pricing", + "metadata": { + "notes": "Rate limits not documented for gemini-exp-1114. Assuming same as gemini-1.5-pro." + } }, "gemini/gemini-1.5-flash-exp-0827": { "max_tokens": 8192, @@ -3526,6 +3555,8 @@ "supports_function_calling": true, "supports_vision": true, "supports_response_schema": true, + "tpm": 4000000, + "rpm": 2000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-1.5-flash-8b-exp-0827": { @@ -3547,6 +3578,9 @@ "supports_system_messages": true, "supports_function_calling": true, "supports_vision": true, + "supports_response_schema": true, + "tpm": 4000000, + "rpm": 4000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-pro": { @@ -3560,7 +3594,10 @@ "litellm_provider": "gemini", "mode": "chat", "supports_function_calling": true, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" + "rpd": 30000, + "tpm": 120000, + "rpm": 360, + "source": "https://ai.google.dev/gemini-api/docs/models/gemini" }, "gemini/gemini-1.5-pro": { "max_tokens": 8192, @@ -3577,6 +3614,8 @@ "supports_vision": true, "supports_tool_choice": true, "supports_response_schema": true, + "tpm": 4000000, + "rpm": 1000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-1.5-pro-002": { @@ -3595,6 +3634,8 @@ "supports_tool_choice": true, "supports_response_schema": true, "supports_prompt_caching": true, + "tpm": 4000000, + "rpm": 1000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-1.5-pro-001": { @@ -3613,6 +3654,8 @@ "supports_tool_choice": true, "supports_response_schema": true, "supports_prompt_caching": true, + "tpm": 4000000, + "rpm": 1000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-1.5-pro-exp-0801": { @@ -3630,6 +3673,8 @@ "supports_vision": true, "supports_tool_choice": true, "supports_response_schema": true, + "tpm": 4000000, + "rpm": 1000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-1.5-pro-exp-0827": { @@ -3647,6 +3692,8 @@ "supports_vision": true, "supports_tool_choice": true, "supports_response_schema": true, + "tpm": 4000000, + "rpm": 1000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-1.5-pro-latest": { @@ -3664,6 +3711,8 @@ "supports_vision": true, "supports_tool_choice": true, "supports_response_schema": true, + "tpm": 4000000, + "rpm": 1000, "source": "https://ai.google.dev/pricing" }, "gemini/gemini-pro-vision": { @@ -3678,6 +3727,9 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, + "rpd": 30000, + "tpm": 120000, + "rpm": 360, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, "gemini/gemini-gemma-2-27b-it": { diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 11506ed3ddd2..dc77f83906f9 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -102,3 +102,17 @@ def test_get_model_info_ollama_chat(): print(mock_client.call_args.kwargs) assert mock_client.call_args.kwargs["json"]["name"] == "mistral" + + +def test_get_model_info_gemini(): + """ + Tests if ALL gemini models have 'tpm' and 'rpm' in the model info + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model_map = litellm.model_cost + for model, info in model_map.items(): + if model.startswith("gemini/") and not "gemma" in model: + assert info.get("tpm") is not None, f"{model} does not have tpm" + assert info.get("rpm") is not None, f"{model} does not have rpm"