diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 481a2a3ecb78..a3e67d8a73f0 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -80,6 +80,44 @@ def __init__( self.turn_off_message_logging = turn_off_message_logging pass + @staticmethod + def get_callback_env_vars(callback_name: Optional[str] = None) -> List[str]: + """ + Return the environment variables associated with a given callback + name as defined in the proxy callback registry. + + Args: + callback_name: The name of the callback to look up. + + Returns: + List[str]: A list of required environment variable names. + """ + if callback_name is None: + return [] + + normalized_name = callback_name.lower() + + alias_map = { + "langfuse_otel": "langfuse", + } + lookup_name = alias_map.get(normalized_name, normalized_name) + + try: + from litellm.proxy._types import AllCallbacks + except Exception: + return [] + + callbacks = AllCallbacks() + callback_info = getattr(callbacks, lookup_name, None) + if callback_info is None: + return [] + + params = getattr(callback_info, "litellm_callback_params", None) + if not params: + return [] + + return list(params) + def log_pre_api_call(self, model, messages, kwargs): pass diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 2d0077571990..612a2c914f40 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2535,6 +2535,14 @@ class AllCallbacks(LiteLLMPydanticObjectBase): ui_callback_name="Lago Billing", ) + traceloop: CallbackOnUI = CallbackOnUI( + litellm_callback_name="traceloop", + litellm_callback_params=[ + "TRACELOOP_API_KEY", + ], + ui_callback_name="Traceloop", + ) + class SpendLogsMetadata(TypedDict): """ diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index fb7ada8ab106..eb312612779a 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -3,8 +3,12 @@ import litellm from litellm import get_secret from litellm._logging import verbose_proxy_logger +from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import CommonProxyErrors, LiteLLMPromptInjectionParams from litellm.proxy.types_utils.utils import get_instance_fn +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + decrypt_value_helper, +) blue_color_code = "\033[94m" reset_color_code = "\033[0m" @@ -382,3 +386,25 @@ def get_metadata_variable_name_from_kwargs( - LiteLLM is now moving to using `litellm_metadata` for our metadata """ return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" + +def process_callback(_callback: str, callback_type: str, environment_variables: dict) -> dict: + """Process a single callback and return its data with environment variables""" + env_vars = CustomLogger.get_callback_env_vars(_callback) + + env_vars_dict: dict[str, str | None] = {} + for _var in env_vars: + env_variable = environment_variables.get(_var, None) + if env_variable is None: + env_vars_dict[_var] = None + else: + # decode + decrypt the value + decrypted_value = decrypt_value_helper( + value=env_variable, key=_var + ) + env_vars_dict[_var] = decrypted_value + + return { + "name": _callback, + "variables": env_vars_dict, + "type": callback_type + } \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e08ca71123e4..205609a645cf 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -47,6 +47,7 @@ TokenCountResponse, ) from litellm.utils import load_credentials_from_list +from litellm.proxy.common_utils.callback_utils import process_callback if TYPE_CHECKING: from aiohttp import ClientSession @@ -153,6 +154,7 @@ def generate_feedback_box(): ) from litellm.exceptions import RejectedRequestError from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_litellm_metadata_from_kwargs, @@ -9581,8 +9583,10 @@ async def get_config(): # noqa: PLR0915 _general_settings = config_data.get("general_settings", {}) environment_variables = config_data.get("environment_variables", {}) - # check if "langfuse" in litellm_settings _success_callbacks = _litellm_settings.get("success_callback", []) + _failure_callbacks = _litellm_settings.get("failure_callback", []) + _success_and_failure_callbacks = _litellm_settings.get("callbacks", []) + _data_to_return = [] """ [ @@ -9593,70 +9597,20 @@ async def get_config(): # noqa: PLR0915 "LANGFUSE_SECRET_KEY": "value", "LANGFUSE_HOST": "value" }, + "type": "success" } ] """ + for _callback in _success_callbacks: - if _callback != "langfuse": - if _callback == "openmeter": - env_vars = [ - "OPENMETER_API_KEY", - ] - elif _callback == "braintrust": - env_vars = [ - "BRAINTRUST_API_KEY", - "BRAINTRUST_API_BASE", - ] - elif _callback == "traceloop": - env_vars = ["TRACELOOP_API_KEY"] - elif _callback == "custom_callback_api": - env_vars = ["GENERIC_LOGGER_ENDPOINT"] - elif _callback == "otel": - env_vars = ["OTEL_EXPORTER", "OTEL_ENDPOINT", "OTEL_HEADERS"] - elif _callback == "langsmith": - env_vars = [ - "LANGSMITH_API_KEY", - "LANGSMITH_PROJECT", - "LANGSMITH_DEFAULT_RUN_NAME", - ] - else: - env_vars = [] - - env_vars_dict = {} - for _var in env_vars: - env_variable = environment_variables.get(_var, None) - if env_variable is None: - env_vars_dict[_var] = None - else: - # decode + decrypt the value - decrypted_value = decrypt_value_helper( - value=env_variable, key=_var - ) - env_vars_dict[_var] = decrypted_value - - _data_to_return.append({"name": _callback, "variables": env_vars_dict}) - elif _callback == "langfuse": - _langfuse_vars = [ - "LANGFUSE_PUBLIC_KEY", - "LANGFUSE_SECRET_KEY", - "LANGFUSE_HOST", - ] - _langfuse_env_vars = {} - for _var in _langfuse_vars: - env_variable = environment_variables.get(_var, None) - if env_variable is None: - _langfuse_env_vars[_var] = None - else: - # decode + decrypt the value - decrypted_value = decrypt_value_helper( - value=env_variable, key=_var - ) - _langfuse_env_vars[_var] = decrypted_value - - _data_to_return.append( - {"name": _callback, "variables": _langfuse_env_vars} - ) + _data_to_return.append(process_callback(_callback, "success", environment_variables)) + + for _callback in _failure_callbacks: + _data_to_return.append(process_callback(_callback, "failure", environment_variables)) + + for _callback in _success_and_failure_callbacks: + _data_to_return.append(process_callback(_callback, "success_and_failure", environment_variables)) # Check if slack alerting is on _alerting = _general_settings.get("alerting", []) diff --git a/tests/local_testing/test_custom_logger.py b/tests/local_testing/test_custom_logger.py index 00e7c2d5aa16..f3dc6a0a7a43 100644 --- a/tests/local_testing/test_custom_logger.py +++ b/tests/local_testing/test_custom_logger.py @@ -103,6 +103,24 @@ async def async_test_logging_fn(self, kwargs, completion_obj, start_time, end_ti ) +def test_get_callback_env_vars(): + env_vars = CustomLogger.get_callback_env_vars("langfuse") + assert env_vars == [ + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LANGFUSE_HOST", + ] + + alias_env_vars = CustomLogger.get_callback_env_vars("langfuse_otel") + assert alias_env_vars == env_vars + + missing_env_vars = CustomLogger.get_callback_env_vars("does_not_exist") + assert missing_env_vars == [] + + none_env_vars = CustomLogger.get_callback_env_vars(None) + assert none_env_vars == [] + + @pytest.mark.asyncio async def test_async_chat_openai_stream(): try: diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index c41ab391df89..17a6f8eb9aee 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -2401,3 +2401,166 @@ def test_non_root_ui_path_logic(monkeypatch, tmp_path, ui_exists, ui_has_content error_calls = [call[0][0] for call in mock_logger.error.call_args_list] assert any("Path exists:" in call for call in error_calls) assert mock_logger.info.call_count == 0 + + +@pytest.mark.asyncio +async def test_get_config_callbacks_with_all_types(client_no_auth): + """ + Test that /get/config/callbacks returns all three callback types: + - success_callback with type="success" + - failure_callback with type="failure" + - callbacks (success_and_failure) with type="success_and_failure" + """ + from litellm.proxy.proxy_server import ProxyConfig + + # Create a mock config with all three callback types + mock_config_data = { + "litellm_settings": { + "success_callback": ["langfuse", "braintrust"], + "failure_callback": ["sentry"], + "callbacks": ["otel", "langsmith"] + }, + "environment_variables": { + "LANGFUSE_PUBLIC_KEY": "test-public-key", + "LANGFUSE_SECRET_KEY": "test-secret-key", + "LANGFUSE_HOST": "https://test.langfuse.com", + "BRAINTRUST_API_KEY": "test-braintrust-key", + "OTEL_EXPORTER": "otlp", + "OTEL_ENDPOINT": "http://localhost:4317", + "LANGSMITH_API_KEY": "test-langsmith-key", + }, + "general_settings": {} + } + + proxy_config = getattr(litellm.proxy.proxy_server, "proxy_config") + + with patch.object( + proxy_config, "get_config", new=AsyncMock(return_value=mock_config_data) + ), patch( + "litellm.proxy.common_utils.callback_utils.decrypt_value_helper", + side_effect=lambda value, key=None: value + ): + response = client_no_auth.get("/get/config/callbacks") + + assert response.status_code == 200 + result = response.json() + + # Verify response structure + assert "status" in result + assert result["status"] == "success" + assert "callbacks" in result + + callbacks = result["callbacks"] + + # Verify we have all 5 callbacks (2 success + 1 failure + 2 success_and_failure) + assert len(callbacks) == 5 + + # Group callbacks by type + success_callbacks = [cb for cb in callbacks if cb.get("type") == "success"] + failure_callbacks = [cb for cb in callbacks if cb.get("type") == "failure"] + success_and_failure_callbacks = [cb for cb in callbacks if cb.get("type") == "success_and_failure"] + + # Verify all callbacks have required fields + for callback in callbacks: + assert "name" in callback + assert "variables" in callback + assert "type" in callback + assert callback["type"] in ["success", "failure", "success_and_failure"] + + # Verify success callbacks + assert len(success_callbacks) == 2 + success_names = [cb["name"] for cb in success_callbacks] + assert "langfuse" in success_names + assert "braintrust" in success_names + + # Verify failure callbacks + assert len(failure_callbacks) == 1 + assert failure_callbacks[0]["name"] == "sentry" + + # Verify success_and_failure callbacks + assert len(success_and_failure_callbacks) == 2 + success_and_failure_names = [cb["name"] for cb in success_and_failure_callbacks] + assert "otel" in success_and_failure_names + assert "langsmith" in success_and_failure_names + + +@pytest.mark.asyncio +async def test_get_config_callbacks_environment_variables(client_no_auth): + """ + Test that /get/config/callbacks correctly includes environment variables + for each callback type with proper decryption. + """ + from litellm.proxy.proxy_server import ProxyConfig + + # Create a mock config with callbacks and their env vars + mock_config_data = { + "litellm_settings": { + "success_callback": ["langfuse"], + "failure_callback": [], + "callbacks": ["otel"] + }, + "environment_variables": { + "LANGFUSE_PUBLIC_KEY": "encrypted-public-key", + "LANGFUSE_SECRET_KEY": "encrypted-secret-key", + "LANGFUSE_HOST": "https://cloud.langfuse.com", + "OTEL_EXPORTER": "otlp", + "OTEL_ENDPOINT": "http://localhost:4317", + "OTEL_HEADERS": "key=value", + }, + "general_settings": {} + } + + # Mock decrypt to prepend "decrypted-" to values + def mock_decrypt(value, key=None): + if value and isinstance(value, str) and "encrypted" in value: + return f"decrypted-{value}" + return value + + proxy_config = getattr(litellm.proxy.proxy_server, "proxy_config") + + with patch.object( + proxy_config, "get_config", new=AsyncMock(return_value=mock_config_data) + ), patch( + "litellm.proxy.common_utils.callback_utils.decrypt_value_helper", + side_effect=mock_decrypt + ): + response = client_no_auth.get("/get/config/callbacks") + + assert response.status_code == 200 + result = response.json() + + callbacks = result["callbacks"] + + # Find langfuse callback (success type) + langfuse_callback = next( + (cb for cb in callbacks if cb["name"] == "langfuse"), None + ) + assert langfuse_callback is not None + assert langfuse_callback["type"] == "success" + assert "variables" in langfuse_callback + + # Verify langfuse env vars are present and decrypted + langfuse_vars = langfuse_callback["variables"] + assert "LANGFUSE_PUBLIC_KEY" in langfuse_vars + assert langfuse_vars["LANGFUSE_PUBLIC_KEY"] == "decrypted-encrypted-public-key" + assert "LANGFUSE_SECRET_KEY" in langfuse_vars + assert langfuse_vars["LANGFUSE_SECRET_KEY"] == "decrypted-encrypted-secret-key" + assert "LANGFUSE_HOST" in langfuse_vars + assert langfuse_vars["LANGFUSE_HOST"] == "https://cloud.langfuse.com" + + # Find otel callback (success_and_failure type) + otel_callback = next( + (cb for cb in callbacks if cb["name"] == "otel"), None + ) + assert otel_callback is not None + assert otel_callback["type"] == "success_and_failure" + assert "variables" in otel_callback + + # Verify otel env vars are present + otel_vars = otel_callback["variables"] + assert "OTEL_EXPORTER" in otel_vars + assert otel_vars["OTEL_EXPORTER"] == "otlp" + assert "OTEL_ENDPOINT" in otel_vars + assert otel_vars["OTEL_ENDPOINT"] == "http://localhost:4317" + assert "OTEL_HEADERS" in otel_vars + assert otel_vars["OTEL_HEADERS"] == "key=value" diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py index b9ed4b9b5089..877f0092182c 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -9,6 +9,9 @@ get_remaining_tokens_and_requests_from_request_data, ) +from unittest.mock import patch +from litellm.proxy.common_utils.callback_utils import process_callback + def test_get_remaining_tokens_and_requests_from_request_data(): model_group = "openrouter/google/gemini-2.0-flash-001" @@ -27,3 +30,47 @@ def test_get_remaining_tokens_and_requests_from_request_data(): f"x-litellm-key-remaining-requests-{expected_name}": 100, f"x-litellm-key-remaining-tokens-{expected_name}": 200, } + + +@patch( + "litellm.proxy.common_utils.callback_utils.CustomLogger.get_callback_env_vars", + return_value=["API_KEY", "MISSING_VAR"], +) +@patch( + "litellm.proxy.common_utils.callback_utils.decrypt_value_helper", + side_effect=lambda value, key: f"decrypted-{key}", +) +def test_process_callback_with_env_vars(mock_decrypt, mock_get_env_vars): + environment_variables = { + "API_KEY": "ENC_VALUE", + "UNUSED": "SHOULD_BE_IGNORED", + } + + result = process_callback( + _callback="my_callback", + callback_type="input", + environment_variables=environment_variables, + ) + + assert result["name"] == "my_callback" + assert result["type"] == "input" + assert result["variables"] == { + "API_KEY": "decrypted-API_KEY", + "MISSING_VAR": None, + } + + +@patch( + "litellm.proxy.common_utils.callback_utils.CustomLogger.get_callback_env_vars", + return_value=[], +) +def test_process_callback_with_no_required_env_vars(mock_get_env_vars): + result = process_callback( + _callback="another_callback", + callback_type="output", + environment_variables={"SHOULD_NOT_BE_USED": "VALUE"}, + ) + + assert result["name"] == "another_callback" + assert result["type"] == "output" + assert result["variables"] == {} diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 25836a59753f..e9014c1af19f 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -19,11 +19,12 @@ import { Tab, SelectItem, Icon, + Badge, } from "@tremor/react"; import { PencilAltIcon, TrashIcon } from "@heroicons/react/outline"; -import { Modal, Typography, Form, Input, Select, Button as Button2 } from "antd"; +import { Modal, Typography, Form, Input, Select, Button as Button2, Tooltip } from "antd"; import NotificationsManager from "./molecules/notifications_manager"; import EmailSettings from "./email_settings"; @@ -57,6 +58,7 @@ interface AlertingVariables { interface AlertingObject { name: string; + type?: "success" | "failure" | "success_and_failure"; variables: AlertingVariables; } @@ -413,54 +415,101 @@ const Settings: React.FC = ({ accessToken, userRole, userID, Active Logging Callbacks - + Callback Name - {/* Callback Env Vars */} + Callback Type + Actions - {callbacks.map((callback, index) => ( - - - {callback.name} - - - - { - setSelectedEditCallback(callback); - setShowEditCallback(true); - }} - /> - handleDeleteCallback(callback.name)} - className="text-red-500 hover:text-red-700 cursor-pointer" - /> - - - - - ))} + {callbacks.map((callback, index) => { + const canEdit = !callback.type || callback.type === "success"; + const tooltipMessage = + callback.type === "failure" + ? "Modifications and deletion of failure type callbacks are not yet supported in the UI" + : callback.type === "success_and_failure" + ? "Modifications and deletion of success and failure type callbacks are not yet supported in the UI" + : ""; + + const getBadgeColor = (type?: string) => { + if (type === "success") return "green"; + if (type === "failure") return "red"; + if (type === "success_and_failure") return "blue"; + return "gray"; + }; + + const getBadgeLabel = (type?: string) => { + if (type === "success") return "Success Only"; + if (type === "failure") return "Failure Only"; + if (type === "success_and_failure") return "Success & Failure"; + return "Unknown"; + }; + + return ( + + + {callback.name} + + + {callback.type ? ( + {getBadgeLabel(callback.type)} + ) : ( + Unknown + )} + + +
+ + { + if (canEdit) { + setSelectedEditCallback(callback); + setShowEditCallback(true); + } + }} + className={canEdit ? "cursor-pointer" : "opacity-40 cursor-not-allowed"} + /> + + + { + if (canEdit) { + handleDeleteCallback(callback.name); + } + }} + className={ + canEdit + ? "text-red-500 hover:text-red-700 cursor-pointer" + : "text-red-300 opacity-40 cursor-not-allowed" + } + /> + + +
+ + + ); + })}