From af11f3b84fa06b35968b4d05b572cfe98dcda440 Mon Sep 17 00:00:00 2001 From: Futureppo Date: Thu, 8 Jan 2026 18:16:40 +0800 Subject: [PATCH 1/7] =?UTF-8?q?feat(context):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E6=8C=89=E6=A8=A1=E5=9E=8B=E8=83=BD=E5=8A=9B=E6=B8=85=E7=90=86?= =?UTF-8?q?=E5=8E=86=E5=8F=B2=E4=B8=8A=E4=B8=8B=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- astrbot/core/config/default.py | 9 ++ .../method/agent_sub_stages/internal.py | 141 ++++++++++++++++++ .../en-US/features/config-metadata.json | 6 +- .../zh-CN/features/config-metadata.json | 6 +- 4 files changed, 160 insertions(+), 2 deletions(-) diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index 7d5b89334..b94141af0 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -97,6 +97,7 @@ "dequeue_context_length": 1, "streaming_response": False, "show_tool_use_status": False, + "sanitize_context_by_modalities": True, "agent_runner_type": "local", "dify_agent_runner_provider_id": "", "coze_agent_runner_provider_id": "", @@ -2643,6 +2644,14 @@ class ChatProviderTemplate(TypedDict): "provider_settings.agent_runner_type": "local", }, }, + "provider_settings.sanitize_context_by_modalities": { + "description": "按模型能力清理历史上下文", + "type": "bool", + "hint": "开启后，在每次请求 LLM 前会按当前 provider 的 modalities 删除 contexts 中不支持的图片/工具调用结构（会改变模型看到的历史）", + "condition": { + "provider_settings.agent_runner_type": "local", + }, + }, "provider_settings.max_agent_step": { "description": "工具调用轮数上限", "type": "int", diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py index 474b40e34..c3121ada8 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py @@ -52,6 +52,10 @@ async def initialize(self, ctx: PipelineContext) -> None: self.max_step = 30 self.show_tool_use: bool = settings.get("show_tool_use_status", True) self.show_reasoning = settings.get("display_reasoning_text", False) + self.sanitize_context_by_modalities: bool = settings.get( + "sanitize_context_by_modalities", + False, + ) self.kb_agentic_mode: bool = conf.get("kb_agentic_mode", False) file_extract_conf: dict = settings.get("file_extract", {}) @@ -202,6 +206,140 @@ def _modalities_fix( ) req.func_tool = None + def _sanitize_context_by_modalities( + self, + provider: Provider, + req: ProviderRequest, + ) -> None: + """Sanitize `req.contexts` (including history) by current provider modalities.""" + if not self.sanitize_context_by_modalities: + return + + if not isinstance(req.contexts, list) or not req.contexts: + return + + modalities_sentinel = object() + raw_modalities = provider.provider_config.get("modalities", modalities_sentinel) + # Backward compatibility: if modalities is not configured, do not sanitize. + if raw_modalities is modalities_sentinel: + return + if not isinstance(raw_modalities, list): + return + + normalized_modalities = { + str(modality).lower() + for modality in raw_modalities + if isinstance(modality, str) and modality + } + supports_image = bool({"image", "image_url", "vision"} & normalized_modalities) + supports_tool_use = bool( + {"tool_use", "tools", "tool", "function_call", "function"} + & normalized_modalities + ) + + if supports_image and supports_tool_use: + return + + placeholder_texts = {"[图片]", "[image]", "[Image]", "[IMAGE]"} + + def is_only_image_placeholder(parts: list) -> bool: + if not parts: + return False + for part in parts: + if isinstance(part, dict): + if str(part.get("type", "")).lower() != "text": + return False + text = part.get("text", "") + if not isinstance(text, str) or text.strip() not in placeholder_texts: + return False + elif isinstance(part, str): + if part.strip() not in placeholder_texts: + return False + else: + return False + return True + + sanitized_contexts: list[dict] = [] + removed_image_blocks = 0 + removed_tool_messages = 0 + removed_tool_calls = 0 + + for msg in req.contexts: + if not isinstance(msg, dict): + continue + + role = msg.get("role") + if not isinstance(role, str): + role = "" + + new_msg: dict = msg + + # tool_use sanitize + if not supports_tool_use: + if role == "tool": + removed_tool_messages += 1 + continue + if role == "assistant" and ( + "tool_calls" in new_msg or "function_call" in new_msg + ): + new_msg = dict(new_msg) + if "tool_calls" in new_msg: + removed_tool_calls += 1 + if "function_call" in new_msg: + removed_tool_calls += 1 + new_msg.pop("tool_calls", None) + new_msg.pop("function_call", None) + new_msg.pop("tool_call_id", None) + + # image sanitize + if not supports_image: + content = new_msg.get("content") + if isinstance(content, list): + filtered_parts: list = [] + removed_any_image = False + for part in content: + if isinstance(part, dict): + part_type = str(part.get("type", "")).lower() + if part_type in {"image_url", "image"}: + removed_any_image = True + removed_image_blocks += 1 + continue + filtered_parts.append(part) + + if removed_any_image: + if not filtered_parts or is_only_image_placeholder( + filtered_parts + ): + continue + new_msg = dict(new_msg) + new_msg["content"] = filtered_parts + + # drop empty assistant messages (e.g. only tool_calls without content) + if role == "assistant": + content = new_msg.get("content") + has_tool_calls = bool( + new_msg.get("tool_calls") or new_msg.get("function_call") + ) + if not has_tool_calls: + if content is None: + continue + if isinstance(content, str) and not content.strip(): + continue + if isinstance(content, list) and len(content) == 0: + continue + + sanitized_contexts.append(new_msg) + + if removed_image_blocks or removed_tool_messages or removed_tool_calls: + logger.debug( + "sanitize_context_by_modalities applied: " + f"removed_image_blocks={removed_image_blocks}, " + f"removed_tool_messages={removed_tool_messages}, " + f"removed_tool_calls={removed_tool_calls}" + ) + + req.contexts = sanitized_contexts + def _plugin_tool_fix( self, event: AstrMessageEvent, @@ -447,6 +585,9 @@ async def process( # filter tools, only keep tools from this pipeline's selected plugins self._plugin_tool_fix(event, req) + # sanitize contexts (including history) by provider modalities + self._sanitize_context_by_modalities(provider, req) + stream_to_general = ( self.unsupported_streaming_strategy == "turn_off" and not event.platform_meta.support_streaming_message diff --git a/dashboard/src/i18n/locales/en-US/features/config-metadata.json b/dashboard/src/i18n/locales/en-US/features/config-metadata.json index e0f694c33..0768039d6 100644 --- a/dashboard/src/i18n/locales/en-US/features/config-metadata.json +++ b/dashboard/src/i18n/locales/en-US/features/config-metadata.json @@ -187,6 +187,10 @@ "show_tool_use_status": { "description": "Output Function Call Status" }, + "sanitize_context_by_modalities": { + "description": "Sanitize History by Modalities", + "hint": "When enabled, sanitizes contexts before each LLM request by removing image blocks and tool-call structures that the current provider's modalities do not support (this changes what the model sees)." + }, "max_agent_step": { "description": "Maximum Tool Call Rounds" }, @@ -524,4 +528,4 @@ } } } -} \ No newline at end of file +} diff --git a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json index 589aa54a0..e3fcd1eae 100644 --- a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json +++ b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json @@ -184,6 +184,10 @@ "show_tool_use_status": { "description": "输出函数调用状态" }, + "sanitize_context_by_modalities": { + "description": "按模型能力清理历史上下文", + "hint": "开启后，在每次请求 LLM 前会按当前 provider 的 modalities 自动删除历史 contexts 中不支持的图片块/工具调用结构（会改变模型看到的历史）" + }, "max_agent_step": { "description": "工具调用轮数上限" }, @@ -522,4 +526,4 @@ } } } -} \ No newline at end of file +} From 8f17d02da1f91b12d164cf12ad722560547fd5e0 Mon Sep 17 00:00:00 2001 From: Futureppo Date: Thu, 8 Jan 2026 18:33:43 +0800 Subject: [PATCH 2/7] =?UTF-8?q?fix(config):=20=E6=9B=B4=E6=96=B0=E5=8E=86?= =?UTF-8?q?=E5=8F=B2=E4=B8=8A=E4=B8=8B=E6=96=87=E6=B8=85=E7=90=86=E6=8F=90?= =?UTF-8?q?=E7=A4=BA=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- astrbot/core/config/default.py | 2 +- dashboard/src/i18n/locales/zh-CN/features/config-metadata.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index b94141af0..a885242bf 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -2647,7 +2647,7 @@ class ChatProviderTemplate(TypedDict): "provider_settings.sanitize_context_by_modalities": { "description": "按模型能力清理历史上下文", "type": "bool", - "hint": "开启后，在每次请求 LLM 前会按当前 provider 的 modalities 删除 contexts 中不支持的图片/工具调用结构（会改变模型看到的历史）", + "hint": "开启后，在每次请求 LLM 前会按当前模型提供商中所选择的模型能力删除对话中不支持的图片/工具调用结构（会改变模型看到的历史）", "condition": { "provider_settings.agent_runner_type": "local", }, diff --git a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json index e3fcd1eae..7e88d9a7e 100644 --- a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json +++ b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json @@ -186,7 +186,7 @@ }, "sanitize_context_by_modalities": { "description": "按模型能力清理历史上下文", - "hint": "开启后，在每次请求 LLM 前会按当前 provider 的 modalities 自动删除历史 contexts 中不支持的图片块/工具调用结构（会改变模型看到的历史）" + "hint": "开启后，在每次请求 LLM 前会按当前模型提供商中所选择的模型能力删除对话中不支持的图片/工具调用结构（会改变模型看到的历史）" }, "max_agent_step": { "description": "工具调用轮数上限" From 8ac553179f1e74b4bf85866b155e906c9e01bafc Mon Sep 17 00:00:00 2001 From: Soulter <905617992@qq.com> Date: Sun, 11 Jan 2026 15:12:13 +0800 Subject: [PATCH 3/7] chore: ruff format --- .../process_stage/method/agent_sub_stages/internal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py index c3121ada8..bd9569d0e 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py @@ -250,7 +250,10 @@ def is_only_image_placeholder(parts: list) -> bool: if str(part.get("type", "")).lower() != "text": return False text = part.get("text", "") - if not isinstance(text, str) or text.strip() not in placeholder_texts: + if ( + not isinstance(text, str) + or text.strip() not in placeholder_texts + ): return False elif isinstance(part, str): if part.strip() not in placeholder_texts: From 9cb5a869bdd0154110c0e0d905a0f4d0dadf6bd7 Mon Sep 17 00:00:00 2001 From: Soulter <905617992@qq.com> Date: Sun, 11 Jan 2026 15:33:38 +0800 Subject: [PATCH 4/7] fix: simplify modality checks and sanitize context handling --- .../method/agent_sub_stages/internal.py | 57 +++---------------- 1 file changed, 9 insertions(+), 48 deletions(-) diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py index bd9569d0e..4bf693e7c 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py @@ -218,50 +218,17 @@ def _sanitize_context_by_modalities( if not isinstance(req.contexts, list) or not req.contexts: return - modalities_sentinel = object() - raw_modalities = provider.provider_config.get("modalities", modalities_sentinel) - # Backward compatibility: if modalities is not configured, do not sanitize. - if raw_modalities is modalities_sentinel: - return - if not isinstance(raw_modalities, list): + modalities = provider.provider_config.get("modalities", None) + # if modalities is not configured, do not sanitize. + if not modalities or not isinstance(modalities, list): return - normalized_modalities = { - str(modality).lower() - for modality in raw_modalities - if isinstance(modality, str) and modality - } - supports_image = bool({"image", "image_url", "vision"} & normalized_modalities) - supports_tool_use = bool( - {"tool_use", "tools", "tool", "function_call", "function"} - & normalized_modalities - ) + supports_image = bool("image" in modalities) + supports_tool_use = bool("tool_use" in modalities) if supports_image and supports_tool_use: return - placeholder_texts = {"[图片]", "[image]", "[Image]", "[IMAGE]"} - - def is_only_image_placeholder(parts: list) -> bool: - if not parts: - return False - for part in parts: - if isinstance(part, dict): - if str(part.get("type", "")).lower() != "text": - return False - text = part.get("text", "") - if ( - not isinstance(text, str) - or text.strip() not in placeholder_texts - ): - return False - elif isinstance(part, str): - if part.strip() not in placeholder_texts: - return False - else: - return False - return True - sanitized_contexts: list[dict] = [] removed_image_blocks = 0 removed_tool_messages = 0 @@ -280,15 +247,16 @@ def is_only_image_placeholder(parts: list) -> bool: # tool_use sanitize if not supports_tool_use: if role == "tool": + # tool response block removed_tool_messages += 1 continue if role == "assistant" and ( "tool_calls" in new_msg or "function_call" in new_msg ): - new_msg = dict(new_msg) + # assistant message with tool calls if "tool_calls" in new_msg: removed_tool_calls += 1 - if "function_call" in new_msg: + elif "function_call" in new_msg: removed_tool_calls += 1 new_msg.pop("tool_calls", None) new_msg.pop("function_call", None) @@ -310,11 +278,6 @@ def is_only_image_placeholder(parts: list) -> bool: filtered_parts.append(part) if removed_any_image: - if not filtered_parts or is_only_image_placeholder( - filtered_parts - ): - continue - new_msg = dict(new_msg) new_msg["content"] = filtered_parts # drop empty assistant messages (e.g. only tool_calls without content) @@ -324,12 +287,10 @@ def is_only_image_placeholder(parts: list) -> bool: new_msg.get("tool_calls") or new_msg.get("function_call") ) if not has_tool_calls: - if content is None: + if not content: continue if isinstance(content, str) and not content.strip(): continue - if isinstance(content, list) and len(content) == 0: - continue sanitized_contexts.append(new_msg) From d2674d0f4665b8fe805089c9643fa6253c8c721e Mon Sep 17 00:00:00 2001 From: Soulter <905617992@qq.com> Date: Sun, 11 Jan 2026 15:34:59 +0800 Subject: [PATCH 5/7] fix(config): disable context sanitization by modalities --- astrbot/core/config/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index a885242bf..4fffab42f 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -97,7 +97,7 @@ "dequeue_context_length": 1, "streaming_response": False, "show_tool_use_status": False, - "sanitize_context_by_modalities": True, + "sanitize_context_by_modalities": False, "agent_runner_type": "local", "dify_agent_runner_provider_id": "", "coze_agent_runner_provider_id": "", From 25d6f5219c5c66faac587d95c7ac0fda357e0cb5 Mon Sep 17 00:00:00 2001 From: Soulter <905617992@qq.com> Date: Sun, 11 Jan 2026 15:37:18 +0800 Subject: [PATCH 6/7] fix(agent): skip messages with empty roles in InternalAgentSubStage --- .../process_stage/method/agent_sub_stages/internal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py index 4bf693e7c..0ecfa916f 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py @@ -239,8 +239,8 @@ def _sanitize_context_by_modalities( continue role = msg.get("role") - if not isinstance(role, str): - role = "" + if not role: + continue new_msg: dict = msg From aca598820e69eff56d7bf05f3d35634921278aeb Mon Sep 17 00:00:00 2001 From: Soulter <905617992@qq.com> Date: Sun, 11 Jan 2026 15:38:08 +0800 Subject: [PATCH 7/7] fix(agent): refine tool call handling in InternalAgentSubStage --- .../process_stage/method/agent_sub_stages/internal.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py index 0ecfa916f..6969177c8 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py @@ -250,16 +250,11 @@ def _sanitize_context_by_modalities( # tool response block removed_tool_messages += 1 continue - if role == "assistant" and ( - "tool_calls" in new_msg or "function_call" in new_msg - ): + if role == "assistant" and "tool_calls" in new_msg: # assistant message with tool calls if "tool_calls" in new_msg: removed_tool_calls += 1 - elif "function_call" in new_msg: - removed_tool_calls += 1 new_msg.pop("tool_calls", None) - new_msg.pop("function_call", None) new_msg.pop("tool_call_id", None) # image sanitize @@ -283,9 +278,7 @@ def _sanitize_context_by_modalities( # drop empty assistant messages (e.g. only tool_calls without content) if role == "assistant": content = new_msg.get("content") - has_tool_calls = bool( - new_msg.get("tool_calls") or new_msg.get("function_call") - ) + has_tool_calls = bool(new_msg.get("tool_calls")) if not has_tool_calls: if not content: continue