multiple: structured output tracing standard metadata

libs/core/langchain_core/language_models/chat_models.py

@@ -365,11 +365,28 @@ def stream(
         else:
             config = ensure_config(config)
             messages = self._convert_input(input).to_messages()
+            structured_output_format = kwargs.pop("structured_output_format", None)
+            if structured_output_format:
+                try:
+                    structured_output_format_dict = {
+                        "structured_output_format": {
+                            "kwargs": structured_output_format.get("kwargs", {}),
+                            "schema": convert_to_openai_tool(
+                                structured_output_format["schema"]
+                            ),
+                        }
+                    }
+                except ValueError:
+                    structured_output_format_dict = {}
+            else:
+                structured_output_format_dict = {}
+
             params = self._get_invocation_params(stop=stop, **kwargs)
             options = {"stop": stop, **kwargs}
             inheritable_metadata = {
                 **(config.get("metadata") or {}),
                 **self._get_ls_params(stop=stop, **kwargs),
+                **structured_output_format_dict,
             }
             callback_manager = CallbackManager.configure(
                 config.get("callbacks"),
@@ -441,11 +458,29 @@ async def astream(
 
         config = ensure_config(config)
         messages = self._convert_input(input).to_messages()
+
+        structured_output_format = kwargs.pop("structured_output_format", None)
+        if structured_output_format:
+            try:
+                structured_output_format_dict = {
+                    "structured_output_format": {
+                        "kwargs": structured_output_format.get("kwargs", {}),
+                        "schema": convert_to_openai_tool(
+                            structured_output_format["schema"]
+                        ),
+                    }
+                }
+            except ValueError:
+                structured_output_format_dict = {}
+        else:
+            structured_output_format_dict = {}
+
         params = self._get_invocation_params(stop=stop, **kwargs)
         options = {"stop": stop, **kwargs}
         inheritable_metadata = {
             **(config.get("metadata") or {}),
             **self._get_ls_params(stop=stop, **kwargs),
+            **structured_output_format_dict,
         }
         callback_manager = AsyncCallbackManager.configure(
             config.get("callbacks"),
@@ -606,11 +641,28 @@ def generate(
             An LLMResult, which contains a list of candidate Generations for each input
                 prompt and additional model provider-specific output.
         """
+        structured_output_format = kwargs.pop("structured_output_format", None)
+        if structured_output_format:
+            try:
+                structured_output_format_dict = {
+                    "structured_output_format": {
+                        "kwargs": structured_output_format.get("kwargs", {}),
+                        "schema": convert_to_openai_tool(
+                            structured_output_format["schema"]
+                        ),
+                    }
+                }
+            except ValueError:
+                structured_output_format_dict = {}
+        else:
+            structured_output_format_dict = {}
+
         params = self._get_invocation_params(stop=stop, **kwargs)
         options = {"stop": stop}
         inheritable_metadata = {
             **(metadata or {}),
             **self._get_ls_params(stop=stop, **kwargs),
+            **structured_output_format_dict,
         }
 
         callback_manager = CallbackManager.configure(
@@ -697,11 +749,28 @@ async def agenerate(
             An LLMResult, which contains a list of candidate Generations for each input
                 prompt and additional model provider-specific output.
         """
+        structured_output_format = kwargs.pop("structured_output_format", None)
+        if structured_output_format:
+            try:
+                structured_output_format_dict = {
+                    "structured_output_format": {
+                        "kwargs": structured_output_format.get("kwargs", {}),
+                        "schema": convert_to_openai_tool(
+                            structured_output_format["schema"]
+                        ),
+                    }
+                }
+            except ValueError:
+                structured_output_format_dict = {}
+        else:
+            structured_output_format_dict = {}
+
         params = self._get_invocation_params(stop=stop, **kwargs)
         options = {"stop": stop}
         inheritable_metadata = {
             **(metadata or {}),
             **self._get_ls_params(stop=stop, **kwargs),
+            **structured_output_format_dict,
         }
 
         callback_manager = AsyncCallbackManager.configure(
@@ -1240,7 +1309,12 @@ class AnswerWithJustification(BaseModel):
         if self.bind_tools is BaseChatModel.bind_tools:
             msg = "with_structured_output is not implemented for this model."
             raise NotImplementedError(msg)
-        llm = self.bind_tools([schema], tool_choice="any")
+
+        llm = self.bind_tools(
+            [schema],
+            tool_choice="any",
+            structured_output_format={"kwargs": {}, "schema": schema},
+        )
         if isinstance(schema, type) and is_basemodel_subclass(schema):
             output_parser: OutputParserLike = PydanticToolsParser(
                 tools=[cast(TypeBaseModel, schema)], first_tool_only=True

libs/partners/anthropic/langchain_anthropic/chat_models.py

@@ -1111,9 +1111,13 @@ class AnswerWithJustification(BaseModel):
                 Added support for TypedDict class as `schema`.
 
         """  # noqa: E501
-
-        tool_name = convert_to_anthropic_tool(schema)["name"]
-        llm = self.bind_tools([schema], tool_choice=tool_name)
+        formatted_tool = convert_to_anthropic_tool(schema)
+        tool_name = formatted_tool["name"]
+        llm = self.bind_tools(
+            [schema],
+            tool_choice=tool_name,
+            structured_output_format={"kwargs": {}, "schema": formatted_tool},
+        )
         if isinstance(schema, type) and is_basemodel_subclass(schema):
             output_parser: OutputParserLike = PydanticToolsParser(
                 tools=[schema], first_tool_only=True

libs/partners/fireworks/langchain_fireworks/chat_models.py

@@ -965,8 +965,16 @@ class AnswerWithJustification(BaseModel):
                     "schema must be specified when method is 'function_calling'. "
                     "Received None."
                 )
-            tool_name = convert_to_openai_tool(schema)["function"]["name"]
-            llm = self.bind_tools([schema], tool_choice=tool_name)
+            formatted_tool = convert_to_openai_tool(schema)
+            tool_name = formatted_tool["function"]["name"]
+            llm = self.bind_tools(
+                [schema],
+                tool_choice=tool_name,
+                structured_output_format={
+                    "kwargs": {"method": "function_calling"},
+                    "schema": formatted_tool,
+                },
+            )
             if is_pydantic_schema:
                 output_parser: OutputParserLike = PydanticToolsParser(
                     tools=[schema],  # type: ignore[list-item]
@@ -977,7 +985,13 @@ class AnswerWithJustification(BaseModel):
                     key_name=tool_name, first_tool_only=True
                 )
         elif method == "json_mode":
-            llm = self.bind(response_format={"type": "json_object"})
+            llm = self.bind(
+                response_format={"type": "json_object"},
+                structured_output_format={
+                    "kwargs": {"method": "json_mode"},
+                    "schema": schema,
+                },
+            )
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[type-var, arg-type]
                 if is_pydantic_schema

libs/partners/groq/langchain_groq/chat_models.py

@@ -996,8 +996,16 @@ class AnswerWithJustification(BaseModel):
                     "schema must be specified when method is 'function_calling'. "
                     "Received None."
                 )
-            tool_name = convert_to_openai_tool(schema)["function"]["name"]
-            llm = self.bind_tools([schema], tool_choice=tool_name)
+            formatted_tool = convert_to_openai_tool(schema)
+            tool_name = formatted_tool["function"]["name"]
+            llm = self.bind_tools(
+                [schema],
+                tool_choice=tool_name,
+                structured_output_format={
+                    "kwargs": {"method": "function_calling"},
+                    "schema": formatted_tool,
+                },
+            )
             if is_pydantic_schema:
                 output_parser: OutputParserLike = PydanticToolsParser(
                     tools=[schema],  # type: ignore[list-item]
@@ -1008,7 +1016,13 @@ class AnswerWithJustification(BaseModel):
                     key_name=tool_name, first_tool_only=True
                 )
         elif method == "json_mode":
-            llm = self.bind(response_format={"type": "json_object"})
+            llm = self.bind(
+                response_format={"type": "json_object"},
+                structured_output_format={
+                    "kwargs": {"method": "json_mode"},
+                    "schema": schema,
+                },
+            )
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[type-var, arg-type]
                 if is_pydantic_schema

libs/partners/mistralai/langchain_mistralai/chat_models.py

@@ -931,7 +931,14 @@ class AnswerWithJustification(BaseModel):
                 )
             # TODO: Update to pass in tool name as tool_choice if/when Mistral supports
             # specifying a tool.
-            llm = self.bind_tools([schema], tool_choice="any")
+            llm = self.bind_tools(
+                [schema],
+                tool_choice="any",
+                structured_output_format={
+                    "kwargs": {"method": "function_calling"},
+                    "schema": schema,
+                },
+            )
             if is_pydantic_schema:
                 output_parser: OutputParserLike = PydanticToolsParser(
                     tools=[schema],  # type: ignore[list-item]
@@ -943,7 +950,16 @@ class AnswerWithJustification(BaseModel):
                     key_name=key_name, first_tool_only=True
                 )
         elif method == "json_mode":
-            llm = self.bind(response_format={"type": "json_object"})
+            llm = self.bind(
+                response_format={"type": "json_object"},
+                structured_output_format={
+                    "kwargs": {
+                        # this is correct - name difference with mistral api
+                        "method": "json_mode"
+                    },
+                    "schema": schema,
+                },
+            )
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[type-var, arg-type]
                 if is_pydantic_schema
@@ -956,7 +972,13 @@ class AnswerWithJustification(BaseModel):
                     "Received None."
                 )
             response_format = _convert_to_openai_response_format(schema, strict=True)
-            llm = self.bind(response_format=response_format)
+            llm = self.bind(
+                response_format=response_format,
+                structured_output_format={
+                    "kwargs": {"method": "json_schema"},
+                    "schema": schema,
+                },
+            )
 
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[arg-type]

libs/partners/ollama/langchain_ollama/chat_models.py

@@ -1085,8 +1085,16 @@ class AnswerWithJustification(BaseModel):
                     "schema must be specified when method is not 'json_mode'. "
                     "Received None."
                 )
-            tool_name = convert_to_openai_tool(schema)["function"]["name"]
-            llm = self.bind_tools([schema], tool_choice=tool_name)
+            formatted_tool = convert_to_openai_tool(schema)
+            tool_name = formatted_tool["function"]["name"]
+            llm = self.bind_tools(
+                [schema],
+                tool_choice=tool_name,
+                structured_output_format={
+                    "kwargs": {"method": method},
+                    "schema": formatted_tool,
+                },
+            )
             if is_pydantic_schema:
                 output_parser: Runnable = PydanticToolsParser(
                     tools=[schema],  # type: ignore[list-item]
@@ -1097,7 +1105,13 @@ class AnswerWithJustification(BaseModel):
                     key_name=tool_name, first_tool_only=True
                 )
         elif method == "json_mode":
-            llm = self.bind(format="json")
+            llm = self.bind(
+                format="json",
+                structured_output_format={
+                    "kwargs": {"method": method},
+                    "schema": schema,
+                },
+            )
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[arg-type]
                 if is_pydantic_schema
@@ -1111,7 +1125,13 @@ class AnswerWithJustification(BaseModel):
                 )
             if is_pydantic_schema:
                 schema = cast(TypeBaseModel, schema)
-                llm = self.bind(format=schema.model_json_schema())
+                llm = self.bind(
+                    format=schema.model_json_schema(),
+                    structured_output_format={
+                        "kwargs": {"method": method},
+                        "schema": schema,
+                    },
+                )
                 output_parser = PydanticOutputParser(pydantic_object=schema)
             else:
                 if is_typeddict(schema):
@@ -1126,7 +1146,13 @@ class AnswerWithJustification(BaseModel):
                 else:
                     # is JSON schema
                     response_format = schema
-                llm = self.bind(format=response_format)
+                llm = self.bind(
+                    format=response_format,
+                    structured_output_format={
+                        "kwargs": {"method": method},
+                        "schema": response_format,
+                    },
+                )
                 output_parser = JsonOutputParser()
         else:
             raise ValueError(

libs/partners/ollama/tests/integration_tests/chat_models/test_chat_models_standard.py

@@ -31,8 +31,8 @@ def supports_json_mode(self) -> bool:
             "Fails with 'AssertionError'. Ollama does not support 'tool_choice' yet."
         )
     )
-    def test_structured_output(self, model: BaseChatModel) -> None:
-        super().test_structured_output(model)
+    def test_structured_output(self, model: BaseChatModel, schema_type: str) -> None:
+        super().test_structured_output(model, schema_type)
 
     @pytest.mark.xfail(
         reason=(

libs/partners/openai/langchain_openai/chat_models/base.py

@@ -1390,7 +1390,13 @@ def with_structured_output(
                 )
             tool_name = convert_to_openai_tool(schema)["function"]["name"]
             bind_kwargs = self._filter_disabled_params(
-                tool_choice=tool_name, parallel_tool_calls=False, strict=strict
+                tool_choice=tool_name,
+                parallel_tool_calls=False,
+                strict=strict,
+                structured_output_format={
+                    "kwargs": {"method": method},
+                    "schema": schema,
+                },
             )
 
             llm = self.bind_tools([schema], **bind_kwargs)
@@ -1404,7 +1410,13 @@ def with_structured_output(
                     key_name=tool_name, first_tool_only=True
                 )
         elif method == "json_mode":
-            llm = self.bind(response_format={"type": "json_object"})
+            llm = self.bind(
+                response_format={"type": "json_object"},
+                structured_output_format={
+                    "kwargs": {"method": method},
+                    "schema": schema,
+                },
+            )
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[arg-type]
                 if is_pydantic_schema
@@ -1417,7 +1429,13 @@ def with_structured_output(
                     "Received None."
                 )
             response_format = _convert_to_openai_response_format(schema, strict=strict)
-            llm = self.bind(response_format=response_format)
+            llm = self.bind(
+                response_format=response_format,
+                structured_output_format={
+                    "kwargs": {"method": method},
+                    "schema": convert_to_openai_tool(schema),
+                },
+            )
             if is_pydantic_schema:
                 output_parser = _oai_structured_outputs_parser.with_types(
                     output_type=cast(type, schema)