From ad2afec4f2f620c6c525c5b24885dfd0dea0817f Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Fri, 24 Jan 2025 16:53:19 -0800
Subject: [PATCH 01/19] core,openai: rfc structured output tracing

---
 .../langchain_core/language_models/base.py     |  9 ++++++++-
 .../language_models/chat_models.py             | 18 +++++++++++++++++-
 .../langchain_openai/chat_models/base.py       | 15 ++++++++++++---
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/libs/core/langchain_core/language_models/base.py b/libs/core/langchain_core/language_models/base.py
index 12445f2560f53..b6fa3a4006730 100644
--- a/libs/core/langchain_core/language_models/base.py
+++ b/libs/core/langchain_core/language_models/base.py
@@ -49,6 +49,7 @@ class LangSmithParams(TypedDict, total=False):
     """Max tokens for generation."""
     ls_stop: Optional[list[str]]
     """Stop words for generation."""
+    structured_output_format: Optional[dict]
 
 
 @cache  # Cache the tokenizer
@@ -233,7 +234,13 @@ async def agenerate_prompt(
         """
 
     def with_structured_output(
-        self, schema: Union[dict, type], **kwargs: Any
+        self,
+        schema: Union[dict, type],
+        *,
+        method: Literal[
+            "function_calling", "json_mode", "json_schema"
+        ] = "function_calling",
+        **kwargs: Any,
     ) -> Runnable[LanguageModelInput, Union[dict, BaseModel]]:
         """Not implemented on this class."""
         # Implement this on child class if there is a way of steering the model to
diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.py
index 6aaaf7d4ca80a..9346fb4b0a260 100644
--- a/libs/core/langchain_core/language_models/chat_models.py
+++ b/libs/core/langchain_core/language_models/chat_models.py
@@ -553,6 +553,9 @@ def _get_ls_params(
         elif hasattr(self, "max_tokens") and isinstance(self.max_tokens, int):
             ls_params["ls_max_tokens"] = self.max_tokens
 
+        if "structured_output_format" in kwargs:
+            ls_params["structured_output_format"] = kwargs["structured_output_format"]
+
         return ls_params
 
     def _get_llm_string(self, stop: Optional[list[str]] = None, **kwargs: Any) -> str:
@@ -1123,6 +1126,9 @@ def with_structured_output(
         self,
         schema: Union[typing.Dict, type],  # noqa: UP006
         *,
+        method: Literal[
+            "function_calling", "json_mode", "json_schema"
+        ] = "function_calling",
         include_raw: bool = False,
         **kwargs: Any,
     ) -> Runnable[LanguageModelInput, Union[typing.Dict, BaseModel]]:  # noqa: UP006
@@ -1240,7 +1246,17 @@ class AnswerWithJustification(BaseModel):
         if self.bind_tools is BaseChatModel.bind_tools:
             msg = "with_structured_output is not implemented for this model."
             raise NotImplementedError(msg)
-        llm = self.bind_tools([schema], tool_choice="any")
+
+        # default implementation only supports function_calling as method
+        if method != "function_calling":
+            msg = "Only method='function_calling' is supported by this model."
+            raise ValueError(msg)
+
+        llm = self.bind_tools(
+            [schema],
+            tool_choice="any",
+            structured_output_format={"method": method, "schema": schema},
+        )
         if isinstance(schema, type) and is_basemodel_subclass(schema):
             output_parser: OutputParserLike = PydanticToolsParser(
                 tools=[cast(TypeBaseModel, schema)], first_tool_only=True
diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py
index 982979e8949a0..0a6ddc9498264 100644
--- a/libs/partners/openai/langchain_openai/chat_models/base.py
+++ b/libs/partners/openai/langchain_openai/chat_models/base.py
@@ -1390,7 +1390,10 @@ def with_structured_output(
                 )
             tool_name = convert_to_openai_tool(schema)["function"]["name"]
             bind_kwargs = self._filter_disabled_params(
-                tool_choice=tool_name, parallel_tool_calls=False, strict=strict
+                tool_choice=tool_name,
+                parallel_tool_calls=False,
+                strict=strict,
+                structured_output_format={"method": method, "schema": schema},
             )
 
             llm = self.bind_tools([schema], **bind_kwargs)
@@ -1404,7 +1407,10 @@ def with_structured_output(
                     key_name=tool_name, first_tool_only=True
                 )
         elif method == "json_mode":
-            llm = self.bind(response_format={"type": "json_object"})
+            llm = self.bind(
+                response_format={"type": "json_object"},
+                structured_output_format={"method": method, "schema": schema},
+            )
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[arg-type]
                 if is_pydantic_schema
@@ -1417,7 +1423,10 @@ def with_structured_output(
                     "Received None."
                 )
             response_format = _convert_to_openai_response_format(schema, strict=strict)
-            llm = self.bind(response_format=response_format)
+            llm = self.bind(
+                response_format=response_format,
+                structured_output_format={"method": method, "schema": schema},
+            )
             if is_pydantic_schema:
                 output_parser = _oai_structured_outputs_parser.with_types(
                     output_type=cast(type, schema)

From 231d279ff94b87834ddc180eb3720f92c893b2aa Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Fri, 24 Jan 2025 17:40:12 -0800
Subject: [PATCH 02/19] x

---
 .../core/langchain_core/language_models/base.py |  1 -
 .../language_models/chat_models.py              | 17 ++++++++++++++---
 .../openai/langchain_openai/chat_models/base.py |  3 ++-
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/libs/core/langchain_core/language_models/base.py b/libs/core/langchain_core/language_models/base.py
index b6fa3a4006730..ecdadc621a708 100644
--- a/libs/core/langchain_core/language_models/base.py
+++ b/libs/core/langchain_core/language_models/base.py
@@ -49,7 +49,6 @@ class LangSmithParams(TypedDict, total=False):
     """Max tokens for generation."""
     ls_stop: Optional[list[str]]
     """Stop words for generation."""
-    structured_output_format: Optional[dict]
 
 
 @cache  # Cache the tokenizer
diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.py
index 9346fb4b0a260..7bbc5b31f76d0 100644
--- a/libs/core/langchain_core/language_models/chat_models.py
+++ b/libs/core/langchain_core/language_models/chat_models.py
@@ -553,9 +553,6 @@ def _get_ls_params(
         elif hasattr(self, "max_tokens") and isinstance(self.max_tokens, int):
             ls_params["ls_max_tokens"] = self.max_tokens
 
-        if "structured_output_format" in kwargs:
-            ls_params["structured_output_format"] = kwargs["structured_output_format"]
-
         return ls_params
 
     def _get_llm_string(self, stop: Optional[list[str]] = None, **kwargs: Any) -> str:
@@ -609,11 +606,25 @@ def generate(
             An LLMResult, which contains a list of candidate Generations for each input
                 prompt and additional model provider-specific output.
         """
+        structured_output_format = kwargs.pop("structured_output_format", None)
+        if False and structured_output_format:
+            structured_output_format_dict = {
+                "structured_output_format": {
+                    "method": structured_output_format["method"],
+                    "schema": convert_to_openai_tool(
+                        structured_output_format["schema"]
+                    ),
+                }
+            }
+        else:
+            structured_output_format_dict = {}
+
         params = self._get_invocation_params(stop=stop, **kwargs)
         options = {"stop": stop}
         inheritable_metadata = {
             **(metadata or {}),
             **self._get_ls_params(stop=stop, **kwargs),
+            **structured_output_format_dict,
         }
 
         callback_manager = CallbackManager.configure(
diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py
index 0a6ddc9498264..6776f5227feb2 100644
--- a/libs/partners/openai/langchain_openai/chat_models/base.py
+++ b/libs/partners/openai/langchain_openai/chat_models/base.py
@@ -1423,9 +1423,10 @@ def with_structured_output(
                     "Received None."
                 )
             response_format = _convert_to_openai_response_format(schema, strict=strict)
+            print("HERE")
             llm = self.bind(
                 response_format=response_format,
-                structured_output_format={"method": method, "schema": schema},
+                # structured_output_format={"method": method, "schema": convert_to_openai_tool(schema)},
             )
             if is_pydantic_schema:
                 output_parser = _oai_structured_outputs_parser.with_types(

From 3a487c748a3d83dc1699bfacaba4e270c382a7d1 Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Fri, 24 Jan 2025 17:40:27 -0800
Subject: [PATCH 03/19] x

---
 libs/partners/openai/langchain_openai/chat_models/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py
index 6776f5227feb2..b0e529e8b3c6f 100644
--- a/libs/partners/openai/langchain_openai/chat_models/base.py
+++ b/libs/partners/openai/langchain_openai/chat_models/base.py
@@ -1425,7 +1425,7 @@ def with_structured_output(
             response_format = _convert_to_openai_response_format(schema, strict=strict)
             print("HERE")
             llm = self.bind(
-                response_format=response_format,
+                response_format=response_format
                 # structured_output_format={"method": method, "schema": convert_to_openai_tool(schema)},
             )
             if is_pydantic_schema:

From b8dd24b6bb5c7426fc37ea76db3ca0b19eba081a Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Fri, 24 Jan 2025 17:44:51 -0800
Subject: [PATCH 04/19] x

---
 libs/core/langchain_core/language_models/chat_models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.py
index 7bbc5b31f76d0..6494f1d84bac7 100644
--- a/libs/core/langchain_core/language_models/chat_models.py
+++ b/libs/core/langchain_core/language_models/chat_models.py
@@ -607,7 +607,7 @@ def generate(
                 prompt and additional model provider-specific output.
         """
         structured_output_format = kwargs.pop("structured_output_format", None)
-        if False and structured_output_format:
+        if structured_output_format:
             structured_output_format_dict = {
                 "structured_output_format": {
                     "method": structured_output_format["method"],

From 96987501ecaaabafe0a870ecf5e539f65c862c3c Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Tue, 28 Jan 2025 16:53:24 -0800
Subject: [PATCH 05/19] x

---
 .../langchain_core/language_models/base.py     |  8 +-------
 .../language_models/chat_models.py             | 12 ++----------
 .../langchain_openai/chat_models/base.py       | 18 +++++++++++++-----
 3 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/libs/core/langchain_core/language_models/base.py b/libs/core/langchain_core/language_models/base.py
index ecdadc621a708..12445f2560f53 100644
--- a/libs/core/langchain_core/language_models/base.py
+++ b/libs/core/langchain_core/language_models/base.py
@@ -233,13 +233,7 @@ async def agenerate_prompt(
         """
 
     def with_structured_output(
-        self,
-        schema: Union[dict, type],
-        *,
-        method: Literal[
-            "function_calling", "json_mode", "json_schema"
-        ] = "function_calling",
-        **kwargs: Any,
+        self, schema: Union[dict, type], **kwargs: Any
     ) -> Runnable[LanguageModelInput, Union[dict, BaseModel]]:
         """Not implemented on this class."""
         # Implement this on child class if there is a way of steering the model to
diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.py
index 6494f1d84bac7..bb3cd309db816 100644
--- a/libs/core/langchain_core/language_models/chat_models.py
+++ b/libs/core/langchain_core/language_models/chat_models.py
@@ -610,7 +610,7 @@ def generate(
         if structured_output_format:
             structured_output_format_dict = {
                 "structured_output_format": {
-                    "method": structured_output_format["method"],
+                    "kwargs": {"method": structured_output_format["method"]},
                     "schema": convert_to_openai_tool(
                         structured_output_format["schema"]
                     ),
@@ -1137,9 +1137,6 @@ def with_structured_output(
         self,
         schema: Union[typing.Dict, type],  # noqa: UP006
         *,
-        method: Literal[
-            "function_calling", "json_mode", "json_schema"
-        ] = "function_calling",
         include_raw: bool = False,
         **kwargs: Any,
     ) -> Runnable[LanguageModelInput, Union[typing.Dict, BaseModel]]:  # noqa: UP006
@@ -1258,15 +1255,10 @@ class AnswerWithJustification(BaseModel):
             msg = "with_structured_output is not implemented for this model."
             raise NotImplementedError(msg)
 
-        # default implementation only supports function_calling as method
-        if method != "function_calling":
-            msg = "Only method='function_calling' is supported by this model."
-            raise ValueError(msg)
-
         llm = self.bind_tools(
             [schema],
             tool_choice="any",
-            structured_output_format={"method": method, "schema": schema},
+            structured_output_format={"kwargs": {}, "schema": schema},
         )
         if isinstance(schema, type) and is_basemodel_subclass(schema):
             output_parser: OutputParserLike = PydanticToolsParser(
diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py
index b0e529e8b3c6f..5ac28a717acca 100644
--- a/libs/partners/openai/langchain_openai/chat_models/base.py
+++ b/libs/partners/openai/langchain_openai/chat_models/base.py
@@ -1393,7 +1393,10 @@ def with_structured_output(
                 tool_choice=tool_name,
                 parallel_tool_calls=False,
                 strict=strict,
-                structured_output_format={"method": method, "schema": schema},
+                structured_output_format={
+                    "kwargs": {"method": method},
+                    "schema": schema,
+                },
             )
 
             llm = self.bind_tools([schema], **bind_kwargs)
@@ -1409,7 +1412,10 @@ def with_structured_output(
         elif method == "json_mode":
             llm = self.bind(
                 response_format={"type": "json_object"},
-                structured_output_format={"method": method, "schema": schema},
+                structured_output_format={
+                    "kwargs": {"method": method},
+                    "schema": schema,
+                },
             )
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[arg-type]
@@ -1423,10 +1429,12 @@ def with_structured_output(
                     "Received None."
                 )
             response_format = _convert_to_openai_response_format(schema, strict=strict)
-            print("HERE")
             llm = self.bind(
-                response_format=response_format
-                # structured_output_format={"method": method, "schema": convert_to_openai_tool(schema)},
+                response_format=response_format,
+                structured_output_format={
+                    "kwargs": {"method": method},
+                    "schema": convert_to_openai_tool(schema),
+                },
             )
             if is_pydantic_schema:
                 output_parser = _oai_structured_outputs_parser.with_types(

From fdb0f067a99a475c8fbb4522e6c797671594aa15 Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Tue, 28 Jan 2025 18:23:33 -0800
Subject: [PATCH 06/19] tests

---
 .../integration_tests/chat_models.py          | 95 ++++++++++++++++++-
 1 file changed, 92 insertions(+), 3 deletions(-)

diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
index d1c3ab3e6e3d3..efb39d4726299 100644
--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
@@ -1,9 +1,11 @@
 import base64
 import json
-from typing import List, Optional, cast
+from typing import Any, List, Optional, cast
+from unittest.mock import MagicMock
 
 import httpx
 import pytest
+from langchain_core.callbacks import BaseCallbackHandler
 from langchain_core.language_models import BaseChatModel, GenericFakeChatModel
 from langchain_core.messages import (
     AIMessage,
@@ -1184,12 +1186,99 @@ def has_tool_calling(self) -> bool:
         Joke = _get_joke_class()
         # Pydantic class
         chat = model.with_structured_output(Joke, **self.structured_output_kwargs)
-        result = chat.invoke("Tell me a joke about cats.")
+        mock_callback = MagicMock()
+        mock_callback.on_chat_model_start = MagicMock()
+
+        class TestCallbackHandler(BaseCallbackHandler):
+            metadatas: list[dict | None]
+
+            def __init__(self):
+                super().__init__()
+                self.metadatas = []
+
+            def on_chat_model_start(
+                self,
+                serialized: Any,
+                messages: Any,
+                *,
+                metadata: dict[str, Any] | None = None,
+                **kwargs: Any,
+            ) -> None:
+                self.metadatas.append(metadata)
+
+        invoke_callback = TestCallbackHandler()
+
+        result = chat.invoke(
+            "Tell me a joke about cats.", config={"callbacks": [invoke_callback]}
+        )
+
+        assert (
+            len(invoke_callback.metadatas) == 1
+        ), "Expected on_chat_model_start to be called once"
+        assert isinstance(invoke_callback.metadatas[0], dict)
+        assert isinstance(
+            invoke_callback.metadatas[0]["structured_output_format"]["schema"], dict
+        )
+        assert invoke_callback.metadatas[0]["structured_output_format"]["schema"] == {
+            "type": "function",
+            "function": {
+                "name": "Joke",
+                "description": "Joke to tell user.",
+                "parameters": {
+                    "properties": {
+                        "setup": {
+                            "description": "question to set up a joke",
+                            "type": "string",
+                        },
+                        "punchline": {
+                            "description": "answer to resolve the joke",
+                            "type": "string",
+                        },
+                    },
+                    "required": ["setup", "punchline"],
+                    "type": "object",
+                },
+            },
+        }
+
         assert isinstance(result, Joke)
 
-        for chunk in chat.stream("Tell me a joke about cats."):
+        stream_callback = TestCallbackHandler()
+
+        for chunk in chat.stream(
+            "Tell me a joke about cats.", config={"callbacks": [stream_callback]}
+        ):
             assert isinstance(chunk, Joke)
 
+        assert (
+            len(stream_callback.metadatas) == 1
+        ), "Expected on_chat_model_start to be called once"
+        assert isinstance(stream_callback.metadatas[0], dict)
+        assert isinstance(
+            stream_callback.metadatas[0]["structured_output_format"]["schema"], dict
+        )
+        assert stream_callback.metadatas[0]["structured_output_format"]["schema"] == {
+            "type": "function",
+            "function": {
+                "name": "Joke",
+                "description": "Joke to tell user.",
+                "parameters": {
+                    "properties": {
+                        "setup": {
+                            "description": "question to set up a joke",
+                            "type": "string",
+                        },
+                        "punchline": {
+                            "description": "answer to resolve the joke",
+                            "type": "string",
+                        },
+                    },
+                    "required": ["setup", "punchline"],
+                    "type": "object",
+                },
+            },
+        }
+
         # Schema
         chat = model.with_structured_output(
             Joke.model_json_schema(), **self.structured_output_kwargs

From 5c3e5a6bc66ce8e86516f773b076ebb77da93749 Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Tue, 28 Jan 2025 18:35:19 -0800
Subject: [PATCH 07/19] x

---
 .../language_models/chat_models.py            |  45 ++++++-
 .../integration_tests/chat_models.py          | 110 ++++++++++++++----
 2 files changed, 133 insertions(+), 22 deletions(-)

diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.py
index bb3cd309db816..031b6f58c29b6 100644
--- a/libs/core/langchain_core/language_models/chat_models.py
+++ b/libs/core/langchain_core/language_models/chat_models.py
@@ -365,11 +365,25 @@ def stream(
         else:
             config = ensure_config(config)
             messages = self._convert_input(input).to_messages()
+            structured_output_format = kwargs.pop("structured_output_format", None)
+            if structured_output_format:
+                structured_output_format_dict = {
+                    "structured_output_format": {
+                        "kwargs": structured_output_format.get("kwargs", {}),
+                        "schema": convert_to_openai_tool(
+                            structured_output_format["schema"]
+                        ),
+                    }
+                }
+            else:
+                structured_output_format_dict = {}
+
             params = self._get_invocation_params(stop=stop, **kwargs)
             options = {"stop": stop, **kwargs}
             inheritable_metadata = {
                 **(config.get("metadata") or {}),
                 **self._get_ls_params(stop=stop, **kwargs),
+                **structured_output_format_dict,
             }
             callback_manager = CallbackManager.configure(
                 config.get("callbacks"),
@@ -441,11 +455,26 @@ async def astream(
 
         config = ensure_config(config)
         messages = self._convert_input(input).to_messages()
+
+        structured_output_format = kwargs.pop("structured_output_format", None)
+        if structured_output_format:
+            structured_output_format_dict = {
+                "structured_output_format": {
+                    "kwargs": structured_output_format.get("kwargs", {}),
+                    "schema": convert_to_openai_tool(
+                        structured_output_format["schema"]
+                    ),
+                }
+            }
+        else:
+            structured_output_format_dict = {}
+
         params = self._get_invocation_params(stop=stop, **kwargs)
         options = {"stop": stop, **kwargs}
         inheritable_metadata = {
             **(config.get("metadata") or {}),
             **self._get_ls_params(stop=stop, **kwargs),
+            **structured_output_format_dict,
         }
         callback_manager = AsyncCallbackManager.configure(
             config.get("callbacks"),
@@ -610,7 +639,7 @@ def generate(
         if structured_output_format:
             structured_output_format_dict = {
                 "structured_output_format": {
-                    "kwargs": {"method": structured_output_format["method"]},
+                    "kwargs": structured_output_format.get("kwargs", {}),
                     "schema": convert_to_openai_tool(
                         structured_output_format["schema"]
                     ),
@@ -711,11 +740,25 @@ async def agenerate(
             An LLMResult, which contains a list of candidate Generations for each input
                 prompt and additional model provider-specific output.
         """
+        structured_output_format = kwargs.pop("structured_output_format", None)
+        if structured_output_format:
+            structured_output_format_dict = {
+                "structured_output_format": {
+                    "kwargs": structured_output_format.get("kwargs", {}),
+                    "schema": convert_to_openai_tool(
+                        structured_output_format["schema"]
+                    ),
+                }
+            }
+        else:
+            structured_output_format_dict = {}
+
         params = self._get_invocation_params(stop=stop, **kwargs)
         options = {"stop": stop}
         inheritable_metadata = {
             **(metadata or {}),
             **self._get_ls_params(stop=stop, **kwargs),
+            **structured_output_format_dict,
         }
 
         callback_manager = AsyncCallbackManager.configure(
diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
index efb39d4726299..e025316166376 100644
--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
@@ -45,6 +45,24 @@ class Joke(BaseModel):
     return Joke
 
 
+class _TestCallbackHandler(BaseCallbackHandler):
+    metadatas: list[dict | None]
+
+    def __init__(self):
+        super().__init__()
+        self.metadatas = []
+
+    def on_chat_model_start(
+        self,
+        serialized: Any,
+        messages: Any,
+        *,
+        metadata: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        self.metadatas.append(metadata)
+
+
 class _MagicFunctionSchema(BaseModel):
     input: int = Field(..., gt=-1000, lt=1000)
 
@@ -1189,24 +1207,7 @@ def has_tool_calling(self) -> bool:
         mock_callback = MagicMock()
         mock_callback.on_chat_model_start = MagicMock()
 
-        class TestCallbackHandler(BaseCallbackHandler):
-            metadatas: list[dict | None]
-
-            def __init__(self):
-                super().__init__()
-                self.metadatas = []
-
-            def on_chat_model_start(
-                self,
-                serialized: Any,
-                messages: Any,
-                *,
-                metadata: dict[str, Any] | None = None,
-                **kwargs: Any,
-            ) -> None:
-                self.metadatas.append(metadata)
-
-        invoke_callback = TestCallbackHandler()
+        invoke_callback = _TestCallbackHandler()
 
         result = chat.invoke(
             "Tell me a joke about cats.", config={"callbacks": [invoke_callback]}
@@ -1243,7 +1244,7 @@ def on_chat_model_start(
 
         assert isinstance(result, Joke)
 
-        stream_callback = TestCallbackHandler()
+        stream_callback = _TestCallbackHandler()
 
         for chunk in chat.stream(
             "Tell me a joke about cats.", config={"callbacks": [stream_callback]}
@@ -1326,12 +1327,79 @@ def has_tool_calling(self) -> bool:
 
         # Pydantic class
         chat = model.with_structured_output(Joke, **self.structured_output_kwargs)
-        result = await chat.ainvoke("Tell me a joke about cats.")
+        ainvoke_callback = _TestCallbackHandler()
+
+        result = await chat.ainvoke(
+            "Tell me a joke about cats.", config={"callbacks": [ainvoke_callback]}
+        )
         assert isinstance(result, Joke)
 
-        async for chunk in chat.astream("Tell me a joke about cats."):
+        assert (
+            len(ainvoke_callback.metadatas) == 1
+        ), "Expected on_chat_model_start to be called once"
+        assert isinstance(ainvoke_callback.metadatas[0], dict)
+        assert isinstance(
+            ainvoke_callback.metadatas[0]["structured_output_format"]["schema"], dict
+        )
+        assert ainvoke_callback.metadatas[0]["structured_output_format"]["schema"] == {
+            "type": "function",
+            "function": {
+                "name": "Joke",
+                "description": "Joke to tell user.",
+                "parameters": {
+                    "properties": {
+                        "setup": {
+                            "description": "question to set up a joke",
+                            "type": "string",
+                        },
+                        "punchline": {
+                            "description": "answer to resolve the joke",
+                            "type": "string",
+                        },
+                    },
+                    "required": ["setup", "punchline"],
+                    "type": "object",
+                },
+            },
+        }
+
+        astream_callback = _TestCallbackHandler()
+
+        async for chunk in chat.astream(
+            "Tell me a joke about cats.", config={"callbacks": [astream_callback]}
+        ):
             assert isinstance(chunk, Joke)
 
+        assert (
+            len(astream_callback.metadatas) == 1
+        ), "Expected on_chat_model_start to be called once"
+
+        assert isinstance(astream_callback.metadatas[0], dict)
+        assert isinstance(
+            astream_callback.metadatas[0]["structured_output_format"]["schema"], dict
+        )
+        assert astream_callback.metadatas[0]["structured_output_format"]["schema"] == {
+            "type": "function",
+            "function": {
+                "name": "Joke",
+                "description": "Joke to tell user.",
+                "parameters": {
+                    "properties": {
+                        "setup": {
+                            "description": "question to set up a joke",
+                            "type": "string",
+                        },
+                        "punchline": {
+                            "description": "answer to resolve the joke",
+                            "type": "string",
+                        },
+                    },
+                    "required": ["setup", "punchline"],
+                    "type": "object",
+                },
+            },
+        }
+
         # Schema
         chat = model.with_structured_output(
             Joke.model_json_schema(), **self.structured_output_kwargs

From b41087414462d3cec6a2452fd3358518fd8019da Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Tue, 28 Jan 2025 18:40:27 -0800
Subject: [PATCH 08/19] anthropic

---
 .../anthropic/langchain_anthropic/chat_models.py       | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/libs/partners/anthropic/langchain_anthropic/chat_models.py b/libs/partners/anthropic/langchain_anthropic/chat_models.py
index d3d800ed28f58..e58675728df9f 100644
--- a/libs/partners/anthropic/langchain_anthropic/chat_models.py
+++ b/libs/partners/anthropic/langchain_anthropic/chat_models.py
@@ -1111,9 +1111,13 @@ class AnswerWithJustification(BaseModel):
                 Added support for TypedDict class as `schema`.
 
         """  # noqa: E501
-
-        tool_name = convert_to_anthropic_tool(schema)["name"]
-        llm = self.bind_tools([schema], tool_choice=tool_name)
+        formatted_tool = convert_to_anthropic_tool(schema)
+        tool_name = formatted_tool["name"]
+        llm = self.bind_tools(
+            [schema],
+            tool_choice=tool_name,
+            structured_output_format={"kwargs": {}, "schema": formatted_tool},
+        )
         if isinstance(schema, type) and is_basemodel_subclass(schema):
             output_parser: OutputParserLike = PydanticToolsParser(
                 tools=[schema], first_tool_only=True

From 3051bac32d41a200dde8075a735a088b85650bf2 Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Tue, 28 Jan 2025 18:52:43 -0800
Subject: [PATCH 09/19] x

---
 .../langchain_fireworks/chat_models.py        | 20 +++++++++--
 .../groq/langchain_groq/chat_models.py        | 20 +++++++++--
 .../langchain_mistralai/chat_models.py        | 27 ++++++++++++--
 .../ollama/langchain_ollama/chat_models.py    | 36 ++++++++++++++++---
 4 files changed, 89 insertions(+), 14 deletions(-)

diff --git a/libs/partners/fireworks/langchain_fireworks/chat_models.py b/libs/partners/fireworks/langchain_fireworks/chat_models.py
index 9297710470612..42bed993623aa 100644
--- a/libs/partners/fireworks/langchain_fireworks/chat_models.py
+++ b/libs/partners/fireworks/langchain_fireworks/chat_models.py
@@ -965,8 +965,16 @@ class AnswerWithJustification(BaseModel):
                     "schema must be specified when method is 'function_calling'. "
                     "Received None."
                 )
-            tool_name = convert_to_openai_tool(schema)["function"]["name"]
-            llm = self.bind_tools([schema], tool_choice=tool_name)
+            formatted_tool = convert_to_openai_tool(schema)
+            tool_name = formatted_tool["function"]["name"]
+            llm = self.bind_tools(
+                [schema],
+                tool_choice=tool_name,
+                structured_output_format={
+                    "kwargs": {"method": "function_calling"},
+                    "schema": formatted_tool,
+                },
+            )
             if is_pydantic_schema:
                 output_parser: OutputParserLike = PydanticToolsParser(
                     tools=[schema],  # type: ignore[list-item]
@@ -977,7 +985,13 @@ class AnswerWithJustification(BaseModel):
                     key_name=tool_name, first_tool_only=True
                 )
         elif method == "json_mode":
-            llm = self.bind(response_format={"type": "json_object"})
+            llm = self.bind(
+                response_format={"type": "json_object"},
+                structured_output_format={
+                    "kwargs": {"method": "json_mode"},
+                    "schema": schema,
+                },
+            )
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[type-var, arg-type]
                 if is_pydantic_schema
diff --git a/libs/partners/groq/langchain_groq/chat_models.py b/libs/partners/groq/langchain_groq/chat_models.py
index 45e3cf792b323..962f72979b4da 100644
--- a/libs/partners/groq/langchain_groq/chat_models.py
+++ b/libs/partners/groq/langchain_groq/chat_models.py
@@ -996,8 +996,16 @@ class AnswerWithJustification(BaseModel):
                     "schema must be specified when method is 'function_calling'. "
                     "Received None."
                 )
-            tool_name = convert_to_openai_tool(schema)["function"]["name"]
-            llm = self.bind_tools([schema], tool_choice=tool_name)
+            formatted_tool = convert_to_openai_tool(schema)
+            tool_name = formatted_tool["function"]["name"]
+            llm = self.bind_tools(
+                [schema],
+                tool_choice=tool_name,
+                structured_output_format={
+                    "kwargs": {"method": "function_calling"},
+                    "schema": formatted_tool,
+                },
+            )
             if is_pydantic_schema:
                 output_parser: OutputParserLike = PydanticToolsParser(
                     tools=[schema],  # type: ignore[list-item]
@@ -1008,7 +1016,13 @@ class AnswerWithJustification(BaseModel):
                     key_name=tool_name, first_tool_only=True
                 )
         elif method == "json_mode":
-            llm = self.bind(response_format={"type": "json_object"})
+            llm = self.bind(
+                response_format={"type": "json_object"},
+                structured_output_format={
+                    "kwargs": {"method": "json_mode"},
+                    "schema": schema,
+                },
+            )
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[type-var, arg-type]
                 if is_pydantic_schema
diff --git a/libs/partners/mistralai/langchain_mistralai/chat_models.py b/libs/partners/mistralai/langchain_mistralai/chat_models.py
index 78739a5fea17b..2772b393f98b0 100644
--- a/libs/partners/mistralai/langchain_mistralai/chat_models.py
+++ b/libs/partners/mistralai/langchain_mistralai/chat_models.py
@@ -931,7 +931,14 @@ class AnswerWithJustification(BaseModel):
                 )
             # TODO: Update to pass in tool name as tool_choice if/when Mistral supports
             # specifying a tool.
-            llm = self.bind_tools([schema], tool_choice="any")
+            llm = self.bind_tools(
+                [schema],
+                tool_choice="any",
+                structured_output_format={
+                    "kwargs": {"method": "function_calling"},
+                    "schema": schema,
+                },
+            )
             if is_pydantic_schema:
                 output_parser: OutputParserLike = PydanticToolsParser(
                     tools=[schema],  # type: ignore[list-item]
@@ -943,7 +950,15 @@ class AnswerWithJustification(BaseModel):
                     key_name=key_name, first_tool_only=True
                 )
         elif method == "json_mode":
-            llm = self.bind(response_format={"type": "json_object"})
+            llm = self.bind(
+                response_format={"type": "json_object"},
+                structured_output_format={
+                    "kwargs": {
+                        "method": "json_mode"
+                    },  # this is correct - name difference between interface and mistral
+                    "schema": schema,
+                },
+            )
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[type-var, arg-type]
                 if is_pydantic_schema
@@ -956,7 +971,13 @@ class AnswerWithJustification(BaseModel):
                     "Received None."
                 )
             response_format = _convert_to_openai_response_format(schema, strict=True)
-            llm = self.bind(response_format=response_format)
+            llm = self.bind(
+                response_format=response_format,
+                structured_output_format={
+                    "kwargs": {"method": "json_schema"},
+                    "schema": response_format,
+                },
+            )
 
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[arg-type]
diff --git a/libs/partners/ollama/langchain_ollama/chat_models.py b/libs/partners/ollama/langchain_ollama/chat_models.py
index b4f8c0f2d9aab..7a179b2fbedf4 100644
--- a/libs/partners/ollama/langchain_ollama/chat_models.py
+++ b/libs/partners/ollama/langchain_ollama/chat_models.py
@@ -1085,8 +1085,16 @@ class AnswerWithJustification(BaseModel):
                     "schema must be specified when method is not 'json_mode'. "
                     "Received None."
                 )
-            tool_name = convert_to_openai_tool(schema)["function"]["name"]
-            llm = self.bind_tools([schema], tool_choice=tool_name)
+            formatted_tool = convert_to_openai_tool(schema)
+            tool_name = formatted_tool["function"]["name"]
+            llm = self.bind_tools(
+                [schema],
+                tool_choice=tool_name,
+                structured_output_format={
+                    "kwargs": {"method": method},
+                    "schema": formatted_tool,
+                },
+            )
             if is_pydantic_schema:
                 output_parser: Runnable = PydanticToolsParser(
                     tools=[schema],  # type: ignore[list-item]
@@ -1097,7 +1105,13 @@ class AnswerWithJustification(BaseModel):
                     key_name=tool_name, first_tool_only=True
                 )
         elif method == "json_mode":
-            llm = self.bind(format="json")
+            llm = self.bind(
+                format="json",
+                structured_output_format={
+                    "kwargs": {"method": method},
+                    "schema": schema,
+                },
+            )
             output_parser = (
                 PydanticOutputParser(pydantic_object=schema)  # type: ignore[arg-type]
                 if is_pydantic_schema
@@ -1111,7 +1125,13 @@ class AnswerWithJustification(BaseModel):
                 )
             if is_pydantic_schema:
                 schema = cast(TypeBaseModel, schema)
-                llm = self.bind(format=schema.model_json_schema())
+                llm = self.bind(
+                    format=schema.model_json_schema(),
+                    structured_output_format={
+                        "kwargs": {"method": method},
+                        "schema": schema,
+                    },
+                )
                 output_parser = PydanticOutputParser(pydantic_object=schema)
             else:
                 if is_typeddict(schema):
@@ -1126,7 +1146,13 @@ class AnswerWithJustification(BaseModel):
                 else:
                     # is JSON schema
                     response_format = schema
-                llm = self.bind(format=response_format)
+                llm = self.bind(
+                    format=response_format,
+                    structured_output_format={
+                        "kwargs": {"method": method},
+                        "schema": response_format,
+                    },
+                )
                 output_parser = JsonOutputParser()
         else:
             raise ValueError(

From 239852714f8e2f3611e4fe19f406fba879405915 Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Tue, 28 Jan 2025 18:57:28 -0800
Subject: [PATCH 10/19] x

---
 .../language_models/chat_models.py              | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.py
index 031b6f58c29b6..ffa58cd9a839a 100644
--- a/libs/core/langchain_core/language_models/chat_models.py
+++ b/libs/core/langchain_core/language_models/chat_models.py
@@ -458,14 +458,17 @@ async def astream(
 
         structured_output_format = kwargs.pop("structured_output_format", None)
         if structured_output_format:
-            structured_output_format_dict = {
-                "structured_output_format": {
-                    "kwargs": structured_output_format.get("kwargs", {}),
-                    "schema": convert_to_openai_tool(
-                        structured_output_format["schema"]
-                    ),
+            try:
+                structured_output_format_dict = {
+                    "structured_output_format": {
+                        "kwargs": structured_output_format.get("kwargs", {}),
+                        "schema": convert_to_openai_tool(
+                            structured_output_format["schema"]
+                        ),
+                    }
                 }
-            }
+            except ValueError:
+                structured_output_format_dict = {}
         else:
             structured_output_format_dict = {}
 

From 75189d4c2540b85a5cddbfccb72179389db07a38 Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Tue, 28 Jan 2025 19:00:01 -0800
Subject: [PATCH 11/19] x

---
 .../langchain_tests/integration_tests/chat_models.py            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
index e025316166376..269d69e64b0a8 100644
--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
@@ -48,7 +48,7 @@ class Joke(BaseModel):
 class _TestCallbackHandler(BaseCallbackHandler):
     metadatas: list[dict | None]
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__()
         self.metadatas = []
 

From b5cc0eafd237599975dce6770cfbca6bfdd02d29 Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Tue, 28 Jan 2025 19:00:48 -0800
Subject: [PATCH 12/19] x

---
 libs/partners/mistralai/langchain_mistralai/chat_models.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libs/partners/mistralai/langchain_mistralai/chat_models.py b/libs/partners/mistralai/langchain_mistralai/chat_models.py
index 2772b393f98b0..a1e5734fa0934 100644
--- a/libs/partners/mistralai/langchain_mistralai/chat_models.py
+++ b/libs/partners/mistralai/langchain_mistralai/chat_models.py
@@ -954,8 +954,9 @@ class AnswerWithJustification(BaseModel):
                 response_format={"type": "json_object"},
                 structured_output_format={
                     "kwargs": {
+                        # this is correct - name difference with mistral api
                         "method": "json_mode"
-                    },  # this is correct - name difference between interface and mistral
+                    },
                     "schema": schema,
                 },
             )

From 722624902380b10304bdc95432e0d37eb8b9a36a Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Tue, 28 Jan 2025 19:02:15 -0800
Subject: [PATCH 13/19] x

---
 .../langchain_tests/integration_tests/chat_models.py          | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
index 269d69e64b0a8..ef4c126199f2c 100644
--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
@@ -46,7 +46,7 @@ class Joke(BaseModel):
 
 
 class _TestCallbackHandler(BaseCallbackHandler):
-    metadatas: list[dict | None]
+    metadatas: list[Optional[dict]]
 
     def __init__(self) -> None:
         super().__init__()
@@ -57,7 +57,7 @@ def on_chat_model_start(
         serialized: Any,
         messages: Any,
         *,
-        metadata: dict[str, Any] | None = None,
+        metadata: Optional[dict[str, Any]] = None,
         **kwargs: Any,
     ) -> None:
         self.metadatas.append(metadata)

From 47c75f3d39eb6d773d669d3a62359175027c7e5e Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Tue, 28 Jan 2025 19:11:23 -0800
Subject: [PATCH 14/19] x

---
 .../integration_tests/chat_models.py          | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
index ef4c126199f2c..3da3424c5f400 100644
--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
@@ -1213,9 +1213,9 @@ def has_tool_calling(self) -> bool:
             "Tell me a joke about cats.", config={"callbacks": [invoke_callback]}
         )
 
-        assert (
-            len(invoke_callback.metadatas) == 1
-        ), "Expected on_chat_model_start to be called once"
+        assert len(invoke_callback.metadatas) == 1, (
+            "Expected on_chat_model_start to be called once"
+        )
         assert isinstance(invoke_callback.metadatas[0], dict)
         assert isinstance(
             invoke_callback.metadatas[0]["structured_output_format"]["schema"], dict
@@ -1251,9 +1251,9 @@ def has_tool_calling(self) -> bool:
         ):
             assert isinstance(chunk, Joke)
 
-        assert (
-            len(stream_callback.metadatas) == 1
-        ), "Expected on_chat_model_start to be called once"
+        assert len(stream_callback.metadatas) == 1, (
+            "Expected on_chat_model_start to be called once"
+        )
         assert isinstance(stream_callback.metadatas[0], dict)
         assert isinstance(
             stream_callback.metadatas[0]["structured_output_format"]["schema"], dict
@@ -1334,9 +1334,9 @@ def has_tool_calling(self) -> bool:
         )
         assert isinstance(result, Joke)
 
-        assert (
-            len(ainvoke_callback.metadatas) == 1
-        ), "Expected on_chat_model_start to be called once"
+        assert len(ainvoke_callback.metadatas) == 1, (
+            "Expected on_chat_model_start to be called once"
+        )
         assert isinstance(ainvoke_callback.metadatas[0], dict)
         assert isinstance(
             ainvoke_callback.metadatas[0]["structured_output_format"]["schema"], dict
@@ -1370,9 +1370,9 @@ def has_tool_calling(self) -> bool:
         ):
             assert isinstance(chunk, Joke)
 
-        assert (
-            len(astream_callback.metadatas) == 1
-        ), "Expected on_chat_model_start to be called once"
+        assert len(astream_callback.metadatas) == 1, (
+            "Expected on_chat_model_start to be called once"
+        )
 
         assert isinstance(astream_callback.metadatas[0], dict)
         assert isinstance(

From 1df2568c06c7de1dedd63c92538fbdd07321ae03 Mon Sep 17 00:00:00 2001
From: Chester Curme <chester.curme@gmail.com>
Date: Wed, 29 Jan 2025 15:00:11 -0500
Subject: [PATCH 15/19] fix

---
 .../integration_tests/chat_models.py              | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
index 5571224ced93c..290658e216a68 100644
--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
@@ -1,11 +1,7 @@
 import base64
 import json
-<<<<<<< HEAD
-from typing import Any, List, Optional, cast
-from unittest.mock import MagicMock
-=======
 from typing import Any, List, Literal, Optional, cast
->>>>>>> master
+from unittest.mock import MagicMock
 
 import httpx
 import pytest
@@ -1229,7 +1225,7 @@ def has_tool_calling(self) -> bool:
         if not self.has_tool_calling:
             pytest.skip("Test requires tool calling.")
 
-        schema, validation_function = _get_joke_class(schema_type) # type: ignore[arg-type]
+        schema, validation_function = _get_joke_class(schema_type)  # type: ignore[arg-type]
         chat = model.with_structured_output(schema, **self.structured_output_kwargs)
         mock_callback = MagicMock()
         mock_callback.on_chat_model_start = MagicMock()
@@ -1308,7 +1304,9 @@ def has_tool_calling(self) -> bool:
         }
 
     @pytest.mark.parametrize("schema_type", ["pydantic", "typeddict", "json_schema"])
-    async def test_structured_output_async(self, model: BaseChatModel, schema_type: str) -> None:
+    async def test_structured_output_async(
+        self, model: BaseChatModel, schema_type: str
+    ) -> None:
         """Test to verify structured output is generated both on invoke and stream.
 
         This test is optional and should be skipped if the model does not support
@@ -1338,9 +1336,8 @@ def has_tool_calling(self) -> bool:
         if not self.has_tool_calling:
             pytest.skip("Test requires tool calling.")
 
-        schema, validation_function = _get_joke_class(schema_type) # type: ignore[arg-type]
+        schema, validation_function = _get_joke_class(schema_type)  # type: ignore[arg-type]
 
-        # Pydantic class
         chat = model.with_structured_output(schema, **self.structured_output_kwargs)
         ainvoke_callback = _TestCallbackHandler()
 

From aea79973a2ec6a821f9b72c1918a55c971b6b0e2 Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Wed, 29 Jan 2025 12:19:13 -0800
Subject: [PATCH 16/19] x

---
 .../language_models/chat_models.py            | 51 +++++++++++--------
 1 file changed, 30 insertions(+), 21 deletions(-)

diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.py
index ffa58cd9a839a..dca8e9edaea1e 100644
--- a/libs/core/langchain_core/language_models/chat_models.py
+++ b/libs/core/langchain_core/language_models/chat_models.py
@@ -367,14 +367,17 @@ def stream(
             messages = self._convert_input(input).to_messages()
             structured_output_format = kwargs.pop("structured_output_format", None)
             if structured_output_format:
-                structured_output_format_dict = {
-                    "structured_output_format": {
-                        "kwargs": structured_output_format.get("kwargs", {}),
-                        "schema": convert_to_openai_tool(
-                            structured_output_format["schema"]
-                        ),
+                try:
+                    structured_output_format_dict = {
+                        "structured_output_format": {
+                            "kwargs": structured_output_format.get("kwargs", {}),
+                            "schema": convert_to_openai_tool(
+                                structured_output_format["schema"]
+                            ),
+                        }
                     }
-                }
+                except ValueError:
+                    structured_output_format_dict = {}
             else:
                 structured_output_format_dict = {}
 
@@ -640,14 +643,17 @@ def generate(
         """
         structured_output_format = kwargs.pop("structured_output_format", None)
         if structured_output_format:
-            structured_output_format_dict = {
-                "structured_output_format": {
-                    "kwargs": structured_output_format.get("kwargs", {}),
-                    "schema": convert_to_openai_tool(
-                        structured_output_format["schema"]
-                    ),
+            try:
+                structured_output_format_dict = {
+                    "structured_output_format": {
+                        "kwargs": structured_output_format.get("kwargs", {}),
+                        "schema": convert_to_openai_tool(
+                            structured_output_format["schema"]
+                        ),
+                    }
                 }
-            }
+            except ValueError:
+                structured_output_format_dict = {}
         else:
             structured_output_format_dict = {}
 
@@ -745,14 +751,17 @@ async def agenerate(
         """
         structured_output_format = kwargs.pop("structured_output_format", None)
         if structured_output_format:
-            structured_output_format_dict = {
-                "structured_output_format": {
-                    "kwargs": structured_output_format.get("kwargs", {}),
-                    "schema": convert_to_openai_tool(
-                        structured_output_format["schema"]
-                    ),
+            try:
+                structured_output_format_dict = {
+                    "structured_output_format": {
+                        "kwargs": structured_output_format.get("kwargs", {}),
+                        "schema": convert_to_openai_tool(
+                            structured_output_format["schema"]
+                        ),
+                    }
                 }
-            }
+            except ValueError:
+                structured_output_format_dict = {}
         else:
             structured_output_format_dict = {}
 

From 3af8b415e97d6235dc77c7ffc5397ba5379cf58f Mon Sep 17 00:00:00 2001
From: Chester Curme <chester.curme@gmail.com>
Date: Wed, 29 Jan 2025 15:19:48 -0500
Subject: [PATCH 17/19] fix test

---
 .../integration_tests/chat_models.py          | 101 +++---------------
 1 file changed, 16 insertions(+), 85 deletions(-)

diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
index 290658e216a68..f7371c6bcf784 100644
--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
@@ -19,7 +19,10 @@
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.prompts import ChatPromptTemplate
 from langchain_core.tools import BaseTool, tool
-from langchain_core.utils.function_calling import tool_example_to_messages
+from langchain_core.utils.function_calling import (
+    convert_to_openai_tool,
+    tool_example_to_messages,
+)
 from pydantic import BaseModel, Field
 from pydantic.v1 import BaseModel as BaseModelV1
 from pydantic.v1 import Field as FieldV1
@@ -1244,27 +1247,9 @@ def has_tool_calling(self) -> bool:
         assert isinstance(
             invoke_callback.metadatas[0]["structured_output_format"]["schema"], dict
         )
-        assert invoke_callback.metadatas[0]["structured_output_format"]["schema"] == {
-            "type": "function",
-            "function": {
-                "name": "Joke",
-                "description": "Joke to tell user.",
-                "parameters": {
-                    "properties": {
-                        "setup": {
-                            "description": "question to set up a joke",
-                            "type": "string",
-                        },
-                        "punchline": {
-                            "description": "answer to resolve the joke",
-                            "type": "string",
-                        },
-                    },
-                    "required": ["setup", "punchline"],
-                    "type": "object",
-                },
-            },
-        }
+        assert invoke_callback.metadatas[0]["structured_output_format"][
+            "schema"
+        ] == convert_to_openai_tool(schema)
 
         stream_callback = _TestCallbackHandler()
 
@@ -1281,27 +1266,9 @@ def has_tool_calling(self) -> bool:
         assert isinstance(
             stream_callback.metadatas[0]["structured_output_format"]["schema"], dict
         )
-        assert stream_callback.metadatas[0]["structured_output_format"]["schema"] == {
-            "type": "function",
-            "function": {
-                "name": "Joke",
-                "description": "Joke to tell user.",
-                "parameters": {
-                    "properties": {
-                        "setup": {
-                            "description": "question to set up a joke",
-                            "type": "string",
-                        },
-                        "punchline": {
-                            "description": "answer to resolve the joke",
-                            "type": "string",
-                        },
-                    },
-                    "required": ["setup", "punchline"],
-                    "type": "object",
-                },
-            },
-        }
+        assert stream_callback.metadatas[0]["structured_output_format"][
+            "schema"
+        ] == convert_to_openai_tool(schema)
 
     @pytest.mark.parametrize("schema_type", ["pydantic", "typeddict", "json_schema"])
     async def test_structured_output_async(
@@ -1353,27 +1320,9 @@ def has_tool_calling(self) -> bool:
         assert isinstance(
             ainvoke_callback.metadatas[0]["structured_output_format"]["schema"], dict
         )
-        assert ainvoke_callback.metadatas[0]["structured_output_format"]["schema"] == {
-            "type": "function",
-            "function": {
-                "name": "Joke",
-                "description": "Joke to tell user.",
-                "parameters": {
-                    "properties": {
-                        "setup": {
-                            "description": "question to set up a joke",
-                            "type": "string",
-                        },
-                        "punchline": {
-                            "description": "answer to resolve the joke",
-                            "type": "string",
-                        },
-                    },
-                    "required": ["setup", "punchline"],
-                    "type": "object",
-                },
-            },
-        }
+        assert ainvoke_callback.metadatas[0]["structured_output_format"][
+            "schema"
+        ] == convert_to_openai_tool(schema)
 
         astream_callback = _TestCallbackHandler()
 
@@ -1391,27 +1340,9 @@ def has_tool_calling(self) -> bool:
         assert isinstance(
             astream_callback.metadatas[0]["structured_output_format"]["schema"], dict
         )
-        assert astream_callback.metadatas[0]["structured_output_format"]["schema"] == {
-            "type": "function",
-            "function": {
-                "name": "Joke",
-                "description": "Joke to tell user.",
-                "parameters": {
-                    "properties": {
-                        "setup": {
-                            "description": "question to set up a joke",
-                            "type": "string",
-                        },
-                        "punchline": {
-                            "description": "answer to resolve the joke",
-                            "type": "string",
-                        },
-                    },
-                    "required": ["setup", "punchline"],
-                    "type": "object",
-                },
-            },
-        }
+        assert astream_callback.metadatas[0]["structured_output_format"][
+            "schema"
+        ] == convert_to_openai_tool(schema)
 
     @pytest.mark.skipif(PYDANTIC_MAJOR_VERSION != 2, reason="Test requires pydantic 2.")
     def test_structured_output_pydantic_2_v1(self, model: BaseChatModel) -> None:

From 3b59ef2c49a3d1b967b805f26cdca5aa994a4502 Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Wed, 29 Jan 2025 12:20:45 -0800
Subject: [PATCH 18/19] x

---
 libs/partners/mistralai/langchain_mistralai/chat_models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/partners/mistralai/langchain_mistralai/chat_models.py b/libs/partners/mistralai/langchain_mistralai/chat_models.py
index a1e5734fa0934..4e304e29e0db6 100644
--- a/libs/partners/mistralai/langchain_mistralai/chat_models.py
+++ b/libs/partners/mistralai/langchain_mistralai/chat_models.py
@@ -976,7 +976,7 @@ class AnswerWithJustification(BaseModel):
                 response_format=response_format,
                 structured_output_format={
                     "kwargs": {"method": "json_schema"},
-                    "schema": response_format,
+                    "schema": schema,
                 },
             )
 

From 81a232b63acc459500f37d4b360a306e20e4c9b2 Mon Sep 17 00:00:00 2001
From: Chester Curme <chester.curme@gmail.com>
Date: Wed, 29 Jan 2025 15:24:39 -0500
Subject: [PATCH 19/19] fix ollama typing

---
 .../chat_models/test_chat_models_standard.py                  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libs/partners/ollama/tests/integration_tests/chat_models/test_chat_models_standard.py b/libs/partners/ollama/tests/integration_tests/chat_models/test_chat_models_standard.py
index 4b8feccc8afdb..5f990f2251bb5 100644
--- a/libs/partners/ollama/tests/integration_tests/chat_models/test_chat_models_standard.py
+++ b/libs/partners/ollama/tests/integration_tests/chat_models/test_chat_models_standard.py
@@ -31,8 +31,8 @@ def supports_json_mode(self) -> bool:
             "Fails with 'AssertionError'. Ollama does not support 'tool_choice' yet."
         )
     )
-    def test_structured_output(self, model: BaseChatModel) -> None:
-        super().test_structured_output(model)
+    def test_structured_output(self, model: BaseChatModel, schema_type: str) -> None:
+        super().test_structured_output(model, schema_type)
 
     @pytest.mark.xfail(
         reason=(