diff --git a/docs/docs/how_to/index.mdx b/docs/docs/how_to/index.mdx index dad7eab3603af..5706bf002ecba 100644 --- a/docs/docs/how_to/index.mdx +++ b/docs/docs/how_to/index.mdx @@ -72,7 +72,7 @@ See [supported integrations](/docs/integrations/chat/) for details on getting st ### Example selectors -[Example Selectors](/docs/concepts/example_selectors) are responsible for selecting the correct few shot examples to pass to the prompt. +[Example Selectors](/docs/concepts/example_selectors) are responsible for selecting the correct few-shot examples to pass to the prompt. - [How to: use example selectors](/docs/how_to/example_selectors) - [How to: select examples by length](/docs/how_to/example_selectors_length_based) @@ -168,7 +168,7 @@ See [supported integrations](/docs/integrations/vectorstores/) for details on ge Indexing is the process of keeping your vectorstore in-sync with the underlying data source. -- [How to: reindex data to keep your vectorstore in sync with the underlying data source](/docs/how_to/indexing) +- [How to: reindex data to keep your vectorstore in-sync with the underlying data source](/docs/how_to/indexing) ### Tools diff --git a/libs/core/langchain_core/language_models/_utils.py b/libs/core/langchain_core/language_models/_utils.py index 883f8c855eab2..cb80fedb3dd2b 100644 --- a/libs/core/langchain_core/language_models/_utils.py +++ b/libs/core/langchain_core/language_models/_utils.py @@ -1,12 +1,30 @@ import re from collections.abc import Sequence -from typing import Optional +from typing import ( + TYPE_CHECKING, + Literal, + Optional, + TypedDict, + TypeVar, + Union, +) -from langchain_core.messages import BaseMessage +if TYPE_CHECKING: + from langchain_core.messages import BaseMessage +from langchain_core.messages.content import ( + ContentBlock, +) def _is_openai_data_block(block: dict) -> bool: - """Check if the block contains multimodal data in OpenAI Chat Completions format.""" + """Check if the block contains multimodal data in OpenAI Chat Completions format. + + Supports both data and ID-style blocks (e.g. ``'file_data'`` and ``'file_id'``) + + If additional keys are present, they are ignored / will not affect outcome as long + as the required keys are present and valid. + + """ if block.get("type") == "image_url": if ( (set(block.keys()) <= {"type", "image_url", "detail"}) @@ -15,29 +33,43 @@ def _is_openai_data_block(block: dict) -> bool: ): url = image_url.get("url") if isinstance(url, str): + # Required per OpenAI spec + return True + # Ignore `'detail'` since it's optional and specific to OpenAI + + elif block.get("type") == "input_audio": + if (audio := block.get("input_audio")) and isinstance(audio, dict): + audio_data = audio.get("data") + audio_format = audio.get("format") + # Both required per OpenAI spec + if isinstance(audio_data, str) and isinstance(audio_format, str): return True elif block.get("type") == "file": if (file := block.get("file")) and isinstance(file, dict): file_data = file.get("file_data") - if isinstance(file_data, str): - return True - - elif block.get("type") == "input_audio": - if (input_audio := block.get("input_audio")) and isinstance(input_audio, dict): - audio_data = input_audio.get("data") - audio_format = input_audio.get("format") - if isinstance(audio_data, str) and isinstance(audio_format, str): + file_id = file.get("file_id") + # Files can be either base64-encoded or pre-uploaded with an ID + if isinstance(file_data, str) or isinstance(file_id, str): return True else: return False + # Has no `'type'` key return False -def _parse_data_uri(uri: str) -> Optional[dict]: - """Parse a data URI into its components. If parsing fails, return None. +class ParsedDataUri(TypedDict): + source_type: Literal["base64"] + data: str + mime_type: str + + +def _parse_data_uri(uri: str) -> Optional[ParsedDataUri]: + """Parse a data URI into its components. + + If parsing fails, return None. If either MIME type or data is missing, return None. Example: @@ -57,84 +89,217 @@ def _parse_data_uri(uri: str) -> Optional[dict]: match = re.match(regex, uri) if match is None: return None + + mime_type = match.group("mime_type") + data = match.group("data") + if not mime_type or not data: + return None + return { "source_type": "base64", - "data": match.group("data"), - "mime_type": match.group("mime_type"), + "data": data, + "mime_type": mime_type, } -def _convert_openai_format_to_data_block(block: dict) -> dict: - """Convert OpenAI image content block to standard data content block. +def _normalize_messages( + messages: Sequence["BaseMessage"], +) -> list["BaseMessage"]: + """Normalize message formats to LangChain v1 standard content blocks. - If parsing fails, pass-through. + Chat models already implement support for: + - Images in OpenAI Chat Completions format + These will be passed through unchanged + - LangChain v1 standard content blocks - Args: - block: The OpenAI image content block to convert. + This function extends support to: + - `Audio `__ and + `file `__ data in OpenAI + Chat Completions format + - Images are technically supported but we expect chat models to handle them + directly; this may change in the future + - LangChain v0 standard content blocks for backward compatibility - Returns: - The converted standard data content block. - """ - if block["type"] == "image_url": - parsed = _parse_data_uri(block["image_url"]["url"]) - if parsed is not None: - parsed["type"] = "image" - return parsed - return block - - if block["type"] == "file": - parsed = _parse_data_uri(block["file"]["file_data"]) - if parsed is not None: - parsed["type"] = "file" - if filename := block["file"].get("filename"): - parsed["filename"] = filename - return parsed - return block - - if block["type"] == "input_audio": - data = block["input_audio"].get("data") - audio_format = block["input_audio"].get("format") - if data and audio_format: - return { - "type": "audio", - "source_type": "base64", - "data": data, - "mime_type": f"audio/{audio_format}", + .. versionchanged:: 1.0.0 + In previous versions, this function returned messages in LangChain v0 format. + Now, it returns messages in LangChain v1 format, which upgraded chat models now + expect to receive when passing back in message history. For backward + compatibility, this function will convert v0 message content to v1 format. + + .. dropdown:: v0 Content Block Schemas + + ``URLContentBlock``: + + .. codeblock:: + + { + mime_type: NotRequired[str] + type: Literal['image', 'audio', 'file'], + source_type: Literal['url'], + url: str, } - return block - return block + ``Base64ContentBlock``: + + .. codeblock:: + + { + mime_type: NotRequired[str] + type: Literal['image', 'audio', 'file'], + source_type: Literal['base64'], + data: str, + } + ``IDContentBlock``: -def _normalize_messages(messages: Sequence[BaseMessage]) -> list[BaseMessage]: - """Extend support for message formats. + (In practice, this was never used) + + .. codeblock:: + + { + type: Literal['image', 'audio', 'file'], + source_type: Literal['id'], + id: str, + } + + ``PlainTextContentBlock``: + + .. codeblock:: + + { + mime_type: NotRequired[str] + type: Literal['file'], + source_type: Literal['text'], + url: str, + } + + If a v1 message is passed in, it will be returned as-is, meaning it is safe to + always pass in v1 messages to this function for assurance. + + For posterity, here are the OpenAI Chat Completions schemas we expect: + + Chat Completions image. Can be URL-based or base64-encoded. Supports MIME types + png, jpeg/jpg, webp, static gif: + { + "type": Literal['image_url'], + "image_url": { + "url": Union["data:$MIME_TYPE;base64,$BASE64_ENCODED_IMAGE", "$IMAGE_URL"], + "detail": Literal['low', 'high', 'auto'] = 'auto', # Supported by OpenAI + } + } + + Chat Completions audio: + { + "type": Literal['input_audio'], + "input_audio": { + "format": Literal['wav', 'mp3'], + "data": str = "$BASE64_ENCODED_AUDIO", + }, + } + + Chat Completions files: either base64 or pre-uploaded file ID + { + "type": Literal['file'], + "file": Union[ + { + "filename": Optional[str] = "$FILENAME", + "file_data": str = "$BASE64_ENCODED_FILE", + }, + { + "file_id": str = "$FILE_ID", # For pre-uploaded files to OpenAI + }, + ], + } - Chat models implement support for images in OpenAI Chat Completions format, as well - as other multimodal data as standard data blocks. This function extends support to - audio and file data in OpenAI Chat Completions format by converting them to standard - data blocks. """ + from langchain_core.messages.block_translators.langchain_v0 import ( + _convert_legacy_v0_content_block_to_v1, + _convert_openai_format_to_data_block, + ) + formatted_messages = [] for message in messages: + # We preserve input messages - the caller may reuse them elsewhere and expects + # them to remain unchanged. We only create a copy if we need to translate. formatted_message = message + if isinstance(message.content, list): for idx, block in enumerate(message.content): + # OpenAI Chat Completions multimodal data blocks to v1 standard if ( isinstance(block, dict) - # Subset to (PDF) files and audio, as most relevant chat models - # support images in OAI format (and some may not yet support the - # standard data block format) - and block.get("type") in {"file", "input_audio"} + and block.get("type") in {"input_audio", "file"} + # Discriminate between OpenAI/LC format since they share `'type'` and _is_openai_data_block(block) ): - if formatted_message is message: - formatted_message = message.model_copy() - # Also shallow-copy content - formatted_message.content = list(formatted_message.content) - - formatted_message.content[idx] = ( # type: ignore[index] # mypy confused by .model_copy - _convert_openai_format_to_data_block(block) - ) + formatted_message = _ensure_message_copy(message, formatted_message) + + converted_block = _convert_openai_format_to_data_block(block) + _update_content_block(formatted_message, idx, converted_block) + + # Convert multimodal LangChain v0 to v1 standard content blocks + elif ( + isinstance(block, dict) + and block.get("type") + in { + "image", + "audio", + "file", + } + and block.get("source_type") # v1 doesn't have `source_type` + in { + "url", + "base64", + "id", + "text", + } + ): + formatted_message = _ensure_message_copy(message, formatted_message) + + converted_block = _convert_legacy_v0_content_block_to_v1(block) + _update_content_block(formatted_message, idx, converted_block) + continue + + # else, pass through blocks that look like they have v1 format unchanged + formatted_messages.append(formatted_message) return formatted_messages + + +T = TypeVar("T", bound="BaseMessage") + + +def _ensure_message_copy(message: T, formatted_message: T) -> T: + """Create a copy of the message if it hasn't been copied yet.""" + if formatted_message is message: + formatted_message = message.model_copy() + # Shallow-copy content list to allow modifications + formatted_message.content = list(formatted_message.content) + return formatted_message + + +def _update_content_block( + formatted_message: "BaseMessage", idx: int, new_block: Union[ContentBlock, dict] +) -> None: + """Update a content block at the given index, handling type issues.""" + # Type ignore needed because: + # - `BaseMessage.content` is typed as `Union[str, list[Union[str, dict]]]` + # - When content is str, indexing fails (index error) + # - When content is list, the items are `Union[str, dict]` but we're assigning + # `Union[ContentBlock, dict]` where ContentBlock is richer than dict + # - This is safe because we only call this when we've verified content is a list and + # we're doing content block conversions + formatted_message.content[idx] = new_block # type: ignore[index, assignment] + + +def _update_message_content_to_blocks(message: T, output_version: str) -> T: + return message.model_copy( + update={ + "content": message.content_blocks, + "response_metadata": { + **message.response_metadata, + "output_version": output_version, + }, + } + ) diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.py index 13c5394409a27..db486eb2c594e 100644 --- a/libs/core/langchain_core/language_models/chat_models.py +++ b/libs/core/langchain_core/language_models/chat_models.py @@ -27,7 +27,10 @@ Callbacks, ) from langchain_core.globals import get_llm_cache -from langchain_core.language_models._utils import _normalize_messages +from langchain_core.language_models._utils import ( + _normalize_messages, + _update_message_content_to_blocks, +) from langchain_core.language_models.base import ( BaseLanguageModel, LangSmithParams, @@ -41,11 +44,11 @@ BaseMessage, HumanMessage, convert_to_messages, + convert_to_openai_data_block, convert_to_openai_image_block, is_data_content_block, message_chunk_to_message, ) -from langchain_core.messages.ai import _LC_ID_PREFIX from langchain_core.outputs import ( ChatGeneration, ChatGenerationChunk, @@ -65,6 +68,7 @@ convert_to_openai_tool, ) from langchain_core.utils.pydantic import TypeBaseModel, is_basemodel_subclass +from langchain_core.utils.utils import LC_ID_PREFIX, from_env if TYPE_CHECKING: import uuid @@ -120,7 +124,7 @@ def _format_for_tracing(messages: list[BaseMessage]) -> list[BaseMessage]: if ( block.get("type") == "image" and is_data_content_block(block) - and block.get("source_type") != "id" + and not ("file_id" in block or block.get("source_type") == "id") ): if message_to_trace is message: # Shallow copy @@ -130,6 +134,19 @@ def _format_for_tracing(messages: list[BaseMessage]) -> list[BaseMessage]: message_to_trace.content[idx] = ( # type: ignore[index] # mypy confused by .model_copy convert_to_openai_image_block(block) ) + elif ( + block.get("type") == "file" + and is_data_content_block(block) + and "base64" in block + ): + if message_to_trace is message: + # Shallow copy + message_to_trace = message.model_copy() + message_to_trace.content = list(message_to_trace.content) + + message_to_trace.content[idx] = convert_to_openai_data_block( # type: ignore[index] + block + ) elif len(block) == 1 and "type" not in block: # Tracing assumes all content blocks have a "type" key. Here # we add this key if it is missing, and there's an obvious @@ -320,6 +337,28 @@ class BaseChatModel(BaseLanguageModel[AIMessage], ABC): """ + output_version: str = Field( + default_factory=from_env("LC_OUTPUT_VERSION", default="v0") + ) + """Version of ``AIMessage`` output format to store in message content. + + ``AIMessage.content_blocks`` will lazily parse the contents of ``content`` into a + standard format. This flag can be used to additionally store the standard format + in message content, e.g., for serialization purposes. + + Supported values: + + - ``"v0"``: provider-specific format in content (can lazily-parse with + ``.content_blocks``) + - ``"v1"``: standardized format in content (consistent with ``.content_blocks``) + + Partner packages (e.g., ``langchain-openai``) can also use this field to roll out + new content formats in a backward-compatible way. + + .. versionadded:: 1.0 + + """ + @model_validator(mode="before") @classmethod def raise_deprecation(cls, values: dict) -> Any: @@ -469,7 +508,7 @@ def stream( **kwargs: Any, ) -> Iterator[AIMessageChunk]: if not self._should_stream(async_api=False, **{**kwargs, "stream": True}): - # model doesn't implement streaming, so use default implementation + # Model doesn't implement streaming, so use default implementation yield cast( "AIMessageChunk", self.invoke(input, config=config, stop=stop, **kwargs), @@ -516,11 +555,16 @@ def stream( try: input_messages = _normalize_messages(messages) - run_id = "-".join((_LC_ID_PREFIX, str(run_manager.run_id))) + run_id = "-".join((LC_ID_PREFIX, str(run_manager.run_id))) for chunk in self._stream(input_messages, stop=stop, **kwargs): if chunk.message.id is None: chunk.message.id = run_id chunk.message.response_metadata = _gen_info_and_msg_metadata(chunk) + if self.output_version == "v1": + # Overwrite .content with .content_blocks + chunk.message = _update_message_content_to_blocks( + chunk.message, "v1" + ) run_manager.on_llm_new_token( cast("str", chunk.message.content), chunk=chunk ) @@ -609,7 +653,7 @@ async def astream( try: input_messages = _normalize_messages(messages) - run_id = "-".join((_LC_ID_PREFIX, str(run_manager.run_id))) + run_id = "-".join((LC_ID_PREFIX, str(run_manager.run_id))) async for chunk in self._astream( input_messages, stop=stop, @@ -618,6 +662,11 @@ async def astream( if chunk.message.id is None: chunk.message.id = run_id chunk.message.response_metadata = _gen_info_and_msg_metadata(chunk) + if self.output_version == "v1": + # Overwrite .content with .content_blocks + chunk.message = _update_message_content_to_blocks( + chunk.message, "v1" + ) await run_manager.on_llm_new_token( cast("str", chunk.message.content), chunk=chunk ) @@ -1075,7 +1124,12 @@ def _generate_with_cache( chunk.message.response_metadata = _gen_info_and_msg_metadata(chunk) if run_manager: if chunk.message.id is None: - chunk.message.id = f"{_LC_ID_PREFIX}-{run_manager.run_id}" + chunk.message.id = f"{LC_ID_PREFIX}-{run_manager.run_id}" + if self.output_version == "v1": + # Overwrite .content with .content_blocks + chunk.message = _update_message_content_to_blocks( + chunk.message, "v1" + ) run_manager.on_llm_new_token( cast("str", chunk.message.content), chunk=chunk ) @@ -1088,10 +1142,17 @@ def _generate_with_cache( else: result = self._generate(messages, stop=stop, **kwargs) + if self.output_version == "v1": + # Overwrite .content with .content_blocks + for generation in result.generations: + generation.message = _update_message_content_to_blocks( + generation.message, "v1" + ) + # Add response metadata to each generation for idx, generation in enumerate(result.generations): if run_manager and generation.message.id is None: - generation.message.id = f"{_LC_ID_PREFIX}-{run_manager.run_id}-{idx}" + generation.message.id = f"{LC_ID_PREFIX}-{run_manager.run_id}-{idx}" generation.message.response_metadata = _gen_info_and_msg_metadata( generation ) @@ -1148,7 +1209,12 @@ async def _agenerate_with_cache( chunk.message.response_metadata = _gen_info_and_msg_metadata(chunk) if run_manager: if chunk.message.id is None: - chunk.message.id = f"{_LC_ID_PREFIX}-{run_manager.run_id}" + chunk.message.id = f"{LC_ID_PREFIX}-{run_manager.run_id}" + if self.output_version == "v1": + # Overwrite .content with .content_blocks + chunk.message = _update_message_content_to_blocks( + chunk.message, "v1" + ) await run_manager.on_llm_new_token( cast("str", chunk.message.content), chunk=chunk ) @@ -1161,10 +1227,17 @@ async def _agenerate_with_cache( else: result = await self._agenerate(messages, stop=stop, **kwargs) + if self.output_version == "v1": + # Overwrite .content with .content_blocks + for generation in result.generations: + generation.message = _update_message_content_to_blocks( + generation.message, "v1" + ) + # Add response metadata to each generation for idx, generation in enumerate(result.generations): if run_manager and generation.message.id is None: - generation.message.id = f"{_LC_ID_PREFIX}-{run_manager.run_id}-{idx}" + generation.message.id = f"{LC_ID_PREFIX}-{run_manager.run_id}-{idx}" generation.message.response_metadata = _gen_info_and_msg_metadata( generation ) @@ -1211,6 +1284,7 @@ def _stream( run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs: Any, ) -> Iterator[ChatGenerationChunk]: + # We expect that subclasses implement this method if they support streaming. raise NotImplementedError async def _astream( diff --git a/libs/core/langchain_core/language_models/fake_chat_models.py b/libs/core/langchain_core/language_models/fake_chat_models.py index 3430db60f2d43..e0c49021fc861 100644 --- a/libs/core/langchain_core/language_models/fake_chat_models.py +++ b/libs/core/langchain_core/language_models/fake_chat_models.py @@ -19,7 +19,7 @@ class FakeMessagesListChatModel(BaseChatModel): - """Fake ChatModel for testing purposes.""" + """Fake ``ChatModel`` for testing purposes.""" responses: list[BaseMessage] """List of responses to **cycle** through in order.""" @@ -211,10 +211,11 @@ class GenericFakeChatModel(BaseChatModel): """Generic fake chat model that can be used to test the chat model interface. * Chat model should be usable in both sync and async tests - * Invokes on_llm_new_token to allow for testing of callback related code for new + * Invokes ``on_llm_new_token`` to allow for testing of callback related code for new tokens. * Includes logic to break messages into message chunk to facilitate testing of streaming. + """ messages: Iterator[Union[AIMessage, str]] @@ -229,6 +230,7 @@ class GenericFakeChatModel(BaseChatModel): .. warning:: Streaming is not implemented yet. We should try to implement it in the future by delegating to invoke and then breaking the resulting output into message chunks. + """ @override @@ -352,6 +354,7 @@ class ParrotFakeChatModel(BaseChatModel): """Generic fake chat model that can be used to test the chat model interface. * Chat model should be usable in both sync and async tests + """ @override diff --git a/libs/core/langchain_core/messages/__init__.py b/libs/core/langchain_core/messages/__init__.py index fe87e964af291..dfbf1ff3b7559 100644 --- a/libs/core/langchain_core/messages/__init__.py +++ b/libs/core/langchain_core/messages/__init__.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING from langchain_core._import_utils import import_attr +from langchain_core.utils.utils import LC_AUTO_PREFIX, LC_ID_PREFIX, ensure_id if TYPE_CHECKING: from langchain_core.messages.ai import ( @@ -32,10 +33,32 @@ messages_to_dict, ) from langchain_core.messages.chat import ChatMessage, ChatMessageChunk - from langchain_core.messages.content_blocks import ( + from langchain_core.messages.content import ( + Annotation, + AudioContentBlock, + Citation, + CodeInterpreterCall, + CodeInterpreterOutput, + CodeInterpreterResult, + ContentBlock, + DataContentBlock, + FileContentBlock, + ImageContentBlock, + NonStandardAnnotation, + NonStandardContentBlock, + PlainTextContentBlock, + ReasoningContentBlock, + TextContentBlock, + VideoContentBlock, + WebSearchCall, + WebSearchResult, convert_to_openai_data_block, convert_to_openai_image_block, is_data_content_block, + is_reasoning_block, + is_text_block, + is_tool_call_block, + is_tool_call_chunk, ) from langchain_core.messages.function import FunctionMessage, FunctionMessageChunk from langchain_core.messages.human import HumanMessage, HumanMessageChunk @@ -63,34 +86,59 @@ ) __all__ = ( + "LC_AUTO_PREFIX", + "LC_ID_PREFIX", "AIMessage", "AIMessageChunk", + "Annotation", "AnyMessage", + "AudioContentBlock", "BaseMessage", "BaseMessageChunk", "ChatMessage", "ChatMessageChunk", + "Citation", + "CodeInterpreterCall", + "CodeInterpreterOutput", + "CodeInterpreterResult", + "ContentBlock", + "DataContentBlock", + "FileContentBlock", "FunctionMessage", "FunctionMessageChunk", "HumanMessage", "HumanMessageChunk", + "ImageContentBlock", "InvalidToolCall", "MessageLikeRepresentation", + "NonStandardAnnotation", + "NonStandardContentBlock", + "PlainTextContentBlock", + "ReasoningContentBlock", "RemoveMessage", "SystemMessage", "SystemMessageChunk", + "TextContentBlock", "ToolCall", "ToolCallChunk", "ToolMessage", "ToolMessageChunk", + "VideoContentBlock", + "WebSearchCall", + "WebSearchResult", "_message_from_dict", "convert_to_messages", "convert_to_openai_data_block", "convert_to_openai_image_block", "convert_to_openai_messages", + "ensure_id", "filter_messages", "get_buffer_string", "is_data_content_block", + "is_reasoning_block", + "is_text_block", + "is_tool_call_block", + "is_tool_call_chunk", "merge_content", "merge_message_runs", "message_chunk_to_message", @@ -103,35 +151,57 @@ _dynamic_imports = { "AIMessage": "ai", "AIMessageChunk": "ai", + "Annotation": "content", + "AudioContentBlock": "content", "BaseMessage": "base", "BaseMessageChunk": "base", "merge_content": "base", "message_to_dict": "base", "messages_to_dict": "base", + "Citation": "content", + "ContentBlock": "content", "ChatMessage": "chat", "ChatMessageChunk": "chat", + "CodeInterpreterCall": "content", + "CodeInterpreterOutput": "content", + "CodeInterpreterResult": "content", + "DataContentBlock": "content", + "FileContentBlock": "content", "FunctionMessage": "function", "FunctionMessageChunk": "function", "HumanMessage": "human", "HumanMessageChunk": "human", + "NonStandardAnnotation": "content", + "NonStandardContentBlock": "content", + "PlainTextContentBlock": "content", + "ReasoningContentBlock": "content", "RemoveMessage": "modifier", "SystemMessage": "system", "SystemMessageChunk": "system", + "WebSearchCall": "content", + "WebSearchResult": "content", + "ImageContentBlock": "content", "InvalidToolCall": "tool", + "TextContentBlock": "content", "ToolCall": "tool", "ToolCallChunk": "tool", "ToolMessage": "tool", "ToolMessageChunk": "tool", + "VideoContentBlock": "content", "AnyMessage": "utils", "MessageLikeRepresentation": "utils", "_message_from_dict": "utils", "convert_to_messages": "utils", - "convert_to_openai_data_block": "content_blocks", - "convert_to_openai_image_block": "content_blocks", + "convert_to_openai_data_block": "content", + "convert_to_openai_image_block": "content", "convert_to_openai_messages": "utils", "filter_messages": "utils", "get_buffer_string": "utils", - "is_data_content_block": "content_blocks", + "is_data_content_block": "content", + "is_reasoning_block": "content", + "is_text_block": "content", + "is_tool_call_block": "content", + "is_tool_call_chunk": "content", "merge_message_runs": "utils", "message_chunk_to_message": "utils", "messages_from_dict": "utils", diff --git a/libs/core/langchain_core/messages/ai.py b/libs/core/langchain_core/messages/ai.py index 638e5113c7d98..7efb8565cb382 100644 --- a/libs/core/langchain_core/messages/ai.py +++ b/libs/core/langchain_core/messages/ai.py @@ -9,6 +9,7 @@ from pydantic import model_validator from typing_extensions import NotRequired, Self, TypedDict, overload, override +from langchain_core.messages import content as types from langchain_core.messages.base import ( BaseMessage, BaseMessageChunk, @@ -21,32 +22,23 @@ default_tool_chunk_parser, default_tool_parser, ) -from langchain_core.messages.tool import ( - invalid_tool_call as create_invalid_tool_call, -) -from langchain_core.messages.tool import ( - tool_call as create_tool_call, -) -from langchain_core.messages.tool import ( - tool_call_chunk as create_tool_call_chunk, -) +from langchain_core.messages.tool import invalid_tool_call as create_invalid_tool_call +from langchain_core.messages.tool import tool_call as create_tool_call +from langchain_core.messages.tool import tool_call_chunk as create_tool_call_chunk from langchain_core.utils._merge import merge_dicts, merge_lists from langchain_core.utils.json import parse_partial_json from langchain_core.utils.usage import _dict_int_op +from langchain_core.utils.utils import LC_AUTO_PREFIX, LC_ID_PREFIX logger = logging.getLogger(__name__) -_LC_ID_PREFIX = "run-" - - class InputTokenDetails(TypedDict, total=False): """Breakdown of input token counts. Does *not* need to sum to full input token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -73,6 +65,7 @@ class InputTokenDetails(TypedDict, total=False): Since there was a cache hit, the tokens were read from the cache. More precisely, the model state given these tokens was read from the cache. + """ @@ -82,7 +75,6 @@ class OutputTokenDetails(TypedDict, total=False): Does *not* need to sum to full output token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -101,6 +93,7 @@ class OutputTokenDetails(TypedDict, total=False): Tokens generated by the model in a chain of thought process (i.e. by OpenAI's o1 models) that are not returned as part of model output. + """ @@ -110,7 +103,6 @@ class UsageMetadata(TypedDict): This is a standard representation of token usage that is consistent across models. Example: - .. code-block:: python { @@ -149,6 +141,7 @@ class UsageMetadata(TypedDict): """Breakdown of output token counts. Does *not* need to sum to full output token count. Does *not* need to have all keys. + """ @@ -160,12 +153,14 @@ class AIMessage(BaseMessage): This message represents the output of the model and consists of both the raw output as returned by the model together standardized fields (e.g., tool calls, usage metadata) added by the LangChain framework. + """ example: bool = False """Use to denote that a message is part of an example conversation. At the moment, this is ignored by most models. Usage is discouraged. + """ tool_calls: list[ToolCall] = [] @@ -176,15 +171,25 @@ class AIMessage(BaseMessage): """If provided, usage metadata for a message, such as token counts. This is a standard representation of token usage that is consistent across models. + """ type: Literal["ai"] = "ai" """The type of the message (used for deserialization). Defaults to "ai".""" + @overload + def __init__( + self, + content: Union[str, list[Union[str, dict]]], + **kwargs: Any, + ) -> None: ... + def __init__( - self, content: Union[str, list[Union[str, dict]]], **kwargs: Any + self, + content: Union[str, list[Union[str, dict]]], + **kwargs: Any, ) -> None: - """Pass in content as positional arg. + """Initialize AIMessage. Args: content: The content of the message. @@ -200,6 +205,49 @@ def lc_attributes(self) -> dict: "invalid_tool_calls": self.invalid_tool_calls, } + @property + def content_blocks(self) -> list[types.ContentBlock]: + """Return content blocks of the message.""" + if self.response_metadata.get("output_version") == "v1": + return cast("list[types.ContentBlock]", self.content) + + model_provider = self.response_metadata.get("model_provider") + if model_provider: + from langchain_core.messages.block_translators import get_translator + + translator = get_translator(model_provider) + if translator: + try: + return translator["translate_content_chunk"](self) + except NotImplementedError: + pass + + # Otherwise, use best-effort parsing + blocks = super().content_blocks + + if self.tool_calls: + # Add from tool_calls if missing from content + content_tool_call_ids = { + block.get("id") + for block in self.content + if isinstance(block, dict) and block.get("type") == "tool_call" + } + for tool_call in self.tool_calls: + if (id_ := tool_call.get("id")) and id_ not in content_tool_call_ids: + tool_call_block: types.ToolCall = { + "type": "tool_call", + "id": id_, + "name": tool_call["name"], + "args": tool_call["args"], + } + if "index" in tool_call: + tool_call_block["index"] = tool_call["index"] # type: ignore[typeddict-item] + if "extras" in tool_call: + tool_call_block["extras"] = tool_call["extras"] # type: ignore[typeddict-item] + blocks.append(tool_call_block) + + return blocks + # TODO: remove this logic if possible, reducing breaking nature of changes @model_validator(mode="before") @classmethod @@ -228,7 +276,9 @@ def _backwards_compat_tool_calls(cls, values: dict) -> Any: # Ensure "type" is properly set on all tool call-like dicts. if tool_calls := values.get("tool_calls"): values["tool_calls"] = [ - create_tool_call(**{k: v for k, v in tc.items() if k != "type"}) + create_tool_call( + **{k: v for k, v in tc.items() if k not in ("type", "extras")} + ) for tc in tool_calls ] if invalid_tool_calls := values.get("invalid_tool_calls"): @@ -255,6 +305,7 @@ def pretty_repr(self, html: bool = False) -> str: Returns: A pretty representation of the message. + """ base = super().pretty_repr(html=html) lines = [] @@ -294,7 +345,10 @@ class AIMessageChunk(AIMessage, BaseMessageChunk): # non-chunk variant. type: Literal["AIMessageChunk"] = "AIMessageChunk" # type: ignore[assignment] """The type of the message (used for deserialization). - Defaults to "AIMessageChunk".""" + + Defaults to ``AIMessageChunk``. + + """ tool_call_chunks: list[ToolCallChunk] = [] """If provided, tool call chunks associated with the message.""" @@ -307,6 +361,45 @@ def lc_attributes(self) -> dict: "invalid_tool_calls": self.invalid_tool_calls, } + @property + def content_blocks(self) -> list[types.ContentBlock]: + """Return content blocks of the message.""" + if self.response_metadata.get("output_version") == "v1": + return cast("list[types.ContentBlock]", self.content) + + model_provider = self.response_metadata.get("model_provider") + if model_provider: + from langchain_core.messages.block_translators import get_translator + + translator = get_translator(model_provider) + if translator: + try: + return translator["translate_content_chunk"](self) + except NotImplementedError: + pass + + # Otherwise, use best-effort parsing + blocks = super().content_blocks + + if self.tool_call_chunks and not self.content: + blocks = [ + block + for block in blocks + if block["type"] not in ("tool_call", "invalid_tool_call") + ] + for tool_call_chunk in self.tool_call_chunks: + tc: types.ToolCallChunk = { + "type": "tool_call_chunk", + "id": tool_call_chunk.get("id"), + "name": tool_call_chunk.get("name"), + "args": tool_call_chunk.get("args"), + } + if (idx := tool_call_chunk.get("index")) is not None: + tc["index"] = idx + blocks.append(tc) + + return blocks + @model_validator(mode="after") def init_tool_calls(self) -> Self: """Initialize tool calls from tool call chunks. @@ -319,6 +412,7 @@ def init_tool_calls(self) -> Self: Raises: ValueError: If the tool call chunks are malformed. + """ if not self.tool_call_chunks: if self.tool_calls: @@ -441,17 +535,27 @@ def add_ai_message_chunks( chunk_id = None candidates = [left.id] + [o.id for o in others] - # first pass: pick the first non-run-* id + # first pass: pick the first provider-assigned id (non-run-* and non-lc_*) for id_ in candidates: - if id_ and not id_.startswith(_LC_ID_PREFIX): + if ( + id_ + and not id_.startswith(LC_ID_PREFIX) + and not id_.startswith(LC_AUTO_PREFIX) + ): chunk_id = id_ break else: - # second pass: no provider-assigned id found, just take the first non-null + # second pass: prefer lc_run-* ids over lc_* ids for id_ in candidates: - if id_: + if id_ and id_.startswith(LC_ID_PREFIX): chunk_id = id_ break + else: + # third pass: take any remaining id (auto-generated lc_* ids) + for id_ in candidates: + if id_: + chunk_id = id_ + break return left.__class__( example=left.example, @@ -522,9 +626,9 @@ def add_usage( def subtract_usage( left: Optional[UsageMetadata], right: Optional[UsageMetadata] ) -> UsageMetadata: - """Recursively subtract two UsageMetadata objects. + """Recursively subtract two ``UsageMetadata`` objects. - Token counts cannot be negative so the actual operation is max(left - right, 0). + Token counts cannot be negative so the actual operation is ``max(left - right, 0)``. Example: .. code-block:: python diff --git a/libs/core/langchain_core/messages/base.py b/libs/core/langchain_core/messages/base.py index ba976286b75d8..3c7199ae060a1 100644 --- a/libs/core/langchain_core/messages/base.py +++ b/libs/core/langchain_core/messages/base.py @@ -2,11 +2,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Optional, Union, cast, overload from pydantic import ConfigDict, Field from langchain_core.load.serializable import Serializable +from langchain_core.messages import content as types from langchain_core.utils import get_bolded_text from langchain_core.utils._merge import merge_dicts, merge_lists from langchain_core.utils.interactive_env import is_interactive_env @@ -20,7 +21,7 @@ class BaseMessage(Serializable): """Base abstract message class. - Messages are the inputs and outputs of ChatModels. + Messages are the inputs and outputs of ``ChatModel``s. """ content: Union[str, list[Union[str, dict]]] @@ -31,17 +32,18 @@ class BaseMessage(Serializable): For example, for a message from an AI, this could include tool calls as encoded by the model provider. + """ response_metadata: dict = Field(default_factory=dict) - """Response metadata. For example: response headers, logprobs, token counts, model - name.""" + """Examples: response headers, logprobs, token counts, model name.""" type: str """The type of the message. Must be a string that is unique to the message type. The purpose of this field is to allow for easy identification of the message type when deserializing messages. + """ name: Optional[str] = None @@ -51,20 +53,33 @@ class BaseMessage(Serializable): Usage of this field is optional, and whether it's used or not is up to the model implementation. + """ id: Optional[str] = Field(default=None, coerce_numbers_to_str=True) - """An optional unique identifier for the message. This should ideally be - provided by the provider/model which created the message.""" + """An optional unique identifier for the message. + + This should ideally be provided by the provider/model which created the message. + + """ model_config = ConfigDict( extra="allow", ) + @overload + def __init__( + self, + content: Union[str, list[Union[str, dict]]], + **kwargs: Any, + ) -> None: ... + def __init__( - self, content: Union[str, list[Union[str, dict]]], **kwargs: Any + self, + content: Union[str, list[Union[str, dict]]], + **kwargs: Any, ) -> None: - """Pass in content as positional arg. + """Initialize BaseMessage. Args: content: The string contents of the message. @@ -73,7 +88,7 @@ def __init__( @classmethod def is_lc_serializable(cls) -> bool: - """BaseMessage is serializable. + """``BaseMessage`` is serializable. Returns: True @@ -84,15 +99,80 @@ def is_lc_serializable(cls) -> bool: def get_lc_namespace(cls) -> list[str]: """Get the namespace of the langchain object. - Default is ["langchain", "schema", "messages"]. + Default is ``['langchain', 'schema', 'messages']``. + """ return ["langchain", "schema", "messages"] + @property + def content_blocks(self) -> list[types.ContentBlock]: + r"""Return ``content`` as a list of standardized :class:`~langchain_core.messages.content.ContentBlock`\s. + + .. important:: + + To use this property correctly, the corresponding ``ChatModel`` must support + ``message_version='v1'`` or higher (and it must be set): + + .. code-block:: python + + from langchain.chat_models import init_chat_model + llm = init_chat_model("...", message_version="v1") + + # or + + from langchain-openai import ChatOpenAI + llm = ChatOpenAI(model="gpt-4o", message_version="v1") + + Otherwise, the property will perform best-effort parsing to standard types, + though some content may be misinterpreted. + + .. versionadded:: 1.0.0 + + """ # noqa: E501 + from langchain_core.messages import content as types + from langchain_core.messages.block_translators.anthropic import ( + _convert_to_v1_from_anthropic_input, + ) + from langchain_core.messages.block_translators.langchain_v0 import ( + _convert_v0_multimodal_input_to_v1, + ) + from langchain_core.messages.block_translators.openai import ( + _convert_to_v1_from_chat_completions_input, + ) + + blocks: list[types.ContentBlock] = [] + + # First pass: convert to standard blocks + content = ( + [self.content] + if isinstance(self.content, str) and self.content + else self.content + ) + for item in content: + if isinstance(item, str): + blocks.append({"type": "text", "text": item}) + elif isinstance(item, dict): + item_type = item.get("type") + if item_type not in types.KNOWN_BLOCK_TYPES: + blocks.append({"type": "non_standard", "value": item}) + else: + blocks.append(cast("types.ContentBlock", item)) + + # Subsequent passes: attempt to unpack non-standard blocks + for parsing_step in [ + _convert_v0_multimodal_input_to_v1, + _convert_to_v1_from_chat_completions_input, + _convert_to_v1_from_anthropic_input, + ]: + blocks = parsing_step(blocks) + return blocks + def text(self) -> str: - """Get the text content of the message. + """Get the text ``content`` of the message. Returns: The text content of the message. + """ if isinstance(self.content, str): return self.content @@ -127,6 +207,7 @@ def pretty_repr( Returns: A pretty representation of the message. + """ title = get_msg_title_repr(self.type.title() + " Message", bold=html) # TODO: handle non-string content. @@ -146,13 +227,16 @@ def merge_content( """Merge multiple message contents. Args: - first_content: The first content. Can be a string or a list. - contents: The other contents. Can be a string or a list. + first_content: The first ``content``. Can be a string or a list. + contents: The other ``content``s. Can be a string or a list. Returns: The merged content. + """ - merged = first_content + merged: Union[str, list[Union[str, dict]]] + merged = "" if first_content is None else first_content + for content in contents: # If current is a string if isinstance(merged, str): @@ -173,8 +257,8 @@ def merge_content( # If second content is an empty string, treat as a no-op elif content == "": pass - else: - # Otherwise, add the second content as a new element of the list + # Otherwise, add the second content as a new element of the list + elif merged: merged.append(content) return merged @@ -200,9 +284,10 @@ def __add__(self, other: Any) -> BaseMessageChunk: # type: ignore[override] For example, - `AIMessageChunk(content="Hello") + AIMessageChunk(content=" World")` + ``AIMessageChunk(content="Hello") + AIMessageChunk(content=" World")`` + + will give ``AIMessageChunk(content="Hello World")`` - will give `AIMessageChunk(content="Hello World")` """ if isinstance(other, BaseMessageChunk): # If both are (subclasses of) BaseMessageChunk, @@ -250,8 +335,9 @@ def message_to_dict(message: BaseMessage) -> dict: message: Message to convert. Returns: - Message as a dict. The dict will have a "type" key with the message type - and a "data" key with the message data as a dict. + Message as a dict. The dict will have a ``type`` key with the message type + and a ``data`` key with the message data as a dict. + """ return {"type": message.type, "data": message.model_dump()} @@ -260,10 +346,11 @@ def messages_to_dict(messages: Sequence[BaseMessage]) -> list[dict]: """Convert a sequence of Messages to a list of dictionaries. Args: - messages: Sequence of messages (as BaseMessages) to convert. + messages: Sequence of messages (as ``BaseMessage``s) to convert. Returns: List of messages as dicts. + """ return [message_to_dict(m) for m in messages] @@ -277,6 +364,7 @@ def get_msg_title_repr(title: str, *, bold: bool = False) -> str: Returns: The title representation. + """ padded = " " + title + " " sep_len = (80 - len(padded)) // 2 diff --git a/libs/core/langchain_core/messages/block_translators/__init__.py b/libs/core/langchain_core/messages/block_translators/__init__.py new file mode 100644 index 0000000000000..bb9673a7c373b --- /dev/null +++ b/libs/core/langchain_core/messages/block_translators/__init__.py @@ -0,0 +1,89 @@ +"""Derivations of standard content blocks from provider content.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Callable + +if TYPE_CHECKING: + from langchain_core.messages import AIMessage, AIMessageChunk + from langchain_core.messages import content as types + +# Provider to translator mapping +PROVIDER_TRANSLATORS: dict[str, dict[str, Callable[..., list[types.ContentBlock]]]] = {} + + +def register_translator( + provider: str, + translate_content: Callable[[AIMessage], list[types.ContentBlock]], + translate_content_chunk: Callable[[AIMessageChunk], list[types.ContentBlock]], +) -> None: + """Register content translators for a provider. + + Args: + provider: The model provider name (e.g. ``'openai'``, ``'anthropic'``). + translate_content: Function to translate ``AIMessage`` content. + translate_content_chunk: Function to translate ``AIMessageChunk`` content. + """ + PROVIDER_TRANSLATORS[provider] = { + "translate_content": translate_content, + "translate_content_chunk": translate_content_chunk, + } + + +def get_translator( + provider: str, +) -> dict[str, Callable[..., list[types.ContentBlock]]] | None: + """Get the translator functions for a provider. + + Args: + provider: The model provider name. + + Returns: + Dictionary with ``'translate_content'`` and ``'translate_content_chunk'`` + functions, or None if no translator is registered for the provider. + """ + return PROVIDER_TRANSLATORS.get(provider) + + +def _register_translators() -> None: + """Register all translators in langchain-core. + + A unit test ensures all modules in ``block_translators`` are represented here. + + For translators implemented outside langchain-core, they can be registered by + calling ``register_translator`` from within the integration package. + """ + from langchain_core.messages.block_translators.anthropic import ( + _register_anthropic_translator, + ) + from langchain_core.messages.block_translators.bedrock import ( + _register_bedrock_translator, + ) + from langchain_core.messages.block_translators.bedrock_converse import ( + _register_bedrock_converse_translator, + ) + from langchain_core.messages.block_translators.google_genai import ( + _register_google_genai_translator, + ) + from langchain_core.messages.block_translators.google_vertexai import ( + _register_google_vertexai_translator, + ) + from langchain_core.messages.block_translators.groq import _register_groq_translator + from langchain_core.messages.block_translators.ollama import ( + _register_ollama_translator, + ) + from langchain_core.messages.block_translators.openai import ( + _register_openai_translator, + ) + + _register_bedrock_translator() + _register_bedrock_converse_translator() + _register_anthropic_translator() + _register_google_genai_translator() + _register_google_vertexai_translator() + _register_groq_translator() + _register_ollama_translator() + _register_openai_translator() + + +_register_translators() diff --git a/libs/core/langchain_core/messages/block_translators/anthropic.py b/libs/core/langchain_core/messages/block_translators/anthropic.py new file mode 100644 index 0000000000000..655cfaa2857ef --- /dev/null +++ b/libs/core/langchain_core/messages/block_translators/anthropic.py @@ -0,0 +1,438 @@ +"""Derivations of standard content blocks from Anthropic content.""" + +import json +from collections.abc import Iterable +from typing import Any, cast + +from langchain_core.messages import AIMessage, AIMessageChunk +from langchain_core.messages import content as types + + +def _populate_extras( + standard_block: types.ContentBlock, block: dict[str, Any], known_fields: set[str] +) -> types.ContentBlock: + """Mutate a block, populating extras.""" + if standard_block.get("type") == "non_standard": + return standard_block + + for key, value in block.items(): + if key not in known_fields: + if "extras" not in block: + # Below type-ignores are because mypy thinks a non-standard block can + # get here, although we exclude them above. + standard_block["extras"] = {} # type: ignore[typeddict-unknown-key] + standard_block["extras"][key] = value # type: ignore[typeddict-item] + + return standard_block + + +def _convert_to_v1_from_anthropic_input( + content: list[types.ContentBlock], +) -> list[types.ContentBlock]: + """Attempt to unpack non-standard blocks.""" + + def _iter_blocks() -> Iterable[types.ContentBlock]: + blocks: list[dict[str, Any]] = [ + cast("dict[str, Any]", block) + if block.get("type") != "non_standard" + else block["value"] # type: ignore[typeddict-item] # this is only non-standard blocks + for block in content + ] + for block in blocks: + block_type = block.get("type") + + if ( + block_type == "document" + and "source" in block + and "type" in block["source"] + ): + if block["source"]["type"] == "base64": + file_block: types.FileContentBlock = { + "type": "file", + "base64": block["source"]["data"], + "mime_type": block["source"]["media_type"], + } + _populate_extras(file_block, block, {"type", "source"}) + yield file_block + + elif block["source"]["type"] == "url": + file_block = { + "type": "file", + "url": block["source"]["url"], + } + _populate_extras(file_block, block, {"type", "source"}) + yield file_block + + elif block["source"]["type"] == "file": + file_block = { + "type": "file", + "id": block["source"]["file_id"], + } + _populate_extras(file_block, block, {"type", "source"}) + yield file_block + + elif block["source"]["type"] == "text": + plain_text_block: types.PlainTextContentBlock = { + "type": "text-plain", + "text": block["source"]["data"], + "mime_type": block.get("media_type", "text/plain"), + } + _populate_extras(plain_text_block, block, {"type", "source"}) + yield plain_text_block + + else: + yield {"type": "non_standard", "value": block} + + elif ( + block_type == "image" + and "source" in block + and "type" in block["source"] + ): + if block["source"]["type"] == "base64": + image_block: types.ImageContentBlock = { + "type": "image", + "base64": block["source"]["data"], + "mime_type": block["source"]["media_type"], + } + _populate_extras(image_block, block, {"type", "source"}) + yield image_block + + elif block["source"]["type"] == "url": + image_block = { + "type": "image", + "url": block["source"]["url"], + } + _populate_extras(image_block, block, {"type", "source"}) + yield image_block + + elif block["source"]["type"] == "file": + image_block = { + "type": "image", + "id": block["source"]["file_id"], + } + _populate_extras(image_block, block, {"type", "source"}) + yield image_block + + else: + yield {"type": "non_standard", "value": block} + + elif block_type in types.KNOWN_BLOCK_TYPES: + yield cast("types.ContentBlock", block) + + else: + yield {"type": "non_standard", "value": block} + + return list(_iter_blocks()) + + +def _convert_citation_to_v1(citation: dict[str, Any]) -> types.Annotation: + citation_type = citation.get("type") + + if citation_type == "web_search_result_location": + url_citation: types.Citation = { + "type": "citation", + "cited_text": citation["cited_text"], + "url": citation["url"], + } + if title := citation.get("title"): + url_citation["title"] = title + known_fields = {"type", "cited_text", "url", "title", "index", "extras"} + for key, value in citation.items(): + if key not in known_fields: + if "extras" not in url_citation: + url_citation["extras"] = {} + url_citation["extras"][key] = value + + return url_citation + + if citation_type in ( + "char_location", + "content_block_location", + "page_location", + "search_result_location", + ): + document_citation: types.Citation = { + "type": "citation", + "cited_text": citation["cited_text"], + } + if "document_title" in citation: + document_citation["title"] = citation["document_title"] + elif title := citation.get("title"): + document_citation["title"] = title + else: + pass + known_fields = { + "type", + "cited_text", + "document_title", + "title", + "index", + "extras", + } + for key, value in citation.items(): + if key not in known_fields: + if "extras" not in document_citation: + document_citation["extras"] = {} + document_citation["extras"][key] = value + + return document_citation + + return { + "type": "non_standard_annotation", + "value": citation, + } + + +def _convert_to_v1_from_anthropic(message: AIMessage) -> list[types.ContentBlock]: + """Convert Anthropic message content to v1 format.""" + if isinstance(message.content, str): + message.content = [{"type": "text", "text": message.content}] + + def _iter_blocks() -> Iterable[types.ContentBlock]: + for block in message.content: + if not isinstance(block, dict): + continue + block_type = block.get("type") + + if block_type == "text": + if citations := block.get("citations"): + text_block: types.TextContentBlock = { + "type": "text", + "text": block.get("text", ""), + "annotations": [_convert_citation_to_v1(a) for a in citations], + } + else: + text_block = {"type": "text", "text": block["text"]} + if "index" in block: + text_block["index"] = block["index"] + yield text_block + + elif block_type == "thinking": + reasoning_block: types.ReasoningContentBlock = { + "type": "reasoning", + "reasoning": block.get("thinking", ""), + } + if "index" in block: + reasoning_block["index"] = block["index"] + known_fields = {"type", "thinking", "index", "extras"} + for key in block: + if key not in known_fields: + if "extras" not in reasoning_block: + reasoning_block["extras"] = {} + reasoning_block["extras"][key] = block[key] + yield reasoning_block + + elif block_type == "tool_use": + if ( + isinstance(message, AIMessageChunk) + and len(message.tool_call_chunks) == 1 + ): + tool_call_chunk: types.ToolCallChunk = ( + message.tool_call_chunks[0].copy() # type: ignore[assignment] + ) + if "type" not in tool_call_chunk: + tool_call_chunk["type"] = "tool_call_chunk" + yield tool_call_chunk + elif ( + not isinstance(message, AIMessageChunk) + and len(message.tool_calls) == 1 + ): + tool_call_block: types.ToolCall = { + "type": "tool_call", + "name": message.tool_calls[0]["name"], + "args": message.tool_calls[0]["args"], + "id": message.tool_calls[0].get("id"), + } + if "index" in block: + tool_call_block["index"] = block["index"] + yield tool_call_block + else: + tool_call_block = { + "type": "tool_call", + "name": block.get("name", ""), + "args": block.get("input", {}), + "id": block.get("id", ""), + } + yield tool_call_block + + elif ( + block_type == "input_json_delta" + and isinstance(message, AIMessageChunk) + and len(message.tool_call_chunks) == 1 + ): + tool_call_chunk = ( + message.tool_call_chunks[0].copy() # type: ignore[assignment] + ) + if "type" not in tool_call_chunk: + tool_call_chunk["type"] = "tool_call_chunk" + yield tool_call_chunk + + elif block_type == "server_tool_use": + if block.get("name") == "web_search": + web_search_call: types.WebSearchCall = {"type": "web_search_call"} + + if query := block.get("input", {}).get("query"): + web_search_call["query"] = query + + elif block.get("input") == {} and "partial_json" in block: + try: + input_ = json.loads(block["partial_json"]) + if isinstance(input_, dict) and "query" in input_: + web_search_call["query"] = input_["query"] + except json.JSONDecodeError: + pass + + if "id" in block: + web_search_call["id"] = block["id"] + if "index" in block: + web_search_call["index"] = block["index"] + known_fields = {"type", "name", "input", "id", "index"} + for key, value in block.items(): + if key not in known_fields: + if "extras" not in web_search_call: + web_search_call["extras"] = {} + web_search_call["extras"][key] = value + yield web_search_call + + elif block.get("name") == "code_execution": + code_interpreter_call: types.CodeInterpreterCall = { + "type": "code_interpreter_call" + } + + if code := block.get("input", {}).get("code"): + code_interpreter_call["code"] = code + + elif block.get("input") == {} and "partial_json" in block: + try: + input_ = json.loads(block["partial_json"]) + if isinstance(input_, dict) and "code" in input_: + code_interpreter_call["code"] = input_["code"] + except json.JSONDecodeError: + pass + + if "id" in block: + code_interpreter_call["id"] = block["id"] + if "index" in block: + code_interpreter_call["index"] = block["index"] + known_fields = {"type", "name", "input", "id", "index"} + for key, value in block.items(): + if key not in known_fields: + if "extras" not in code_interpreter_call: + code_interpreter_call["extras"] = {} + code_interpreter_call["extras"][key] = value + yield code_interpreter_call + + else: + new_block: types.NonStandardContentBlock = { + "type": "non_standard", + "value": block, + } + if "index" in new_block["value"]: + new_block["index"] = new_block["value"].pop("index") + yield new_block + + elif block_type == "web_search_tool_result": + web_search_result: types.WebSearchResult = {"type": "web_search_result"} + if "tool_use_id" in block: + web_search_result["id"] = block["tool_use_id"] + if "index" in block: + web_search_result["index"] = block["index"] + + if web_search_result_content := block.get("content", []): + if "extras" not in web_search_result: + web_search_result["extras"] = {} + urls = [] + extra_content = [] + for result_content in web_search_result_content: + if isinstance(result_content, dict): + if "url" in result_content: + urls.append(result_content["url"]) + extra_content.append(result_content) + web_search_result["extras"]["content"] = extra_content + if urls: + web_search_result["urls"] = urls + yield web_search_result + + elif block_type == "code_execution_tool_result": + code_interpreter_result: types.CodeInterpreterResult = { + "type": "code_interpreter_result", + "output": [], + } + if "tool_use_id" in block: + code_interpreter_result["id"] = block["tool_use_id"] + if "index" in block: + code_interpreter_result["index"] = block["index"] + + code_interpreter_output: types.CodeInterpreterOutput = { + "type": "code_interpreter_output" + } + + code_execution_content = block.get("content", {}) + if code_execution_content.get("type") == "code_execution_result": + if "return_code" in code_execution_content: + code_interpreter_output["return_code"] = code_execution_content[ + "return_code" + ] + if "stdout" in code_execution_content: + code_interpreter_output["stdout"] = code_execution_content[ + "stdout" + ] + if stderr := code_execution_content.get("stderr"): + code_interpreter_output["stderr"] = stderr + if ( + output := code_interpreter_output.get("content") + ) and isinstance(output, list): + if "extras" not in code_interpreter_result: + code_interpreter_result["extras"] = {} + code_interpreter_result["extras"]["content"] = output + for output_block in output: + if "file_id" in output_block: + if "file_ids" not in code_interpreter_output: + code_interpreter_output["file_ids"] = [] + code_interpreter_output["file_ids"].append( + output_block["file_id"] + ) + code_interpreter_result["output"].append(code_interpreter_output) + + elif ( + code_execution_content.get("type") + == "code_execution_tool_result_error" + ): + if "extras" not in code_interpreter_result: + code_interpreter_result["extras"] = {} + code_interpreter_result["extras"]["error_code"] = ( + code_execution_content.get("error_code") + ) + + yield code_interpreter_result + + else: + new_block = {"type": "non_standard", "value": block} + if "index" in new_block["value"]: + new_block["index"] = new_block["value"].pop("index") + yield new_block + + return list(_iter_blocks()) + + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: + """Derive standard content blocks from a message with OpenAI content.""" + return _convert_to_v1_from_anthropic(message) + + +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: + """Derive standard content blocks from a message chunk with OpenAI content.""" + return _convert_to_v1_from_anthropic(message) + + +def _register_anthropic_translator() -> None: + """Register the Anthropic translator with the central registry. + + Run automatically when the module is imported. + """ + from langchain_core.messages.block_translators import register_translator + + register_translator("anthropic", translate_content, translate_content_chunk) + + +_register_anthropic_translator() diff --git a/libs/core/langchain_core/messages/block_translators/bedrock.py b/libs/core/langchain_core/messages/block_translators/bedrock.py new file mode 100644 index 0000000000000..796d45336b17f --- /dev/null +++ b/libs/core/langchain_core/messages/block_translators/bedrock.py @@ -0,0 +1,45 @@ +"""Derivations of standard content blocks from Amazon (Bedrock) content.""" + +import warnings + +from langchain_core.messages import AIMessage, AIMessageChunk +from langchain_core.messages import content as types + +WARNED = False + + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: # noqa: ARG001 + """Derive standard content blocks from a message with Bedrock content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Bedrock." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True + raise NotImplementedError + + +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: # noqa: ARG001 + """Derive standard content blocks from a chunk with Bedrock content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Bedrock." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True + raise NotImplementedError + + +def _register_bedrock_translator() -> None: + """Register the Bedrock translator with the central registry. + + Run automatically when the module is imported. + """ + from langchain_core.messages.block_translators import register_translator + + register_translator("bedrock", translate_content, translate_content_chunk) + + +_register_bedrock_translator() diff --git a/libs/core/langchain_core/messages/block_translators/bedrock_converse.py b/libs/core/langchain_core/messages/block_translators/bedrock_converse.py new file mode 100644 index 0000000000000..6249c9107a93c --- /dev/null +++ b/libs/core/langchain_core/messages/block_translators/bedrock_converse.py @@ -0,0 +1,47 @@ +"""Derivations of standard content blocks from Amazon (Bedrock Converse) content.""" + +import warnings + +from langchain_core.messages import AIMessage, AIMessageChunk +from langchain_core.messages import content as types + +WARNED = False + + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: # noqa: ARG001 + """Derive standard content blocks from a message with Bedrock Converse content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Bedrock " + "Converse." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True + raise NotImplementedError + + +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: # noqa: ARG001 + """Derive standard content blocks from a chunk with Bedrock Converse content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Bedrock " + "Converse." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True + raise NotImplementedError + + +def _register_bedrock_converse_translator() -> None: + """Register the Bedrock Converse translator with the central registry. + + Run automatically when the module is imported. + """ + from langchain_core.messages.block_translators import register_translator + + register_translator("bedrock_converse", translate_content, translate_content_chunk) + + +_register_bedrock_converse_translator() diff --git a/libs/core/langchain_core/messages/block_translators/google_genai.py b/libs/core/langchain_core/messages/block_translators/google_genai.py new file mode 100644 index 0000000000000..bd4de65c3b0ce --- /dev/null +++ b/libs/core/langchain_core/messages/block_translators/google_genai.py @@ -0,0 +1,45 @@ +"""Derivations of standard content blocks from Google (GenAI) content.""" + +import warnings + +from langchain_core.messages import AIMessage, AIMessageChunk +from langchain_core.messages import content as types + +WARNED = False + + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: # noqa: ARG001 + """Derive standard content blocks from a message with Google (GenAI) content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Google GenAI." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True + raise NotImplementedError + + +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: # noqa: ARG001 + """Derive standard content blocks from a chunk with Google (GenAI) content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Google GenAI." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True + raise NotImplementedError + + +def _register_google_genai_translator() -> None: + """Register the Google (GenAI) translator with the central registry. + + Run automatically when the module is imported. + """ + from langchain_core.messages.block_translators import register_translator + + register_translator("google_genai", translate_content, translate_content_chunk) + + +_register_google_genai_translator() diff --git a/libs/core/langchain_core/messages/block_translators/google_vertexai.py b/libs/core/langchain_core/messages/block_translators/google_vertexai.py new file mode 100644 index 0000000000000..e49ee384058ee --- /dev/null +++ b/libs/core/langchain_core/messages/block_translators/google_vertexai.py @@ -0,0 +1,47 @@ +"""Derivations of standard content blocks from Google (VertexAI) content.""" + +import warnings + +from langchain_core.messages import AIMessage, AIMessageChunk +from langchain_core.messages import content as types + +WARNED = False + + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: # noqa: ARG001 + """Derive standard content blocks from a message with Google (VertexAI) content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Google " + "VertexAI." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True + raise NotImplementedError + + +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: # noqa: ARG001 + """Derive standard content blocks from a chunk with Google (VertexAI) content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Google " + "VertexAI." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True + raise NotImplementedError + + +def _register_google_vertexai_translator() -> None: + """Register the Google (VertexAI) translator with the central registry. + + Run automatically when the module is imported. + """ + from langchain_core.messages.block_translators import register_translator + + register_translator("google_vertexai", translate_content, translate_content_chunk) + + +_register_google_vertexai_translator() diff --git a/libs/core/langchain_core/messages/block_translators/groq.py b/libs/core/langchain_core/messages/block_translators/groq.py new file mode 100644 index 0000000000000..6a96b1775f429 --- /dev/null +++ b/libs/core/langchain_core/messages/block_translators/groq.py @@ -0,0 +1,45 @@ +"""Derivations of standard content blocks from Groq content.""" + +import warnings + +from langchain_core.messages import AIMessage, AIMessageChunk +from langchain_core.messages import content as types + +WARNED = False + + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: # noqa: ARG001 + """Derive standard content blocks from a message with Groq content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Groq." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True + raise NotImplementedError + + +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: # noqa: ARG001 + """Derive standard content blocks from a message chunk with Groq content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Groq." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True + raise NotImplementedError + + +def _register_groq_translator() -> None: + """Register the Groq translator with the central registry. + + Run automatically when the module is imported. + """ + from langchain_core.messages.block_translators import register_translator + + register_translator("groq", translate_content, translate_content_chunk) + + +_register_groq_translator() diff --git a/libs/core/langchain_core/messages/block_translators/langchain_v0.py b/libs/core/langchain_core/messages/block_translators/langchain_v0.py new file mode 100644 index 0000000000000..5fde4c0fcb0d4 --- /dev/null +++ b/libs/core/langchain_core/messages/block_translators/langchain_v0.py @@ -0,0 +1,295 @@ +"""Derivations of standard content blocks from LangChain v0 multimodal content.""" + +from typing import Any, Union, cast + +from langchain_core.language_models._utils import _parse_data_uri +from langchain_core.messages import content as types + + +def _convert_v0_multimodal_input_to_v1( + blocks: list[types.ContentBlock], +) -> list[types.ContentBlock]: + """Convert v0 multimodal blocks to v1 format. + + Processes non_standard blocks that might be v0 format and converts them + to proper v1 ContentBlocks. + + Args: + blocks: List of content blocks to process. + + Returns: + Updated list with v0 blocks converted to v1 format. + """ + converted_blocks = [] + unpacked_blocks: list[dict[str, Any]] = [ + cast("dict[str, Any]", block) + if block.get("type") != "non_standard" + else block["value"] # type: ignore[typeddict-item] # this is only non-standard blocks + for block in blocks + ] + for block in unpacked_blocks: + if block.get("type") in {"image", "audio", "file"} and "source_type" in block: + converted_block = _convert_legacy_v0_content_block_to_v1(block) + converted_blocks.append(cast("types.ContentBlock", converted_block)) + elif block.get("type") in types.KNOWN_BLOCK_TYPES: + converted_blocks.append(cast("types.ContentBlock", block)) + else: + converted_blocks.append({"type": "non_standard", "value": block}) + + return converted_blocks + + +def _convert_legacy_v0_content_block_to_v1( + block: dict, +) -> Union[types.ContentBlock, dict]: + """Convert a LangChain v0 content block to v1 format. + + Preserves unknown keys as extras to avoid data loss. + + Returns the original block unchanged if it's not in v0 format. + + """ + + def _extract_v0_extras(block_dict: dict, known_keys: set[str]) -> dict[str, Any]: + """Extract unknown keys from v0 block to preserve as extras.""" + return {k: v for k, v in block_dict.items() if k not in known_keys} + + # Check if this is actually a v0 format block + block_type = block.get("type") + if block_type not in {"image", "audio", "file"} or "source_type" not in block: + # Not a v0 format block, return unchanged + return block + + if block.get("type") == "image": + source_type = block.get("source_type") + if source_type == "url": + known_keys = {"type", "source_type", "url", "mime_type"} + extras = _extract_v0_extras(block, known_keys) + if "id" in block: + return types.create_image_block( + url=block["url"], + mime_type=block.get("mime_type"), + id=block["id"], + **extras, + ) + + # Don't construct with an ID if not present in original block + v1_block = types.ImageContentBlock(type="image", url=block["url"]) + if block.get("mime_type"): + v1_block["mime_type"] = block["mime_type"] + + for key, value in extras.items(): + if value is not None: + v1_block["extras"] = {} + v1_block["extras"][key] = value + return v1_block + if source_type == "base64": + known_keys = {"type", "source_type", "data", "mime_type"} + extras = _extract_v0_extras(block, known_keys) + if "id" in block: + return types.create_image_block( + base64=block["data"], + mime_type=block.get("mime_type"), + id=block["id"], + **extras, + ) + + v1_block = types.ImageContentBlock(type="image", base64=block["data"]) + if block.get("mime_type"): + v1_block["mime_type"] = block["mime_type"] + + for key, value in extras.items(): + if value is not None: + v1_block["extras"] = {} + v1_block["extras"][key] = value + return v1_block + if source_type == "id": + known_keys = {"type", "source_type", "id"} + extras = _extract_v0_extras(block, known_keys) + # For id `source_type`, `id` is the file reference, not block ID + v1_block = types.ImageContentBlock(type="image", file_id=block["id"]) + + for key, value in extras.items(): + if value is not None: + v1_block["extras"] = {} + v1_block["extras"][key] = value + + return v1_block + elif block.get("type") == "audio": + source_type = block.get("source_type") + if source_type == "url": + known_keys = {"type", "source_type", "url", "mime_type"} + extras = _extract_v0_extras(block, known_keys) + return types.create_audio_block( + url=block["url"], mime_type=block.get("mime_type"), **extras + ) + if source_type == "base64": + known_keys = {"type", "source_type", "data", "mime_type"} + extras = _extract_v0_extras(block, known_keys) + return types.create_audio_block( + base64=block["data"], mime_type=block.get("mime_type"), **extras + ) + if source_type == "id": + known_keys = {"type", "source_type", "id"} + extras = _extract_v0_extras(block, known_keys) + return types.create_audio_block(file_id=block["id"], **extras) + elif block.get("type") == "file": + source_type = block.get("source_type") + if source_type == "url": + known_keys = {"type", "source_type", "url", "mime_type"} + extras = _extract_v0_extras(block, known_keys) + return types.create_file_block( + url=block["url"], mime_type=block.get("mime_type"), **extras + ) + if source_type == "base64": + known_keys = {"type", "source_type", "data", "mime_type"} + extras = _extract_v0_extras(block, known_keys) + return types.create_file_block( + base64=block["data"], mime_type=block.get("mime_type"), **extras + ) + if source_type == "id": + known_keys = {"type", "source_type", "id"} + extras = _extract_v0_extras(block, known_keys) + return types.create_file_block(file_id=block["id"], **extras) + if source_type == "text": + known_keys = {"type", "source_type", "url", "mime_type"} + extras = _extract_v0_extras(block, known_keys) + return types.create_plaintext_block( + # In v0, URL points to the text file content + text=block["url"], + **extras, + ) + + # If we can't convert, return the block unchanged + return block + + +def _convert_openai_format_to_data_block( + block: dict, +) -> Union[types.ContentBlock, dict[Any, Any]]: + """Convert OpenAI image/audio/file content block to respective v1 multimodal block. + + We expect that the incoming block is verified to be in OpenAI Chat Completions + format. + + If parsing fails, passes block through unchanged. + + Mappings (Chat Completions to LangChain v1): + - Image -> `ImageContentBlock` + - Audio -> `AudioContentBlock` + - File -> `FileContentBlock` + + """ + + # Extract extra keys to put them in `extras` + def _extract_extras(block_dict: dict, known_keys: set[str]) -> dict[str, Any]: + """Extract unknown keys from block to preserve as extras.""" + return {k: v for k, v in block_dict.items() if k not in known_keys} + + # base64-style image block + if (block["type"] == "image_url") and ( + parsed := _parse_data_uri(block["image_url"]["url"]) + ): + known_keys = {"type", "image_url"} + extras = _extract_extras(block, known_keys) + + # Also extract extras from nested image_url dict + image_url_known_keys = {"url"} + image_url_extras = _extract_extras(block["image_url"], image_url_known_keys) + + # Merge extras + all_extras = {**extras} + for key, value in image_url_extras.items(): + if key == "detail": # Don't rename + all_extras["detail"] = value + else: + all_extras[f"image_url_{key}"] = value + + return types.create_image_block( + # Even though this is labeled as `url`, it can be base64-encoded + base64=parsed["data"], + mime_type=parsed["mime_type"], + **all_extras, + ) + + # url-style image block + if (block["type"] == "image_url") and isinstance( + block["image_url"].get("url"), str + ): + known_keys = {"type", "image_url"} + extras = _extract_extras(block, known_keys) + + image_url_known_keys = {"url"} + image_url_extras = _extract_extras(block["image_url"], image_url_known_keys) + + all_extras = {**extras} + for key, value in image_url_extras.items(): + if key == "detail": # Don't rename + all_extras["detail"] = value + else: + all_extras[f"image_url_{key}"] = value + + return types.create_image_block( + url=block["image_url"]["url"], + **all_extras, + ) + + # base64-style audio block + # audio is only represented via raw data, no url or ID option + if block["type"] == "input_audio": + known_keys = {"type", "input_audio"} + extras = _extract_extras(block, known_keys) + + # Also extract extras from nested audio dict + audio_known_keys = {"data", "format"} + audio_extras = _extract_extras(block["input_audio"], audio_known_keys) + + all_extras = {**extras} + for key, value in audio_extras.items(): + all_extras[f"audio_{key}"] = value + + return types.create_audio_block( + base64=block["input_audio"]["data"], + mime_type=f"audio/{block['input_audio']['format']}", + **all_extras, + ) + + # id-style file block + if block.get("type") == "file" and "file_id" in block.get("file", {}): + known_keys = {"type", "file"} + extras = _extract_extras(block, known_keys) + + file_known_keys = {"file_id"} + file_extras = _extract_extras(block["file"], file_known_keys) + + all_extras = {**extras} + for key, value in file_extras.items(): + all_extras[f"file_{key}"] = value + + return types.create_file_block( + file_id=block["file"]["file_id"], + **all_extras, + ) + + # base64-style file block + if block["type"] == "file": + known_keys = {"type", "file"} + extras = _extract_extras(block, known_keys) + + file_known_keys = {"file_data", "filename"} + file_extras = _extract_extras(block["file"], file_known_keys) + + all_extras = {**extras} + for key, value in file_extras.items(): + all_extras[f"file_{key}"] = value + + filename = block["file"].get("filename") + return types.create_file_block( + base64=block["file"]["file_data"], + mime_type="application/pdf", + filename=filename, + **all_extras, + ) + + # Escape hatch + return block diff --git a/libs/core/langchain_core/messages/block_translators/ollama.py b/libs/core/langchain_core/messages/block_translators/ollama.py new file mode 100644 index 0000000000000..736ecfe06513e --- /dev/null +++ b/libs/core/langchain_core/messages/block_translators/ollama.py @@ -0,0 +1,45 @@ +"""Derivations of standard content blocks from Ollama content.""" + +import warnings + +from langchain_core.messages import AIMessage, AIMessageChunk +from langchain_core.messages import content as types + +WARNED = False + + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: # noqa: ARG001 + """Derive standard content blocks from a message with Ollama content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Ollama." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True + raise NotImplementedError + + +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: # noqa: ARG001 + """Derive standard content blocks from a message chunk with Ollama content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Ollama." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True + raise NotImplementedError + + +def _register_ollama_translator() -> None: + """Register the Ollama translator with the central registry. + + Run automatically when the module is imported. + """ + from langchain_core.messages.block_translators import register_translator + + register_translator("ollama", translate_content, translate_content_chunk) + + +_register_ollama_translator() diff --git a/libs/core/langchain_core/messages/block_translators/openai.py b/libs/core/langchain_core/messages/block_translators/openai.py new file mode 100644 index 0000000000000..a9e5db98d9d54 --- /dev/null +++ b/libs/core/langchain_core/messages/block_translators/openai.py @@ -0,0 +1,429 @@ +"""Derivations of standard content blocks from OpenAI content.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, Optional, Union, cast + +from langchain_core.language_models._utils import ( + _is_openai_data_block, +) +from langchain_core.messages import content as types +from langchain_core.messages.block_translators.langchain_v0 import ( + _convert_openai_format_to_data_block, +) + +if TYPE_CHECKING: + from langchain_core.messages import AIMessage, AIMessageChunk + + +# v1 / Chat Completions +def _convert_to_v1_from_chat_completions( + message: AIMessage, +) -> list[types.ContentBlock]: + """Mutate a Chat Completions message to v1 format.""" + content_blocks: list[types.ContentBlock] = [] + if isinstance(message.content, str): + if message.content: + content_blocks = [{"type": "text", "text": message.content}] + else: + content_blocks = [] + + for tool_call in message.tool_calls: + content_blocks.append( + { + "type": "tool_call", + "name": tool_call["name"], + "args": tool_call["args"], + "id": tool_call.get("id"), + } + ) + + return content_blocks + + +def _convert_to_v1_from_chat_completions_input( + blocks: list[types.ContentBlock], +) -> list[types.ContentBlock]: + """Convert OpenAI Chat Completions format blocks to v1 format. + + Processes non_standard blocks that might be OpenAI format and converts them + to proper ContentBlocks. If conversion fails, leaves them as non_standard. + + Args: + blocks: List of content blocks to process. + + Returns: + Updated list with OpenAI blocks converted to v1 format. + """ + from langchain_core.messages import content as types + + converted_blocks = [] + unpacked_blocks: list[dict[str, Any]] = [ + cast("dict[str, Any]", block) + if block.get("type") != "non_standard" + else block["value"] # type: ignore[typeddict-item] # this is only non-standard blocks + for block in blocks + ] + for block in unpacked_blocks: + if block.get("type") in { + "image_url", + "input_audio", + "file", + } and _is_openai_data_block(block): + converted_block = _convert_openai_format_to_data_block(block) + # If conversion succeeded, use it; otherwise keep as non_standard + if ( + isinstance(converted_block, dict) + and converted_block.get("type") in types.KNOWN_BLOCK_TYPES + ): + converted_blocks.append(cast("types.ContentBlock", converted_block)) + else: + converted_blocks.append({"type": "non_standard", "value": block}) + elif block.get("type") in types.KNOWN_BLOCK_TYPES: + converted_blocks.append(cast("types.ContentBlock", block)) + else: + converted_blocks.append({"type": "non_standard", "value": block}) + + return converted_blocks + + +def _convert_to_v1_from_chat_completions_chunk( + chunk: AIMessageChunk, +) -> list[types.ContentBlock]: + """Mutate a Chat Completions chunk to v1 format.""" + content_blocks: list[types.ContentBlock] = [] + if isinstance(chunk.content, str): + if chunk.content: + content_blocks = [{"type": "text", "text": chunk.content}] + else: + content_blocks = [] + + for tool_call_chunk in chunk.tool_call_chunks: + tc: types.ToolCallChunk = { + "type": "tool_call_chunk", + "id": tool_call_chunk.get("id"), + "name": tool_call_chunk.get("name"), + "args": tool_call_chunk.get("args"), + } + if (idx := tool_call_chunk.get("index")) is not None: + tc["index"] = idx + content_blocks.append(tc) + + return content_blocks + + +def _convert_from_v1_to_chat_completions(message: AIMessage) -> AIMessage: + """Convert a v1 message to the Chat Completions format.""" + if isinstance(message.content, list): + new_content: list = [] + for block in message.content: + if isinstance(block, dict): + block_type = block.get("type") + if block_type == "text": + # Strip annotations + new_content.append({"type": "text", "text": block["text"]}) + elif block_type in ("reasoning", "tool_call"): + pass + else: + new_content.append(block) + else: + new_content.append(block) + return message.model_copy(update={"content": new_content}) + + return message + + +# v1 / Responses +def _convert_annotation_to_v1(annotation: dict[str, Any]) -> types.Annotation: + annotation_type = annotation.get("type") + + if annotation_type == "url_citation": + known_fields = { + "type", + "url", + "title", + "cited_text", + "start_index", + "end_index", + } + url_citation = cast("types.Citation", {}) + for field in ("end_index", "start_index", "title"): + if field in annotation: + url_citation[field] = annotation[field] + url_citation["type"] = "citation" + url_citation["url"] = annotation["url"] + for field, value in annotation.items(): + if field not in known_fields: + if "extras" not in url_citation: + url_citation["extras"] = {} + url_citation["extras"][field] = value + return url_citation + + if annotation_type == "file_citation": + known_fields = { + "type", + "title", + "cited_text", + "start_index", + "end_index", + "filename", + } + document_citation: types.Citation = {"type": "citation"} + if "filename" in annotation: + document_citation["title"] = annotation["filename"] + for field, value in annotation.items(): + if field not in known_fields: + if "extras" not in document_citation: + document_citation["extras"] = {} + document_citation["extras"][field] = value + + return document_citation + + # TODO: standardise container_file_citation? + non_standard_annotation: types.NonStandardAnnotation = { + "type": "non_standard_annotation", + "value": annotation, + } + return non_standard_annotation + + +def _explode_reasoning(block: dict[str, Any]) -> Iterable[types.ReasoningContentBlock]: + if "summary" not in block: + yield cast("types.ReasoningContentBlock", block) + return + + known_fields = {"type", "reasoning", "id", "index"} + unknown_fields = [ + field for field in block if field != "summary" and field not in known_fields + ] + if unknown_fields: + block["extras"] = {} + for field in unknown_fields: + block["extras"][field] = block.pop(field) + + if not block["summary"]: + # [{'id': 'rs_...', 'summary': [], 'type': 'reasoning', 'index': 0}] + block = {k: v for k, v in block.items() if k != "summary"} + if "index" in block: + meaningful_idx = f"{block['index']}_0" + block["index"] = f"lc_rs_{meaningful_idx.encode().hex()}" + yield cast("types.ReasoningContentBlock", block) + return + + # Common part for every exploded line, except 'summary' + common = {k: v for k, v in block.items() if k in known_fields} + + # Optional keys that must appear only in the first exploded item + first_only = block.pop("extras", None) + + for idx, part in enumerate(block["summary"]): + new_block = dict(common) + new_block["reasoning"] = part.get("text", "") + if idx == 0 and first_only: + new_block.update(first_only) + if "index" in new_block: + summary_index = part.get("index", 0) + meaningful_idx = f"{new_block['index']}_{summary_index}" + new_block["index"] = f"lc_rs_{meaningful_idx.encode().hex()}" + + yield cast("types.ReasoningContentBlock", new_block) + + +def _convert_to_v1_from_responses(message: AIMessage) -> list[types.ContentBlock]: + """Convert a Responses message to v1 format.""" + + def _iter_blocks() -> Iterable[types.ContentBlock]: + for raw_block in message.content: + if not isinstance(raw_block, dict): + continue + block = raw_block.copy() + block_type = block.get("type") + + if block_type == "text": + if "text" not in block: + block["text"] = "" + if "annotations" in block: + block["annotations"] = [ + _convert_annotation_to_v1(a) for a in block["annotations"] + ] + if "index" in block: + block["index"] = f"lc_txt_{block['index']}" + yield cast("types.TextContentBlock", block) + + elif block_type == "reasoning": + yield from _explode_reasoning(block) + + elif block_type == "image_generation_call" and ( + result := block.get("result") + ): + new_block = {"type": "image", "base64": result} + if output_format := block.get("output_format"): + new_block["mime_type"] = f"image/{output_format}" + if "id" in block: + new_block["id"] = block["id"] + if "index" in block: + new_block["index"] = f"lc_img_{block['index']}" + for extra_key in ( + "status", + "background", + "output_format", + "quality", + "revised_prompt", + "size", + ): + if extra_key in block: + if "extras" not in new_block: + new_block["extras"] = {} + new_block["extras"][extra_key] = block[extra_key] + yield cast("types.ImageContentBlock", new_block) + + elif block_type == "function_call": + tool_call_block: Optional[ + Union[types.ToolCall, types.InvalidToolCall, types.ToolCallChunk] + ] = None + call_id = block.get("call_id", "") + + from langchain_core.messages import AIMessageChunk + + if ( + isinstance(message, AIMessageChunk) + and len(message.tool_call_chunks) == 1 + ): + tool_call_block = message.tool_call_chunks[0].copy() # type: ignore[assignment] + elif call_id: + for tool_call in message.tool_calls or []: + if tool_call.get("id") == call_id: + tool_call_block = { + "type": "tool_call", + "name": tool_call["name"], + "args": tool_call["args"], + "id": tool_call.get("id"), + } + break + else: + for invalid_tool_call in message.invalid_tool_calls or []: + if invalid_tool_call.get("id") == call_id: + tool_call_block = invalid_tool_call.copy() + break + else: + pass + if tool_call_block: + if "id" in block: + if "extras" not in tool_call_block: + tool_call_block["extras"] = {} + tool_call_block["extras"]["item_id"] = block["id"] + if "index" in block: + tool_call_block["index"] = f"lc_tc_{block['index']}" + yield tool_call_block + + elif block_type == "web_search_call": + web_search_call = {"type": "web_search_call", "id": block["id"]} + if "index" in block: + web_search_call["index"] = f"lc_wsc_{block['index']}" + if ( + "action" in block + and isinstance(block["action"], dict) + and block["action"].get("type") == "search" + and "query" in block["action"] + ): + web_search_call["query"] = block["action"]["query"] + for key in block: + if key not in ("type", "id", "index"): + web_search_call[key] = block[key] + + yield cast("types.WebSearchCall", web_search_call) + + # If .content already has web_search_result, don't add + if not any( + isinstance(other_block, dict) + and other_block.get("type") == "web_search_result" + and other_block.get("id") == block["id"] + for other_block in message.content + ): + web_search_result = {"type": "web_search_result", "id": block["id"]} + if "index" in block and isinstance(block["index"], int): + web_search_result["index"] = f"lc_wsr_{block['index'] + 1}" + yield cast("types.WebSearchResult", web_search_result) + + elif block_type == "code_interpreter_call": + code_interpreter_call = { + "type": "code_interpreter_call", + "id": block["id"], + } + if "code" in block: + code_interpreter_call["code"] = block["code"] + if "index" in block: + code_interpreter_call["index"] = f"lc_cic_{block['index']}" + known_fields = {"type", "id", "language", "code", "extras", "index"} + for key in block: + if key not in known_fields: + if "extras" not in code_interpreter_call: + code_interpreter_call["extras"] = {} + code_interpreter_call["extras"][key] = block[key] + + code_interpreter_result = { + "type": "code_interpreter_result", + "id": block["id"], + } + if "outputs" in block: + code_interpreter_result["outputs"] = block["outputs"] + for output in block["outputs"]: + if ( + isinstance(output, dict) + and (output_type := output.get("type")) + and output_type == "logs" + ): + if "output" not in code_interpreter_result: + code_interpreter_result["output"] = [] + code_interpreter_result["output"].append( + { + "type": "code_interpreter_output", + "stdout": output.get("logs", ""), + } + ) + + if "status" in block: + code_interpreter_result["status"] = block["status"] + if "index" in block and isinstance(block["index"], int): + code_interpreter_result["index"] = f"lc_cir_{block['index'] + 1}" + + yield cast("types.CodeInterpreterCall", code_interpreter_call) + yield cast("types.CodeInterpreterResult", code_interpreter_result) + + elif block_type in types.KNOWN_BLOCK_TYPES: + yield cast("types.ContentBlock", block) + else: + new_block = {"type": "non_standard", "value": block} + if "index" in new_block["value"]: + new_block["index"] = f"lc_ns_{new_block['value'].pop('index')}" + yield cast("types.NonStandardContentBlock", new_block) + + return list(_iter_blocks()) + + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: + """Derive standard content blocks from a message with OpenAI content.""" + if isinstance(message.content, str): + return _convert_to_v1_from_chat_completions(message) + return _convert_to_v1_from_responses(message) + + +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: + """Derive standard content blocks from a message chunk with OpenAI content.""" + if isinstance(message.content, str): + return _convert_to_v1_from_chat_completions_chunk(message) + return _convert_to_v1_from_responses(message) + + +def _register_openai_translator() -> None: + """Register the OpenAI translator with the central registry. + + Run automatically when the module is imported. + """ + from langchain_core.messages.block_translators import register_translator + + register_translator("openai", translate_content, translate_content_chunk) + + +_register_openai_translator() diff --git a/libs/core/langchain_core/messages/chat.py b/libs/core/langchain_core/messages/chat.py index a4791423fad79..fa44425302def 100644 --- a/libs/core/langchain_core/messages/chat.py +++ b/libs/core/langchain_core/messages/chat.py @@ -30,7 +30,10 @@ class ChatMessageChunk(ChatMessage, BaseMessageChunk): # non-chunk variant. type: Literal["ChatMessageChunk"] = "ChatMessageChunk" # type: ignore[assignment] """The type of the message (used during serialization). - Defaults to "ChatMessageChunk".""" + + Defaults to ``ChatMessageChunk``. + + """ @override def __add__(self, other: Any) -> BaseMessageChunk: # type: ignore[override] diff --git a/libs/core/langchain_core/messages/content.py b/libs/core/langchain_core/messages/content.py new file mode 100644 index 0000000000000..845c3b481cecc --- /dev/null +++ b/libs/core/langchain_core/messages/content.py @@ -0,0 +1,1568 @@ +"""Standard, multimodal content blocks for Large Language Model I/O. + +.. warning:: + This module is under active development. The API is unstable and subject to + change in future releases. + +This module provides standardized data structures for representing inputs to and +outputs from LLMs. The core abstraction is the **Content Block**, a ``TypedDict``. + +**Rationale** + +Different LLM providers use distinct and incompatible API schemas. This module +provides a unified, provider-agnostic format to facilitate these interactions. A +message to or from a model is simply a list of content blocks, allowing for the natural +interleaving of text, images, and other content in a single ordered sequence. + +An adapter for a specific provider is responsible for translating this standard list of +blocks into the format required by its API. + +**Extensibility** + +Data **not yet mapped** to a standard block may be represented using the +``NonStandardContentBlock``, which allows for provider-specific data to be included +without losing the benefits of type checking and validation. + +Furthermore, provider-specific fields **within** a standard block are fully supported +by default in the ``extras`` field of each block. This allows for additional metadata +to be included without breaking the standard structure. + +.. warning:: + Do not heavily rely on the ``extras`` field for provider-specific data! This field + is subject to deprecation in future releases as we move towards PEP 728. + +.. note:: + Following widespread adoption of `PEP 728 `__, we + will add ``extra_items=Any`` as a param to Content Blocks. This will signify to type + checkers that additional provider-specific fields are allowed outside of the + ``extras`` field, and that will become the new standard approach to adding + provider-specific metadata. + + .. dropdown:: + + **Example with PEP 728 provider-specific fields:** + + .. code-block:: python + + # Content block definition + # NOTE: `extra_items=Any` + class TextContentBlock(TypedDict, extra_items=Any): + type: Literal["text"] + id: NotRequired[str] + text: str + annotations: NotRequired[list[Annotation]] + index: NotRequired[int] + + .. code-block:: python + + from langchain_core.messages.content import TextContentBlock + + # Create a text content block with provider-specific fields + my_block: TextContentBlock = { + # Add required fields + "type": "text", + "text": "Hello, world!", + # Additional fields not specified in the TypedDict + # These are valid with PEP 728 and are typed as Any + "openai_metadata": {"model": "gpt-4", "temperature": 0.7}, + "anthropic_usage": {"input_tokens": 10, "output_tokens": 20}, + "custom_field": "any value", + } + + # Mutating an existing block to add provider-specific fields + openai_data = my_block["openai_metadata"] # Type: Any + + PEP 728 is enabled with ``# type: ignore[call-arg]`` comments to suppress + warnings from type checkers that don't yet support it. The functionality works + correctly in Python 3.13+ and will be fully supported as the ecosystem catches + up. + +**Key Block Types** + +The module defines several types of content blocks, including: + +- ``TextContentBlock``: Standard text output. +- ``Citation``: For annotations that link text output to a source document. +- ``ToolCallContentBlock``: For function calling. +- ``ReasoningContentBlock``: To capture a model's thought process. +- Multimodal data: + - ``ImageContentBlock`` + - ``AudioContentBlock`` + - ``VideoContentBlock`` + - ``PlainTextContentBlock`` (e.g. .txt or .md files) + - ``FileContentBlock`` (e.g. PDFs, etc.) + +**Example Usage** + +.. code-block:: python + + # Direct construction: + from langchain_core.messages.content import TextContentBlock, ImageContentBlock + + multimodal_message: AIMessage(content_blocks= + [ + TextContentBlock(type="text", text="What is shown in this image?"), + ImageContentBlock( + type="image", + url="https://www.langchain.com/images/brand/langchain_logo_text_w_white.png", + mime_type="image/png", + ), + ] + ) + + # Using factories: + from langchain_core.messages.content import create_text_block, create_image_block + + multimodal_message: AIMessage(content= + [ + create_text_block("What is shown in this image?"), + create_image_block( + url="https://www.langchain.com/images/brand/langchain_logo_text_w_white.png", + mime_type="image/png", + ), + ] + ) + +Factory functions offer benefits such as: +- Automatic ID generation (when not provided) +- No need to manually specify the ``type`` field + +""" + +import warnings +from typing import Any, Literal, Optional, Union, get_args, get_type_hints + +from typing_extensions import NotRequired, TypedDict, TypeGuard + +from langchain_core.utils.utils import ensure_id + + +class Citation(TypedDict): + """Annotation for citing data from a document. + + .. note:: + ``start``/``end`` indices refer to the **response text**, + not the source text. This means that the indices are relative to the model's + response, not the original document (as specified in the ``url``). + + .. note:: + ``create_citation`` may also be used as a factory to create a ``Citation``. + Benefits include: + + * Automatic ID generation (when not provided) + * Required arguments strictly validated at creation time + + """ + + type: Literal["citation"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. Either: + + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + url: NotRequired[str] + """URL of the document source.""" + + title: NotRequired[str] + """Source document title. + + For example, the page title for a web page or the title of a paper. + """ + + start_index: NotRequired[int] + """Start index of the **response text** (``TextContentBlock.text``).""" + + end_index: NotRequired[int] + """End index of the **response text** (``TextContentBlock.text``)""" + + cited_text: NotRequired[str] + """Excerpt of source text being cited.""" + + # NOTE: not including spans for the raw document text (such as `text_start_index` + # and `text_end_index`) as this is not currently supported by any provider. The + # thinking is that the `cited_text` should be sufficient for most use cases, and it + # is difficult to reliably extract spans from the raw document text across file + # formats or encoding schemes. + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata.""" + + +class NonStandardAnnotation(TypedDict): + """Provider-specific annotation format.""" + + type: Literal["non_standard_annotation"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + value: dict[str, Any] + """Provider-specific annotation data.""" + + +Annotation = Union[Citation, NonStandardAnnotation] + + +class TextContentBlock(TypedDict): + """Text output from a LLM. + + This typically represents the main text content of a message, such as the response + from a language model or the text of a user message. + + .. note:: + ``create_text_block`` may also be used as a factory to create a + ``TextContentBlock``. Benefits include: + + * Automatic ID generation (when not provided) + * Required arguments strictly validated at creation time + + """ + + type: Literal["text"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + text: str + """Block text.""" + + annotations: NotRequired[list[Annotation]] + """``Citation``s and other annotations.""" + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata.""" + + +class ToolCall(TypedDict): + """Represents a request to call a tool. + + Example: + + .. code-block:: python + + { + "name": "foo", + "args": {"a": 1}, + "id": "123" + } + + This represents a request to call the tool named "foo" with arguments {"a": 1} + and an identifier of "123". + + .. note:: + ``create_tool_call`` may also be used as a factory to create a + ``ToolCall``. Benefits include: + + * Automatic ID generation (when not provided) + * Required arguments strictly validated at creation time + + """ + + type: Literal["tool_call"] + """Used for discrimination.""" + + id: Optional[str] + """An identifier associated with the tool call. + + An identifier is needed to associate a tool call request with a tool + call result in events when multiple concurrent tool calls are made. + + """ + # TODO: Consider making this NotRequired[str] in the future. + + name: str + """The name of the tool to be called.""" + + args: dict[str, Any] + """The arguments to the tool call.""" + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata.""" + + +class ToolCallChunk(TypedDict): + """A chunk of a tool call (e.g., as part of a stream). + + When merging ``ToolCallChunks`` (e.g., via ``AIMessageChunk.__add__``), + all string attributes are concatenated. Chunks are only merged if their + values of ``index`` are equal and not ``None``. + + Example: + + .. code-block:: python + + left_chunks = [ToolCallChunk(name="foo", args='{"a":', index=0)] + right_chunks = [ToolCallChunk(name=None, args='1}', index=0)] + + ( + AIMessageChunk(content="", tool_call_chunks=left_chunks) + + AIMessageChunk(content="", tool_call_chunks=right_chunks) + ).tool_call_chunks == [ToolCallChunk(name='foo', args='{"a":1}', index=0)] + + """ + + # TODO: Consider making fields NotRequired[str] in the future. + + type: Literal["tool_call_chunk"] + """Used for serialization.""" + + id: Optional[str] + """An identifier associated with the tool call. + + An identifier is needed to associate a tool call request with a tool + call result in events when multiple concurrent tool calls are made. + + """ + + name: Optional[str] + """The name of the tool to be called.""" + + args: Optional[str] + """The arguments to the tool call.""" + + index: NotRequired[Union[int, str]] + """The index of the tool call in a sequence.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata.""" + + +class InvalidToolCall(TypedDict): + """Allowance for errors made by LLM. + + Here we add an ``error`` key to surface errors made during generation + (e.g., invalid JSON arguments.) + + """ + + # TODO: Consider making fields NotRequired[str] in the future. + + type: Literal["invalid_tool_call"] + """Used for discrimination.""" + + id: Optional[str] + """An identifier associated with the tool call. + + An identifier is needed to associate a tool call request with a tool + call result in events when multiple concurrent tool calls are made. + + """ + + name: Optional[str] + """The name of the tool to be called.""" + + args: Optional[str] + """The arguments to the tool call.""" + + error: Optional[str] + """An error message associated with the tool call.""" + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata.""" + + +class WebSearchCall(TypedDict): + """Built-in web search tool call.""" + + type: Literal["web_search_call"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + query: NotRequired[str] + """The search query used in the web search tool call.""" + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata.""" + + +class WebSearchResult(TypedDict): + """Result of a built-in web search tool call.""" + + type: Literal["web_search_result"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + urls: NotRequired[list[str]] + """List of URLs returned by the web search tool call.""" + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata.""" + + +class CodeInterpreterCall(TypedDict): + """Built-in code interpreter tool call.""" + + type: Literal["code_interpreter_call"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + language: NotRequired[str] + """The name of the programming language used in the code interpreter tool call.""" + + code: NotRequired[str] + """The code to be executed by the code interpreter.""" + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata.""" + + +class CodeInterpreterOutput(TypedDict): + """Output of a singular code interpreter tool call. + + Full output of a code interpreter tool call is represented by + ``CodeInterpreterResult`` which is a list of these blocks. + + """ + + type: Literal["code_interpreter_output"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + return_code: NotRequired[int] + """Return code of the executed code. + + Example: ``0`` for success, non-zero for failure. + + """ + + stderr: NotRequired[str] + """Standard error output of the executed code.""" + + stdout: NotRequired[str] + """Standard output of the executed code.""" + + file_ids: NotRequired[list[str]] + """List of file IDs generated by the code interpreter.""" + + +class CodeInterpreterResult(TypedDict): + """Result of a code interpreter tool call.""" + + type: Literal["code_interpreter_result"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + output: list[CodeInterpreterOutput] + """List of outputs from the code interpreter tool call.""" + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata.""" + + +class ReasoningContentBlock(TypedDict): + """Reasoning output from a LLM. + + .. note:: + ``create_reasoning_block`` may also be used as a factory to create a + ``ReasoningContentBlock``. Benefits include: + + * Automatic ID generation (when not provided) + * Required arguments strictly validated at creation time + + """ + + type: Literal["reasoning"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + reasoning: NotRequired[str] + """Reasoning text. + + Either the thought summary or the raw reasoning text itself. This is often parsed + from ```` tags in the model's response. + + """ + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata.""" + + +# Note: `title` and `context` are fields that could be used to provide additional +# information about the file, such as a description or summary of its content. +# E.g. with Claude, you can provide a context for a file which is passed to the model. +class ImageContentBlock(TypedDict): + """Image data. + + .. note:: + ``create_image_block`` may also be used as a factory to create a + ``ImageContentBlock``. Benefits include: + + * Automatic ID generation (when not provided) + * Required arguments strictly validated at creation time + + """ + + type: Literal["image"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + file_id: NotRequired[str] + """ID of the image file, e.g., from a file storage system.""" + + mime_type: NotRequired[str] + """MIME type of the image. Required for base64. + + `Examples from IANA `__ + + """ + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + url: NotRequired[str] + """URL of the image.""" + + base64: NotRequired[str] + """Data as a base64 string.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata. This shouldn't be used for the image data itself.""" + + +class VideoContentBlock(TypedDict): + """Video data. + + .. note:: + ``create_video_block`` may also be used as a factory to create a + ``VideoContentBlock``. Benefits include: + + * Automatic ID generation (when not provided) + * Required arguments strictly validated at creation time + + """ + + type: Literal["video"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + file_id: NotRequired[str] + """ID of the video file, e.g., from a file storage system.""" + + mime_type: NotRequired[str] + """MIME type of the video. Required for base64. + + `Examples from IANA `__ + + """ + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + url: NotRequired[str] + """URL of the video.""" + + base64: NotRequired[str] + """Data as a base64 string.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata. This shouldn't be used for the video data itself.""" + + +class AudioContentBlock(TypedDict): + """Audio data. + + .. note:: + ``create_audio_block`` may also be used as a factory to create an + ``AudioContentBlock``. Benefits include: + * Automatic ID generation (when not provided) + * Required arguments strictly validated at creation time + + """ + + type: Literal["audio"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + file_id: NotRequired[str] + """ID of the audio file, e.g., from a file storage system.""" + + mime_type: NotRequired[str] + """MIME type of the audio. Required for base64. + + `Examples from IANA `__ + + """ + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + url: NotRequired[str] + """URL of the audio.""" + + base64: NotRequired[str] + """Data as a base64 string.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata. This shouldn't be used for the audio data itself.""" + + +class PlainTextContentBlock(TypedDict): + """Plaintext data (e.g., from a document). + + .. note:: + Title and context are optional fields that may be passed to the model. See + Anthropic `example `__. + + .. note:: + ``create_plaintext_block`` may also be used as a factory to create a + ``PlainTextContentBlock``. Benefits include: + + * Automatic ID generation (when not provided) + * Required arguments strictly validated at creation time + + """ + + type: Literal["text-plain"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + file_id: NotRequired[str] + """ID of the plaintext file, e.g., from a file storage system.""" + + mime_type: Literal["text/plain"] + """MIME type of the file. Required for base64.""" + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + url: NotRequired[str] + """URL of the plaintext.""" + + base64: NotRequired[str] + """Data as a base64 string.""" + + text: NotRequired[str] + """Plaintext content. This is optional if the data is provided as base64.""" + + title: NotRequired[str] + """Title of the text data, e.g., the title of a document.""" + + context: NotRequired[str] + """Context for the text, e.g., a description or summary of the text's content.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata. This shouldn't be used for the data itself.""" + + +class FileContentBlock(TypedDict): + """File data that doesn't fit into other multimodal blocks. + + This block is intended for files that are not images, audio, or plaintext. For + example, it can be used for PDFs, Word documents, etc. + + If the file is an image, audio, or plaintext, you should use the corresponding + content block type (e.g., ``ImageContentBlock``, ``AudioContentBlock``, + ``PlainTextContentBlock``). + + .. note:: + ``create_file_block`` may also be used as a factory to create a + ``FileContentBlock``. Benefits include: + + * Automatic ID generation (when not provided) + * Required arguments strictly validated at creation time + + """ + + type: Literal["file"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + file_id: NotRequired[str] + """ID of the file, e.g., from a file storage system.""" + + mime_type: NotRequired[str] + """MIME type of the file. Required for base64. + + `Examples from IANA `__ + + """ + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + url: NotRequired[str] + """URL of the file.""" + + base64: NotRequired[str] + """Data as a base64 string.""" + + extras: NotRequired[dict[str, Any]] + """Provider-specific metadata. This shouldn't be used for the file data itself.""" + + +# Future modalities to consider: +# - 3D models +# - Tabular data + + +class NonStandardContentBlock(TypedDict): + """Provider-specific data. + + This block contains data for which there is not yet a standard type. + + The purpose of this block should be to simply hold a provider-specific payload. + If a provider's non-standard output includes reasoning and tool calls, it should be + the adapter's job to parse that payload and emit the corresponding standard + ``ReasoningContentBlock`` and ``ToolCallContentBlocks``. + + Has no ``extras`` field, as provider-specific data should be included in the + ``value`` field. + + .. note:: + ``create_non_standard_block`` may also be used as a factory to create a + ``NonStandardContentBlock``. Benefits include: + + * Automatic ID generation (when not provided) + * Required arguments strictly validated at creation time + + """ + + type: Literal["non_standard"] + """Type of the content block. Used for discrimination.""" + + id: NotRequired[str] + """Content block identifier. + + Either: + - Generated by the provider (e.g., OpenAI's file ID) + - Generated by LangChain upon creation (``UUID4`` prefixed with ``'lc_'``)) + + """ + + value: dict[str, Any] + """Provider-specific data.""" + + index: NotRequired[Union[int, str]] + """Index of block in aggregate response. Used during streaming.""" + + +# --- Aliases --- +DataContentBlock = Union[ + ImageContentBlock, + VideoContentBlock, + AudioContentBlock, + PlainTextContentBlock, + FileContentBlock, +] + +ToolContentBlock = Union[ + ToolCall, + ToolCallChunk, + CodeInterpreterCall, + CodeInterpreterResult, + WebSearchCall, + WebSearchResult, +] + +ContentBlock = Union[ + TextContentBlock, + InvalidToolCall, + ReasoningContentBlock, + NonStandardContentBlock, + DataContentBlock, + ToolContentBlock, +] + + +KNOWN_BLOCK_TYPES = { + # Text output + "text", + "reasoning", + # Tools + "tool_call", + "invalid_tool_call", + "tool_call_chunk", + # Multimodal data + "image", + "audio", + "file", + "text-plain", + "video", + # Server-side tool calls + "code_interpreter_call", + "code_interpreter_result", + "web_search_call", + "web_search_result", + # Catch-all + "non_standard", +} + + +def _get_data_content_block_types() -> tuple[str, ...]: + """Get type literals from DataContentBlock union members dynamically.""" + data_block_types = [] + + for block_type in get_args(DataContentBlock): + hints = get_type_hints(block_type) + if "type" in hints: + type_annotation = hints["type"] + if hasattr(type_annotation, "__args__"): + # This is a Literal type, get the literal value + literal_value = type_annotation.__args__[0] + data_block_types.append(literal_value) + + return tuple(data_block_types) + + +def is_data_content_block(block: dict) -> bool: + """Check if the provided content block is a standard v1 data content block. + + Args: + block: The content block to check. + + Returns: + True if the content block is a data content block, False otherwise. + + """ + if block.get("type") not in _get_data_content_block_types(): + return False + + if any(key in block for key in ("url", "base64", "file_id", "text")): + return True + + # Verify data presence based on source type + if "source_type" in block: + source_type = block["source_type"] + if (source_type == "url" and "url" in block) or ( + source_type == "base64" and "data" in block + ): + return True + if (source_type == "id" and "id" in block) or ( + source_type == "text" and "url" in block + ): + return True + + return False + + +def is_tool_call_block(block: ContentBlock) -> TypeGuard[ToolCall]: + """Type guard to check if a content block is a ``ToolCall``.""" + return block.get("type") == "tool_call" + + +def is_tool_call_chunk(block: ContentBlock) -> TypeGuard[ToolCallChunk]: + """Type guard to check if a content block is a ``ToolCallChunk``.""" + return block.get("type") == "tool_call_chunk" + + +def is_text_block(block: ContentBlock) -> TypeGuard[TextContentBlock]: + """Type guard to check if a content block is a ``TextContentBlock``.""" + return block.get("type") == "text" + + +def is_reasoning_block(block: ContentBlock) -> TypeGuard[ReasoningContentBlock]: + """Type guard to check if a content block is a ``ReasoningContentBlock``.""" + return block.get("type") == "reasoning" + + +def is_invalid_tool_call_block( + block: ContentBlock, +) -> TypeGuard[InvalidToolCall]: + """Type guard to check if a content block is an ``InvalidToolCall``.""" + return block.get("type") == "invalid_tool_call" + + +def convert_to_openai_image_block(block: dict[str, Any]) -> dict: + """Convert ``ImageContentBlock`` to format expected by OpenAI Chat Completions.""" + if "url" in block: + return { + "type": "image_url", + "image_url": { + "url": block["url"], + }, + } + if "base64" in block or block.get("source_type") == "base64": + if "mime_type" not in block: + error_message = "mime_type key is required for base64 data." + raise ValueError(error_message) + mime_type = block["mime_type"] + base64_data = block["data"] if "data" in block else block["base64"] + return { + "type": "image_url", + "image_url": { + "url": f"data:{mime_type};base64,{base64_data}", + }, + } + error_message = "Unsupported source type. Only 'url' and 'base64' are supported." + raise ValueError(error_message) + + +def convert_to_openai_data_block(block: dict) -> dict: + """Format standard data content block to format expected by OpenAI.""" + if block["type"] == "image": + formatted_block = convert_to_openai_image_block(block) + + elif block["type"] == "file": + if "base64" in block or block.get("source_type") == "base64": + # Handle v0 format: {"source_type": "base64", "data": "...", ...} + # Handle v1 format: {"base64": "...", ...} + base64_data = block["data"] if "source_type" in block else block["base64"] + file = {"file_data": f"data:{block['mime_type']};base64,{base64_data}"} + if filename := block.get("filename"): + file["filename"] = filename + elif (extras := block.get("extras")) and ("filename" in extras): + file["filename"] = extras["filename"] + elif (extras := block.get("metadata")) and ("filename" in extras): + # Backward compat + file["filename"] = extras["filename"] + else: + warnings.warn( + "OpenAI may require a filename for file inputs. Specify a filename " + "in the content block: {'type': 'file', 'mime_type': " + "'application/pdf', 'base64': '...', 'filename': 'my-pdf'}", + stacklevel=1, + ) + formatted_block = {"type": "file", "file": file} + elif "file_id" in block or block.get("source_type") == "id": + # Handle v0 format: {"source_type": "id", "id": "...", ...} + # Handle v1 format: {"file_id": "...", ...} + file_id = block["id"] if "source_type" in block else block["file_id"] + formatted_block = {"type": "file", "file": {"file_id": file_id}} + else: + error_msg = "Keys base64 or file_id required for file blocks." + raise ValueError(error_msg) + + elif block["type"] == "audio": + if "base64" in block or block.get("source_type") == "base64": + # Handle v0 format: {"source_type": "base64", "data": "...", ...} + # Handle v1 format: {"base64": "...", ...} + base64_data = block["data"] if "source_type" in block else block["base64"] + audio_format = block["mime_type"].split("/")[-1] + formatted_block = { + "type": "input_audio", + "input_audio": {"data": base64_data, "format": audio_format}, + } + else: + error_msg = "Key base64 is required for audio blocks." + raise ValueError(error_msg) + else: + error_msg = f"Block of type {block['type']} is not supported." + raise ValueError(error_msg) + + return formatted_block + + +def create_text_block( + text: str, + *, + id: Optional[str] = None, + annotations: Optional[list[Annotation]] = None, + index: Optional[Union[int, str]] = None, + **kwargs: Any, +) -> TextContentBlock: + """Create a ``TextContentBlock``. + + Args: + text: The text content of the block. + id: Content block identifier. Generated automatically if not provided. + annotations: ``Citation``s and other annotations for the text. + index: Index of block in aggregate response. Used during streaming. + + Returns: + A properly formatted ``TextContentBlock``. + + .. note:: + The ``id`` is generated automatically if not provided, using a UUID4 format + prefixed with ``'lc_'`` to indicate it is a LangChain-generated ID. + + """ + block = TextContentBlock( + type="text", + text=text, + id=ensure_id(id), + ) + if annotations is not None: + block["annotations"] = annotations + if index is not None: + block["index"] = index + + extras = {k: v for k, v in kwargs.items() if v is not None} + if extras: + block["extras"] = extras + + return block + + +def create_image_block( + *, + url: Optional[str] = None, + base64: Optional[str] = None, + file_id: Optional[str] = None, + mime_type: Optional[str] = None, + id: Optional[str] = None, + index: Optional[Union[int, str]] = None, + **kwargs: Any, +) -> ImageContentBlock: + """Create an ``ImageContentBlock``. + + Args: + url: URL of the image. + base64: Base64-encoded image data. + file_id: ID of the image file from a file storage system. + mime_type: MIME type of the image. Required for base64 data. + id: Content block identifier. Generated automatically if not provided. + index: Index of block in aggregate response. Used during streaming. + + Returns: + A properly formatted ``ImageContentBlock``. + + Raises: + ValueError: If no image source is provided or if ``base64`` is used without + ``mime_type``. + + .. note:: + The ``id`` is generated automatically if not provided, using a UUID4 format + prefixed with ``'lc_'`` to indicate it is a LangChain-generated ID. + + """ + if not any([url, base64, file_id]): + msg = "Must provide one of: url, base64, or file_id" + raise ValueError(msg) + + block = ImageContentBlock(type="image", id=ensure_id(id)) + + if url is not None: + block["url"] = url + if base64 is not None: + block["base64"] = base64 + if file_id is not None: + block["file_id"] = file_id + if mime_type is not None: + block["mime_type"] = mime_type + if index is not None: + block["index"] = index + + extras = {k: v for k, v in kwargs.items() if v is not None} + if extras: + block["extras"] = extras + + return block + + +def create_video_block( + *, + url: Optional[str] = None, + base64: Optional[str] = None, + file_id: Optional[str] = None, + mime_type: Optional[str] = None, + id: Optional[str] = None, + index: Optional[Union[int, str]] = None, + **kwargs: Any, +) -> VideoContentBlock: + """Create a ``VideoContentBlock``. + + Args: + url: URL of the video. + base64: Base64-encoded video data. + file_id: ID of the video file from a file storage system. + mime_type: MIME type of the video. Required for base64 data. + id: Content block identifier. Generated automatically if not provided. + index: Index of block in aggregate response. Used during streaming. + + Returns: + A properly formatted ``VideoContentBlock``. + + Raises: + ValueError: If no video source is provided or if ``base64`` is used without + ``mime_type``. + + .. note:: + The ``id`` is generated automatically if not provided, using a UUID4 format + prefixed with ``'lc_'`` to indicate it is a LangChain-generated ID. + + """ + if not any([url, base64, file_id]): + msg = "Must provide one of: url, base64, or file_id" + raise ValueError(msg) + + if base64 and not mime_type: + msg = "mime_type is required when using base64 data" + raise ValueError(msg) + + block = VideoContentBlock(type="video", id=ensure_id(id)) + + if url is not None: + block["url"] = url + if base64 is not None: + block["base64"] = base64 + if file_id is not None: + block["file_id"] = file_id + if mime_type is not None: + block["mime_type"] = mime_type + if index is not None: + block["index"] = index + + extras = {k: v for k, v in kwargs.items() if v is not None} + if extras: + block["extras"] = extras + + return block + + +def create_audio_block( + *, + url: Optional[str] = None, + base64: Optional[str] = None, + file_id: Optional[str] = None, + mime_type: Optional[str] = None, + id: Optional[str] = None, + index: Optional[Union[int, str]] = None, + **kwargs: Any, +) -> AudioContentBlock: + """Create an ``AudioContentBlock``. + + Args: + url: URL of the audio. + base64: Base64-encoded audio data. + file_id: ID of the audio file from a file storage system. + mime_type: MIME type of the audio. Required for base64 data. + id: Content block identifier. Generated automatically if not provided. + index: Index of block in aggregate response. Used during streaming. + + Returns: + A properly formatted ``AudioContentBlock``. + + Raises: + ValueError: If no audio source is provided or if ``base64`` is used without + ``mime_type``. + + .. note:: + The ``id`` is generated automatically if not provided, using a UUID4 format + prefixed with ``'lc_'`` to indicate it is a LangChain-generated ID. + + """ + if not any([url, base64, file_id]): + msg = "Must provide one of: url, base64, or file_id" + raise ValueError(msg) + + if base64 and not mime_type: + msg = "mime_type is required when using base64 data" + raise ValueError(msg) + + block = AudioContentBlock(type="audio", id=ensure_id(id)) + + if url is not None: + block["url"] = url + if base64 is not None: + block["base64"] = base64 + if file_id is not None: + block["file_id"] = file_id + if mime_type is not None: + block["mime_type"] = mime_type + if index is not None: + block["index"] = index + + extras = {k: v for k, v in kwargs.items() if v is not None} + if extras: + block["extras"] = extras + + return block + + +def create_file_block( + *, + url: Optional[str] = None, + base64: Optional[str] = None, + file_id: Optional[str] = None, + mime_type: Optional[str] = None, + id: Optional[str] = None, + index: Optional[Union[int, str]] = None, + **kwargs: Any, +) -> FileContentBlock: + """Create a ``FileContentBlock``. + + Args: + url: URL of the file. + base64: Base64-encoded file data. + file_id: ID of the file from a file storage system. + mime_type: MIME type of the file. Required for base64 data. + id: Content block identifier. Generated automatically if not provided. + index: Index of block in aggregate response. Used during streaming. + + Returns: + A properly formatted ``FileContentBlock``. + + Raises: + ValueError: If no file source is provided or if ``base64`` is used without + ``mime_type``. + + .. note:: + The ``id`` is generated automatically if not provided, using a UUID4 format + prefixed with ``'lc_'`` to indicate it is a LangChain-generated ID. + + """ + if not any([url, base64, file_id]): + msg = "Must provide one of: url, base64, or file_id" + raise ValueError(msg) + + if base64 and not mime_type: + msg = "mime_type is required when using base64 data" + raise ValueError(msg) + + block = FileContentBlock(type="file", id=ensure_id(id)) + + if url is not None: + block["url"] = url + if base64 is not None: + block["base64"] = base64 + if file_id is not None: + block["file_id"] = file_id + if mime_type is not None: + block["mime_type"] = mime_type + if index is not None: + block["index"] = index + + extras = {k: v for k, v in kwargs.items() if v is not None} + if extras: + block["extras"] = extras + + return block + + +def create_plaintext_block( + text: Optional[str] = None, + url: Optional[str] = None, + base64: Optional[str] = None, + file_id: Optional[str] = None, + title: Optional[str] = None, + context: Optional[str] = None, + id: Optional[str] = None, + index: Optional[Union[int, str]] = None, + **kwargs: Any, +) -> PlainTextContentBlock: + """Create a ``PlainTextContentBlock``. + + Args: + text: The plaintext content. + url: URL of the plaintext file. + base64: Base64-encoded plaintext data. + file_id: ID of the plaintext file from a file storage system. + title: Title of the text data. + context: Context or description of the text content. + id: Content block identifier. Generated automatically if not provided. + index: Index of block in aggregate response. Used during streaming. + + Returns: + A properly formatted ``PlainTextContentBlock``. + + .. note:: + The ``id`` is generated automatically if not provided, using a UUID4 format + prefixed with ``'lc_'`` to indicate it is a LangChain-generated ID. + + """ + block = PlainTextContentBlock( + type="text-plain", + mime_type="text/plain", + id=ensure_id(id), + ) + + if text is not None: + block["text"] = text + if url is not None: + block["url"] = url + if base64 is not None: + block["base64"] = base64 + if file_id is not None: + block["file_id"] = file_id + if title is not None: + block["title"] = title + if context is not None: + block["context"] = context + if index is not None: + block["index"] = index + + extras = {k: v for k, v in kwargs.items() if v is not None} + if extras: + block["extras"] = extras + + return block + + +def create_tool_call( + name: str, + args: dict[str, Any], + *, + id: Optional[str] = None, + index: Optional[Union[int, str]] = None, + **kwargs: Any, +) -> ToolCall: + """Create a ``ToolCall``. + + Args: + name: The name of the tool to be called. + args: The arguments to the tool call. + id: An identifier for the tool call. Generated automatically if not provided. + index: Index of block in aggregate response. Used during streaming. + + Returns: + A properly formatted ``ToolCall``. + + .. note:: + The ``id`` is generated automatically if not provided, using a UUID4 format + prefixed with ``'lc_'`` to indicate it is a LangChain-generated ID. + + """ + block = ToolCall( + type="tool_call", + name=name, + args=args, + id=ensure_id(id), + ) + + if index is not None: + block["index"] = index + + extras = {k: v for k, v in kwargs.items() if v is not None} + if extras: + block["extras"] = extras + + return block + + +def create_reasoning_block( + reasoning: Optional[str] = None, + id: Optional[str] = None, + index: Optional[Union[int, str]] = None, + **kwargs: Any, +) -> ReasoningContentBlock: + """Create a ``ReasoningContentBlock``. + + Args: + reasoning: The reasoning text or thought summary. + id: Content block identifier. Generated automatically if not provided. + index: Index of block in aggregate response. Used during streaming. + + Returns: + A properly formatted ``ReasoningContentBlock``. + + .. note:: + The ``id`` is generated automatically if not provided, using a UUID4 format + prefixed with ``'lc_'`` to indicate it is a LangChain-generated ID. + + """ + block = ReasoningContentBlock( + type="reasoning", + reasoning=reasoning or "", + id=ensure_id(id), + ) + + if index is not None: + block["index"] = index + + extras = {k: v for k, v in kwargs.items() if v is not None} + if extras: + block["extras"] = extras + + return block + + +def create_citation( + *, + url: Optional[str] = None, + title: Optional[str] = None, + start_index: Optional[int] = None, + end_index: Optional[int] = None, + cited_text: Optional[str] = None, + id: Optional[str] = None, + **kwargs: Any, +) -> Citation: + """Create a ``Citation``. + + Args: + url: URL of the document source. + title: Source document title. + start_index: Start index in the response text where citation applies. + end_index: End index in the response text where citation applies. + cited_text: Excerpt of source text being cited. + id: Content block identifier. Generated automatically if not provided. + + Returns: + A properly formatted ``Citation``. + + .. note:: + The ``id`` is generated automatically if not provided, using a UUID4 format + prefixed with ``'lc_'`` to indicate it is a LangChain-generated ID. + + """ + block = Citation(type="citation", id=ensure_id(id)) + + if url is not None: + block["url"] = url + if title is not None: + block["title"] = title + if start_index is not None: + block["start_index"] = start_index + if end_index is not None: + block["end_index"] = end_index + if cited_text is not None: + block["cited_text"] = cited_text + + extras = {k: v for k, v in kwargs.items() if v is not None} + if extras: + block["extras"] = extras + + return block + + +def create_non_standard_block( + value: dict[str, Any], + *, + id: Optional[str] = None, + index: Optional[Union[int, str]] = None, +) -> NonStandardContentBlock: + """Create a ``NonStandardContentBlock``. + + Args: + value: Provider-specific data. + id: Content block identifier. Generated automatically if not provided. + index: Index of block in aggregate response. Used during streaming. + + Returns: + A properly formatted ``NonStandardContentBlock``. + + .. note:: + The ``id`` is generated automatically if not provided, using a UUID4 format + prefixed with ``'lc_'`` to indicate it is a LangChain-generated ID. + + """ + block = NonStandardContentBlock( + type="non_standard", + value=value, + id=ensure_id(id), + ) + + if index is not None: + block["index"] = index + + return block diff --git a/libs/core/langchain_core/messages/content_blocks.py b/libs/core/langchain_core/messages/content_blocks.py deleted file mode 100644 index 83a66fb123a42..0000000000000 --- a/libs/core/langchain_core/messages/content_blocks.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Types for content blocks.""" - -import warnings -from typing import Any, Literal, Union - -from pydantic import TypeAdapter, ValidationError -from typing_extensions import NotRequired, TypedDict - - -class BaseDataContentBlock(TypedDict, total=False): - """Base class for data content blocks.""" - - mime_type: NotRequired[str] - """MIME type of the content block (if needed).""" - - -class URLContentBlock(BaseDataContentBlock): - """Content block for data from a URL.""" - - type: Literal["image", "audio", "file"] - """Type of the content block.""" - source_type: Literal["url"] - """Source type (url).""" - url: str - """URL for data.""" - - -class Base64ContentBlock(BaseDataContentBlock): - """Content block for inline data from a base64 string.""" - - type: Literal["image", "audio", "file"] - """Type of the content block.""" - source_type: Literal["base64"] - """Source type (base64).""" - data: str - """Data as a base64 string.""" - - -class PlainTextContentBlock(BaseDataContentBlock): - """Content block for plain text data (e.g., from a document).""" - - type: Literal["file"] - """Type of the content block.""" - source_type: Literal["text"] - """Source type (text).""" - text: str - """Text data.""" - - -class IDContentBlock(TypedDict): - """Content block for data specified by an identifier.""" - - type: Literal["image", "audio", "file"] - """Type of the content block.""" - source_type: Literal["id"] - """Source type (id).""" - id: str - """Identifier for data source.""" - - -DataContentBlock = Union[ - URLContentBlock, - Base64ContentBlock, - PlainTextContentBlock, - IDContentBlock, -] - -_DataContentBlockAdapter: TypeAdapter[DataContentBlock] = TypeAdapter(DataContentBlock) - - -def is_data_content_block( - content_block: dict, -) -> bool: - """Check if the content block is a standard data content block. - - Args: - content_block: The content block to check. - - Returns: - True if the content block is a data content block, False otherwise. - """ - try: - _ = _DataContentBlockAdapter.validate_python(content_block) - except ValidationError: - return False - else: - return True - - -def convert_to_openai_image_block(content_block: dict[str, Any]) -> dict: - """Convert image content block to format expected by OpenAI Chat Completions API.""" - if content_block["source_type"] == "url": - return { - "type": "image_url", - "image_url": { - "url": content_block["url"], - }, - } - if content_block["source_type"] == "base64": - if "mime_type" not in content_block: - error_message = "mime_type key is required for base64 data." - raise ValueError(error_message) - mime_type = content_block["mime_type"] - return { - "type": "image_url", - "image_url": { - "url": f"data:{mime_type};base64,{content_block['data']}", - }, - } - error_message = "Unsupported source type. Only 'url' and 'base64' are supported." - raise ValueError(error_message) - - -def convert_to_openai_data_block(block: dict) -> dict: - """Format standard data content block to format expected by OpenAI.""" - if block["type"] == "image": - formatted_block = convert_to_openai_image_block(block) - - elif block["type"] == "file": - if block["source_type"] == "base64": - file = {"file_data": f"data:{block['mime_type']};base64,{block['data']}"} - if filename := block.get("filename"): - file["filename"] = filename - elif (metadata := block.get("metadata")) and ("filename" in metadata): - file["filename"] = metadata["filename"] - else: - warnings.warn( - "OpenAI may require a filename for file inputs. Specify a filename " - "in the content block: {'type': 'file', 'source_type': 'base64', " - "'mime_type': 'application/pdf', 'data': '...', " - "'filename': 'my-pdf'}", - stacklevel=1, - ) - formatted_block = {"type": "file", "file": file} - elif block["source_type"] == "id": - formatted_block = {"type": "file", "file": {"file_id": block["id"]}} - else: - error_msg = "source_type base64 or id is required for file blocks." - raise ValueError(error_msg) - - elif block["type"] == "audio": - if block["source_type"] == "base64": - audio_format = block["mime_type"].split("/")[-1] - formatted_block = { - "type": "input_audio", - "input_audio": {"data": block["data"], "format": audio_format}, - } - else: - error_msg = "source_type base64 is required for audio blocks." - raise ValueError(error_msg) - else: - error_msg = f"Block of type {block['type']} is not supported." - raise ValueError(error_msg) - - return formatted_block diff --git a/libs/core/langchain_core/messages/function.py b/libs/core/langchain_core/messages/function.py index fc1018775b7e1..612be1dc10e33 100644 --- a/libs/core/langchain_core/messages/function.py +++ b/libs/core/langchain_core/messages/function.py @@ -15,19 +15,20 @@ class FunctionMessage(BaseMessage): """Message for passing the result of executing a tool back to a model. - FunctionMessage are an older version of the ToolMessage schema, and - do not contain the tool_call_id field. + ``FunctionMessage`` are an older version of the ``ToolMessage`` schema, and + do not contain the ``tool_call_id`` field. - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. + """ name: str """The name of the function that was executed.""" type: Literal["function"] = "function" - """The type of the message (used for serialization). Defaults to "function".""" + """The type of the message (used for serialization). Defaults to ``'function'``.""" class FunctionMessageChunk(FunctionMessage, BaseMessageChunk): @@ -38,7 +39,10 @@ class FunctionMessageChunk(FunctionMessage, BaseMessageChunk): # non-chunk variant. type: Literal["FunctionMessageChunk"] = "FunctionMessageChunk" # type: ignore[assignment] """The type of the message (used for serialization). - Defaults to "FunctionMessageChunk".""" + + Defaults to ``FunctionMessageChunk``. + + """ @override def __add__(self, other: Any) -> BaseMessageChunk: # type: ignore[override] diff --git a/libs/core/langchain_core/messages/human.py b/libs/core/langchain_core/messages/human.py index 1be4cbfa9d3d9..02383f91a876f 100644 --- a/libs/core/langchain_core/messages/human.py +++ b/libs/core/langchain_core/messages/human.py @@ -1,6 +1,6 @@ """Human message.""" -from typing import Any, Literal, Union +from typing import Any, Literal, Union, overload from langchain_core.messages.base import BaseMessage, BaseMessageChunk @@ -8,7 +8,7 @@ class HumanMessage(BaseMessage): """Message from a human. - HumanMessages are messages that are passed in from a human to the model. + ``HumanMessage``s are messages that are passed in from a human to the model. Example: @@ -36,15 +36,29 @@ class HumanMessage(BaseMessage): At the moment, this is ignored by most models. Usage is discouraged. Defaults to False. + """ type: Literal["human"] = "human" - """The type of the message (used for serialization). Defaults to "human".""" + """The type of the message (used for serialization). + + Defaults to ``'human'``. + + """ + + @overload + def __init__( + self, + content: Union[str, list[Union[str, dict]]], + **kwargs: Any, + ) -> None: ... def __init__( - self, content: Union[str, list[Union[str, dict]]], **kwargs: Any + self, + content: Union[str, list[Union[str, dict]]], + **kwargs: Any, ) -> None: - """Pass in content as positional arg. + """Initialize HumanMessage. Args: content: The string contents of the message. diff --git a/libs/core/langchain_core/messages/modifier.py b/libs/core/langchain_core/messages/modifier.py index 5f1602a4908d3..94ce8356fb397 100644 --- a/libs/core/langchain_core/messages/modifier.py +++ b/libs/core/langchain_core/messages/modifier.py @@ -24,6 +24,7 @@ def __init__( Raises: ValueError: If the 'content' field is passed in kwargs. + """ if kwargs.pop("content", None): msg = "RemoveMessage does not support 'content' field." diff --git a/libs/core/langchain_core/messages/system.py b/libs/core/langchain_core/messages/system.py index d63bd53a0fee0..1ef63f83996b0 100644 --- a/libs/core/langchain_core/messages/system.py +++ b/libs/core/langchain_core/messages/system.py @@ -1,7 +1,8 @@ """System message.""" -from typing import Any, Literal, Union +from typing import Any, Literal, Optional, Union, cast, overload +from langchain_core.messages import content as types from langchain_core.messages.base import BaseMessage, BaseMessageChunk @@ -32,18 +33,41 @@ class SystemMessage(BaseMessage): """ type: Literal["system"] = "system" - """The type of the message (used for serialization). Defaults to "system".""" + """The type of the message (used for serialization). + + Defaults to ``'system'``. + + """ + @overload def __init__( - self, content: Union[str, list[Union[str, dict]]], **kwargs: Any - ) -> None: - """Pass in content as positional arg. + self, + content: Union[str, list[Union[str, dict]]], + **kwargs: Any, + ) -> None: ... + + @overload + def __init__( + self, + content: Optional[Union[str, list[Union[str, dict]]]] = None, + content_blocks: Optional[list[types.ContentBlock]] = None, + **kwargs: Any, + ) -> None: ... - Args: - content: The string contents of the message. - kwargs: Additional fields to pass to the message. - """ - super().__init__(content=content, **kwargs) + def __init__( + self, + content: Optional[Union[str, list[Union[str, dict]]]] = None, + content_blocks: Optional[list[types.ContentBlock]] = None, + **kwargs: Any, + ) -> None: + """Specify ``content`` as positional arg or ``content_blocks`` for typing.""" + if content_blocks is not None: + super().__init__( + content=cast("Union[str, list[Union[str, dict]]]", content_blocks), + **kwargs, + ) + else: + super().__init__(content=content, **kwargs) class SystemMessageChunk(SystemMessage, BaseMessageChunk): @@ -54,4 +78,7 @@ class SystemMessageChunk(SystemMessage, BaseMessageChunk): # non-chunk variant. type: Literal["SystemMessageChunk"] = "SystemMessageChunk" # type: ignore[assignment] """The type of the message (used for serialization). - Defaults to "SystemMessageChunk".""" + + Defaults to ``'SystemMessageChunk'``. + + """ diff --git a/libs/core/langchain_core/messages/tool.py b/libs/core/langchain_core/messages/tool.py index 10e2d18917a22..37c43d99fa528 100644 --- a/libs/core/langchain_core/messages/tool.py +++ b/libs/core/langchain_core/messages/tool.py @@ -1,7 +1,7 @@ """Messages for tools.""" import json -from typing import Any, Literal, Optional, Union +from typing import Any, Literal, Optional, Union, overload from uuid import UUID from pydantic import Field, model_validator @@ -14,19 +14,20 @@ class ToolOutputMixin: """Mixin for objects that tools can return directly. - If a custom BaseTool is invoked with a ToolCall and the output of custom code is - not an instance of ToolOutputMixin, the output will automatically be coerced to a - string and wrapped in a ToolMessage. + If a custom BaseTool is invoked with a ``ToolCall`` and the output of custom code is + not an instance of ``ToolOutputMixin``, the output will automatically be coerced to + a string and wrapped in a ``ToolMessage``. + """ class ToolMessage(BaseMessage, ToolOutputMixin): """Message for passing the result of executing a tool back to a model. - ToolMessages contain the result of a tool invocation. Typically, the result - is encoded inside the `content` field. + ``ToolMessage``s contain the result of a tool invocation. Typically, the result + is encoded inside the ``content`` field. - Example: A ToolMessage representing a result of 42 from a tool call with id + Example: A ``ToolMessage`` representing a result of ``42`` from a tool call with id .. code-block:: python @@ -35,7 +36,7 @@ class ToolMessage(BaseMessage, ToolOutputMixin): ToolMessage(content='42', tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL') - Example: A ToolMessage where only part of the tool output is sent to the model + Example: A ``ToolMessage`` where only part of the tool output is sent to the model and the full output is passed in to artifact. .. versionadded:: 0.2.17 @@ -56,7 +57,7 @@ class ToolMessage(BaseMessage, ToolOutputMixin): tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL', ) - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. @@ -66,7 +67,11 @@ class ToolMessage(BaseMessage, ToolOutputMixin): """Tool call that this message is responding to.""" type: Literal["tool"] = "tool" - """The type of the message (used for serialization). Defaults to "tool".""" + """The type of the message (used for serialization). + + Defaults to ``'tool'``. + + """ artifact: Any = None """Artifact of the Tool execution which is not meant to be sent to the model. @@ -76,12 +81,14 @@ class ToolMessage(BaseMessage, ToolOutputMixin): output is needed in other parts of the code. .. versionadded:: 0.2.17 + """ status: Literal["success", "error"] = "success" """Status of the tool invocation. .. versionadded:: 0.2.24 + """ additional_kwargs: dict = Field(default_factory=dict, repr=False) @@ -96,6 +103,7 @@ def coerce_args(cls, values: dict) -> dict: Args: values: The model arguments. + """ content = values["content"] if isinstance(content, tuple): @@ -133,10 +141,19 @@ def coerce_args(cls, values: dict) -> dict: values["tool_call_id"] = str(tool_call_id) return values + @overload def __init__( - self, content: Union[str, list[Union[str, dict]]], **kwargs: Any + self, + content: Union[str, list[Union[str, dict]]], + **kwargs: Any, + ) -> None: ... + + def __init__( + self, + content: Union[str, list[Union[str, dict]]], + **kwargs: Any, ) -> None: - """Create a ToolMessage. + """Initialize ToolMessage. Args: content: The string contents of the message. @@ -190,8 +207,8 @@ class ToolCall(TypedDict): "id": "123" } - This represents a request to call the tool named "foo" with arguments {"a": 1} - and an identifier of "123". + This represents a request to call the tool named ``'foo'`` with arguments + ``{"a": 1}`` and an identifier of ``'123'``. """ @@ -204,6 +221,7 @@ class ToolCall(TypedDict): An identifier is needed to associate a tool call request with a tool call result in events when multiple concurrent tool calls are made. + """ type: NotRequired[Literal["tool_call"]] @@ -220,6 +238,7 @@ def tool_call( name: The name of the tool to be called. args: The arguments to the tool call. id: An identifier associated with the tool call. + """ return ToolCall(name=name, args=args, id=id, type="tool_call") @@ -227,9 +246,9 @@ def tool_call( class ToolCallChunk(TypedDict): """A chunk of a tool call (e.g., as part of a stream). - When merging ToolCallChunks (e.g., via AIMessageChunk.__add__), + When merging ``ToolCallChunk``s (e.g., via ``AIMessageChunk.__add__``), all string attributes are concatenated. Chunks are only merged if their - values of `index` are equal and not None. + values of ``index`` are equal and not None. Example: @@ -270,6 +289,7 @@ def tool_call_chunk( args: The arguments to the tool call. id: An identifier associated with the tool call. index: The index of the tool call in a sequence. + """ return ToolCallChunk( name=name, args=args, id=id, index=index, type="tool_call_chunk" @@ -279,8 +299,9 @@ def tool_call_chunk( class InvalidToolCall(TypedDict): """Allowance for errors made by LLM. - Here we add an `error` key to surface errors made during generation + Here we add an ``error`` key to surface errors made during generation (e.g., invalid JSON arguments.) + """ name: Optional[str] @@ -308,6 +329,7 @@ def invalid_tool_call( args: The arguments to the tool call. id: An identifier associated with the tool call. error: An error message associated with the tool call. + """ return InvalidToolCall( name=name, args=args, id=id, error=error, type="invalid_tool_call" diff --git a/libs/core/langchain_core/messages/utils.py b/libs/core/langchain_core/messages/utils.py index e84dc6c0191ed..292d7f110a4c0 100644 --- a/libs/core/langchain_core/messages/utils.py +++ b/libs/core/langchain_core/messages/utils.py @@ -5,6 +5,7 @@ * Convert messages to strings (serialization) * Convert messages from dicts to Message objects (deserialization) * Filter messages from a list of messages based on name, type or id etc. + """ from __future__ import annotations @@ -31,10 +32,13 @@ from pydantic import Discriminator, Field, Tag from langchain_core.exceptions import ErrorCode, create_message -from langchain_core.messages import convert_to_openai_data_block, is_data_content_block from langchain_core.messages.ai import AIMessage, AIMessageChunk from langchain_core.messages.base import BaseMessage, BaseMessageChunk from langchain_core.messages.chat import ChatMessage, ChatMessageChunk +from langchain_core.messages.content import ( + convert_to_openai_data_block, + is_data_content_block, +) from langchain_core.messages.function import FunctionMessage, FunctionMessageChunk from langchain_core.messages.human import HumanMessage, HumanMessageChunk from langchain_core.messages.modifier import RemoveMessage @@ -86,13 +90,13 @@ def _get_type(v: Any) -> str: def get_buffer_string( messages: Sequence[BaseMessage], human_prefix: str = "Human", ai_prefix: str = "AI" ) -> str: - r"""Convert a sequence of Messages to strings and concatenate them into one string. + r"""Convert a sequence of messages to strings and concatenate them into one string. Args: messages: Messages to be converted to strings. - human_prefix: The prefix to prepend to contents of HumanMessages. + human_prefix: The prefix to prepend to contents of ``HumanMessage``s. Default is "Human". - ai_prefix: THe prefix to prepend to contents of AIMessages. Default is "AI". + ai_prefix: The prefix to prepend to contents of AIMessages. Default is ``'AI'``. Returns: A single string concatenation of all input messages. @@ -171,19 +175,20 @@ def _message_from_dict(message: dict) -> BaseMessage: def messages_from_dict(messages: Sequence[dict]) -> list[BaseMessage]: - """Convert a sequence of messages from dicts to Message objects. + """Convert a sequence of messages from dicts to ``Message`` objects. Args: messages: Sequence of messages (as dicts) to convert. Returns: list of messages (BaseMessages). + """ return [_message_from_dict(m) for m in messages] def message_chunk_to_message(chunk: BaseMessageChunk) -> BaseMessage: - """Convert a message chunk to a message. + """Convert a message chunk to a ``Message``. Args: chunk: Message chunk to convert. @@ -216,10 +221,10 @@ def _create_message_from_message_type( id: Optional[str] = None, **additional_kwargs: Any, ) -> BaseMessage: - """Create a message from a message type and content string. + """Create a message from a ``Message`` type and content string. Args: - message_type: (str) the type of the message (e.g., "human", "ai", etc.). + message_type: (str) the type of the message (e.g., ``'human'``, ``'ai'``, etc.). content: (str) the content string. name: (str) the name of the message. Default is None. tool_call_id: (str) the tool call id. Default is None. @@ -231,8 +236,9 @@ def _create_message_from_message_type( a message of the appropriate type. Raises: - ValueError: if the message type is not one of "human", "user", "ai", - "assistant", "function", "tool", "system", or "developer". + ValueError: if the message type is not one of ``'human'``, ``'user'``, ``'ai'``, + ``'assistant'``, ``'function'``, ``'tool'``, ``'system'``, or + ``'developer'``. """ kwargs: dict[str, Any] = {} if name is not None: @@ -295,15 +301,15 @@ def _create_message_from_message_type( def _convert_to_message(message: MessageLikeRepresentation) -> BaseMessage: - """Instantiate a message from a variety of message formats. + """Instantiate a ``Message`` from a variety of message formats. The message format can be one of the following: - - BaseMessagePromptTemplate - - BaseMessage - - 2-tuple of (role string, template); e.g., ("human", "{user_input}") + - ``BaseMessagePromptTemplate`` + - ``BaseMessage`` + - 2-tuple of (role string, template); e.g., (``'human'``, ``'{user_input}'``) - dict: a message dict with role and content keys - - string: shorthand for ("human", template); e.g., "{user_input}" + - string: shorthand for (``'human'``, template); e.g., ``'{user_input}'`` Args: message: a representation of a message in one of the supported formats. @@ -314,6 +320,7 @@ def _convert_to_message(message: MessageLikeRepresentation) -> BaseMessage: Raises: NotImplementedError: if the message type is not supported. ValueError: if the message dict does not contain the required keys. + """ if isinstance(message, BaseMessage): message_ = message @@ -359,6 +366,7 @@ def convert_to_messages( Returns: list of messages (BaseMessages). + """ # Import here to avoid circular imports from langchain_core.prompt_values import PromptValue @@ -408,31 +416,31 @@ def filter_messages( exclude_ids: Optional[Sequence[str]] = None, exclude_tool_calls: Optional[Sequence[str] | bool] = None, ) -> list[BaseMessage]: - """Filter messages based on name, type or id. + """Filter messages based on ``name``, ``type`` or ``id``. Args: messages: Sequence Message-like objects to filter. include_names: Message names to include. Default is None. exclude_names: Messages names to exclude. Default is None. include_types: Message types to include. Can be specified as string names (e.g. - "system", "human", "ai", ...) or as BaseMessage classes (e.g. - SystemMessage, HumanMessage, AIMessage, ...). Default is None. + ``'system'``, ``'human'``, ``'ai'``, ...) or as ``BaseMessage`` classes (e.g. + ``SystemMessage``, ``HumanMessage``, ``AIMessage``, ...). Default is None. exclude_types: Message types to exclude. Can be specified as string names (e.g. - "system", "human", "ai", ...) or as BaseMessage classes (e.g. - SystemMessage, HumanMessage, AIMessage, ...). Default is None. + ``'system'``, ``'human'``, ``'ai'``, ...) or as ``BaseMessage`` classes (e.g. + ``SystemMessage``, ``HumanMessage``, ``AIMessage``, ...). Default is None. include_ids: Message IDs to include. Default is None. exclude_ids: Message IDs to exclude. Default is None. exclude_tool_calls: Tool call IDs to exclude. Default is None. Can be one of the following: - - `True`: all AIMessages with tool calls and all ToolMessages will be excluded. + - ``True``: all ``AIMessage``s with tool calls and all ``ToolMessage``s will be excluded. - a sequence of tool call IDs to exclude: - - ToolMessages with the corresponding tool call ID will be excluded. - - The `tool_calls` in the AIMessage will be updated to exclude matching tool calls. - If all tool_calls are filtered from an AIMessage, the whole message is excluded. + - ``ToolMessage``s with the corresponding tool call ID will be excluded. + - The ``tool_calls`` in the AIMessage will be updated to exclude matching tool calls. + If all ``tool_calls`` are filtered from an AIMessage, the whole message is excluded. Returns: - A list of Messages that meets at least one of the incl_* conditions and none - of the excl_* conditions. If not incl_* conditions are specified then + A list of Messages that meets at least one of the ``incl_*`` conditions and none + of the ``excl_*`` conditions. If not ``incl_*`` conditions are specified then anything that is not explicitly excluded will be included. Raises: @@ -533,13 +541,14 @@ def merge_message_runs( ) -> list[BaseMessage]: r"""Merge consecutive Messages of the same type. - **NOTE**: ToolMessages are not merged, as each has a distinct tool call id that - can't be merged. + .. note:: + ToolMessages are not merged, as each has a distinct tool call id that can't be + merged. Args: messages: Sequence Message-like objects to merge. chunk_separator: Specify the string to be inserted between message chunks. - Default is "\n". + Default is ``'\n'``. Returns: list of BaseMessages with consecutive runs of message types merged into single @@ -648,22 +657,22 @@ def trim_messages( ) -> list[BaseMessage]: r"""Trim messages to be below a token count. - trim_messages can be used to reduce the size of a chat history to a specified token - count or specified message count. + ``trim_messages`` can be used to reduce the size of a chat history to a specified + token count or specified message count. In either case, if passing the trimmed chat history back into a chat model directly, the resulting chat history should usually satisfy the following properties: 1. The resulting chat history should be valid. Most chat models expect that chat - history starts with either (1) a ``HumanMessage`` or (2) a ``SystemMessage`` followed - by a ``HumanMessage``. To achieve this, set ``start_on="human"``. + history starts with either (1) a ``HumanMessage`` or (2) a ``SystemMessage`` + followed by a ``HumanMessage``. To achieve this, set ``start_on='human'``. In addition, generally a ``ToolMessage`` can only appear after an ``AIMessage`` that involved a tool call. Please see the following link for more information about messages: https://python.langchain.com/docs/concepts/#messages 2. It includes recent messages and drops old messages in the chat history. - To achieve this set the ``strategy="last"``. + To achieve this set the ``strategy='last'``. 3. Usually, the new chat history should include the ``SystemMessage`` if it was present in the original chat history since the ``SystemMessage`` includes special instructions to the chat model. The ``SystemMessage`` is almost always @@ -677,65 +686,66 @@ def trim_messages( Args: messages: Sequence of Message-like objects to trim. max_tokens: Max token count of trimmed messages. - token_counter: Function or llm for counting tokens in a BaseMessage or a list of - BaseMessage. If a BaseLanguageModel is passed in then - BaseLanguageModel.get_num_tokens_from_messages() will be used. - Set to `len` to count the number of **messages** in the chat history. + token_counter: Function or llm for counting tokens in a ``BaseMessage`` or a + list of ``BaseMessage``. If a ``BaseLanguageModel`` is passed in then + ``BaseLanguageModel.get_num_tokens_from_messages()`` will be used. + Set to ``len`` to count the number of **messages** in the chat history. .. note:: - Use `count_tokens_approximately` to get fast, approximate token counts. - This is recommended for using `trim_messages` on the hot path, where + Use ``count_tokens_approximately`` to get fast, approximate token counts. + This is recommended for using ``trim_messages`` on the hot path, where exact token counting is not necessary. strategy: Strategy for trimming. - - "first": Keep the first <= n_count tokens of the messages. - - "last": Keep the last <= n_count tokens of the messages. - Default is "last". + - ``'first'``: Keep the first ``<= n_count`` tokens of the messages. + - ``'last'``: Keep the last ``<= n_count`` tokens of the messages. + Default is ``'last'``. allow_partial: Whether to split a message if only part of the message can be - included. If ``strategy="last"`` then the last partial contents of a message - are included. If ``strategy="first"`` then the first partial contents of a + included. If ``strategy='last'`` then the last partial contents of a message + are included. If ``strategy='first'`` then the first partial contents of a message are included. Default is False. end_on: The message type to end on. If specified then every message after the - last occurrence of this type is ignored. If ``strategy=="last"`` then this + last occurrence of this type is ignored. If ``strategy='last'`` then this is done before we attempt to get the last ``max_tokens``. If - ``strategy=="first"`` then this is done after we get the first - ``max_tokens``. Can be specified as string names (e.g. "system", "human", - "ai", ...) or as BaseMessage classes (e.g. SystemMessage, HumanMessage, - AIMessage, ...). Can be a single type or a list of types. + ``strategy='first'`` then this is done after we get the first + ``max_tokens``. Can be specified as string names (e.g. ``'system'``, + ``'human'``, ``'ai'``, ...) or as ``BaseMessage`` classes (e.g. + ``SystemMessage``, ``HumanMessage``, ``AIMessage``, ...). Can be a single + type or a list of types. Default is None. start_on: The message type to start on. Should only be specified if - ``strategy="last"``. If specified then every message before + ``strategy='last'``. If specified then every message before the first occurrence of this type is ignored. This is done after we trim the initial messages to the last ``max_tokens``. Does not - apply to a SystemMessage at index 0 if ``include_system=True``. Can be - specified as string names (e.g. "system", "human", "ai", ...) or as - BaseMessage classes (e.g. SystemMessage, HumanMessage, AIMessage, ...). Can - be a single type or a list of types. + apply to a ``SystemMessage`` at index 0 if ``include_system=True``. Can be + specified as string names (e.g. ``'system'``, ``'human'``, ``'ai'``, ...) or + as ``BaseMessage`` classes (e.g. ``SystemMessage``, ``HumanMessage``, + ``AIMessage``, ...). Can be a single type or a list of types. Default is None. include_system: Whether to keep the SystemMessage if there is one at index 0. Should only be specified if ``strategy="last"``. Default is False. text_splitter: Function or ``langchain_text_splitters.TextSplitter`` for splitting the string contents of a message. Only used if - ``allow_partial=True``. If ``strategy="last"`` then the last split tokens - from a partial message will be included. if ``strategy=="first"`` then the + ``allow_partial=True``. If ``strategy='last'`` then the last split tokens + from a partial message will be included. if ``strategy='first'`` then the first split tokens from a partial message will be included. Token splitter assumes that separators are kept, so that split contents can be directly concatenated to recreate the original text. Defaults to splitting on newlines. Returns: - list of trimmed BaseMessages. + list of trimmed ``BaseMessage``. Raises: ValueError: if two incompatible arguments are specified or an unrecognized ``strategy`` is specified. Example: - Trim chat history based on token count, keeping the SystemMessage if - present, and ensuring that the chat history starts with a HumanMessage ( - or a SystemMessage followed by a HumanMessage). + Trim chat history based on token count, keeping the ``SystemMessage`` if + present, and ensuring that the chat history starts with a ``HumanMessage`` ( + or a ``SystemMessage`` followed by a ``HumanMessage``). .. code-block:: python @@ -784,9 +794,9 @@ def trim_messages( HumanMessage(content='what do you call a speechless parrot'), ] - Trim chat history based on the message count, keeping the SystemMessage if - present, and ensuring that the chat history starts with a HumanMessage ( - or a SystemMessage followed by a HumanMessage). + Trim chat history based on the message count, keeping the ``SystemMessage`` if + present, and ensuring that the chat history starts with a ``HumanMessage`` ( + or a ``SystemMessage`` followed by a ``HumanMessage``). trim_messages( messages, @@ -952,16 +962,16 @@ def convert_to_openai_messages( in OpenAI, Anthropic, Bedrock Converse, or VertexAI formats. text_format: How to format string or text block contents: - - "string": + - ``'string'``: If a message has a string content, this is left as a string. If - a message has content blocks that are all of type 'text', these are - joined with a newline to make a single string. If a message has - content blocks and at least one isn't of type 'text', then + a message has content blocks that are all of type ``'text'``, these + are joined with a newline to make a single string. If a message has + content blocks and at least one isn't of type ``'text'``, then all blocks are left as dicts. - "block": If a message has a string content, this is turned into a list - with a single content block of type 'text'. If a message has content - blocks these are left as is. + with a single content block of type ``'text'``. If a message has + content blocks these are left as is. Returns: The return type depends on the input type: diff --git a/libs/core/langchain_core/outputs/chat_generation.py b/libs/core/langchain_core/outputs/chat_generation.py index d42f2038d346f..156544db05c57 100644 --- a/libs/core/langchain_core/outputs/chat_generation.py +++ b/libs/core/langchain_core/outputs/chat_generation.py @@ -15,14 +15,14 @@ class ChatGeneration(Generation): """A single chat generation output. - A subclass of Generation that represents the response from a chat model + A subclass of ``Generation`` that represents the response from a chat model that generates chat messages. - The `message` attribute is a structured representation of the chat message. - Most of the time, the message will be of type `AIMessage`. + The ``message`` attribute is a structured representation of the chat message. + Most of the time, the message will be of type ``AIMessage``. Users working with chat models will usually access information via either - `AIMessage` (returned from runnable interfaces) or `LLMResult` (available + ``AIMessage`` (returned from runnable interfaces) or ``LLMResult`` (available via callbacks). """ @@ -31,6 +31,7 @@ class ChatGeneration(Generation): .. warning:: SHOULD NOT BE SET DIRECTLY! + """ message: BaseMessage """The message output by the chat model.""" @@ -50,6 +51,7 @@ def set_text(self) -> Self: Raises: ValueError: If the message is not a string or a list. + """ text = "" if isinstance(self.message.content, str): @@ -69,9 +71,9 @@ def set_text(self) -> Self: class ChatGenerationChunk(ChatGeneration): - """ChatGeneration chunk. + """``ChatGeneration`` chunk. - ChatGeneration chunks can be concatenated with other ChatGeneration chunks. + ``ChatGeneration`` chunks can be concatenated with other ``ChatGeneration`` chunks. """ message: BaseMessageChunk @@ -83,11 +85,11 @@ class ChatGenerationChunk(ChatGeneration): def __add__( self, other: Union[ChatGenerationChunk, list[ChatGenerationChunk]] ) -> ChatGenerationChunk: - """Concatenate two ChatGenerationChunks. + """Concatenate two ``ChatGenerationChunks``. Args: - other: The other ChatGenerationChunk or list of ChatGenerationChunks to - concatenate. + other: The other ``ChatGenerationChunk`` or list of ``ChatGenerationChunk``s + to concatenate. """ if isinstance(other, ChatGenerationChunk): generation_info = merge_dicts( @@ -116,7 +118,7 @@ def __add__( def merge_chat_generation_chunks( chunks: list[ChatGenerationChunk], ) -> Union[ChatGenerationChunk, None]: - """Merge a list of ChatGenerationChunks into a single ChatGenerationChunk.""" + """Merge list of ``ChatGenerationChunk``s into a single ``ChatGenerationChunk``.""" if not chunks: return None diff --git a/libs/core/langchain_core/prompt_values.py b/libs/core/langchain_core/prompt_values.py index 5f5dd7eb6b21b..01827efcd8f70 100644 --- a/libs/core/langchain_core/prompt_values.py +++ b/libs/core/langchain_core/prompt_values.py @@ -107,8 +107,12 @@ class ImageURL(TypedDict, total=False): """Image URL.""" detail: Literal["auto", "low", "high"] - """Specifies the detail level of the image. Defaults to "auto". - Can be "auto", "low", or "high".""" + """Specifies the detail level of the image. Defaults to ``'auto'``. + Can be ``'auto'``, ``'low'``, or ``'high'``. + + This follows OpenAI's Chat Completion API's image URL format. + + """ url: str """Either a URL of the image or the base64 encoded image data.""" diff --git a/libs/core/langchain_core/runnables/base.py b/libs/core/langchain_core/runnables/base.py index 7b872783909f1..935554848c4ec 100644 --- a/libs/core/langchain_core/runnables/base.py +++ b/libs/core/langchain_core/runnables/base.py @@ -2399,7 +2399,7 @@ def as_tool( description: The description of the tool. Defaults to None. arg_types: A dictionary of argument names to types. Defaults to None. message_version: Version of ``ToolMessage`` to return given - :class:`~langchain_core.messages.content_blocks.ToolCall` input. + :class:`~langchain_core.messages.content.ToolCall` input. Returns: A ``BaseTool`` instance. diff --git a/libs/core/langchain_core/utils/_merge.py b/libs/core/langchain_core/utils/_merge.py index 63d49de953c27..7b8465e8d0256 100644 --- a/libs/core/langchain_core/utils/_merge.py +++ b/libs/core/langchain_core/utils/_merge.py @@ -57,6 +57,11 @@ def merge_dicts(left: dict[str, Any], *others: dict[str, Any]) -> dict[str, Any] # "should either occur once or have the same value across " # "all dicts." # ) + if (right_k == "index" and merged[right_k].startswith("lc_")) or ( + right_k in ("id", "output_version", "model_provider") + and merged[right_k] == right_v + ): + continue merged[right_k] += right_v elif isinstance(merged[right_k], dict): merged[right_k] = merge_dicts(merged[right_k], right_v) @@ -93,7 +98,16 @@ def merge_lists(left: Optional[list], *others: Optional[list]) -> Optional[list] merged = other.copy() else: for e in other: - if isinstance(e, dict) and "index" in e and isinstance(e["index"], int): + if ( + isinstance(e, dict) + and "index" in e + and ( + isinstance(e["index"], int) + or ( + isinstance(e["index"], str) and e["index"].startswith("lc_") + ) + ) + ): to_merge = [ i for i, e_left in enumerate(merged) @@ -102,11 +116,35 @@ def merge_lists(left: Optional[list], *others: Optional[list]) -> Optional[list] if to_merge: # TODO: Remove this once merge_dict is updated with special # handling for 'type'. - new_e = ( - {k: v for k, v in e.items() if k != "type"} - if "type" in e - else e - ) + if (left_type := merged[to_merge[0]].get("type")) and ( + e.get("type") == "non_standard" and "value" in e + ): + if left_type != "non_standard": + # standard + non_standard + new_e: dict[str, Any] = { + "extras": { + k: v + for k, v in e["value"].items() + if k != "type" + } + } + else: + # non_standard + non_standard + new_e = { + "value": { + k: v + for k, v in e["value"].items() + if k != "type" + } + } + if "index" in e: + new_e["index"] = e["index"] + else: + new_e = ( + {k: v for k, v in e.items() if k != "type"} + if "type" in e + else e + ) merged[to_merge[0]] = merge_dicts(merged[to_merge[0]], new_e) else: merged.append(e) diff --git a/libs/core/langchain_core/utils/utils.py b/libs/core/langchain_core/utils/utils.py index a7467ec51e998..28becc822e2c3 100644 --- a/libs/core/langchain_core/utils/utils.py +++ b/libs/core/langchain_core/utils/utils.py @@ -9,6 +9,7 @@ from collections.abc import Iterator, Sequence from importlib.metadata import version from typing import Any, Callable, Optional, Union, overload +from uuid import uuid4 from packaging.version import parse from pydantic import SecretStr @@ -466,3 +467,31 @@ def get_secret_from_env() -> Optional[SecretStr]: raise ValueError(msg) return get_secret_from_env + + +LC_AUTO_PREFIX = "lc_" +"""LangChain auto-generated ID prefix for messages and content blocks.""" + +LC_ID_PREFIX = "lc_run-" +"""Internal tracing/callback system identifier. + +Used for: +- Tracing. Every LangChain operation (LLM call, chain execution, tool use, etc.) + gets a unique run_id (UUID) +- Enables tracking parent-child relationships between operations +""" + + +def ensure_id(id_val: Optional[str]) -> str: + """Ensure the ID is a valid string, generating a new UUID if not provided. + + Auto-generated UUIDs are prefixed by ``'lc_'`` to indicate they are + LangChain-generated IDs. + + Args: + id_val: Optional string ID value to validate. + + Returns: + A string ID, either the validated provided value or a newly generated UUID4. + """ + return id_val or str(f"{LC_AUTO_PREFIX}{uuid4()}") diff --git a/libs/core/tests/unit_tests/language_models/chat_models/test_base.py b/libs/core/tests/unit_tests/language_models/chat_models/test_base.py index baf8025381e7c..22d8bc7907f5e 100644 --- a/libs/core/tests/unit_tests/language_models/chat_models/test_base.py +++ b/libs/core/tests/unit_tests/language_models/chat_models/test_base.py @@ -14,11 +14,15 @@ ParrotFakeChatModel, ) from langchain_core.language_models._utils import _normalize_messages -from langchain_core.language_models.fake_chat_models import FakeListChatModelError +from langchain_core.language_models.fake_chat_models import ( + FakeListChatModelError, + GenericFakeChatModel, +) from langchain_core.messages import ( AIMessage, AIMessageChunk, BaseMessage, + BaseMessageChunk, HumanMessage, SystemMessage, ) @@ -40,6 +44,37 @@ from langchain_core.outputs.llm_result import LLMResult +def _content_blocks_equal_ignore_id( + actual: Union[str, list[Any]], expected: Union[str, list[Any]] +) -> bool: + """Compare content blocks, ignoring auto-generated `id` fields. + + Args: + actual: Actual content from response (string or list of content blocks). + expected: Expected content to compare against (string or list of blocks). + + Returns: + True if content matches (excluding `id` fields), False otherwise. + + """ + if isinstance(actual, str) or isinstance(expected, str): + return actual == expected + + if len(actual) != len(expected): + return False + for actual_block, expected_block in zip(actual, expected): + actual_without_id = ( + {k: v for k, v in actual_block.items() if k != "id"} + if isinstance(actual_block, dict) and "id" in actual_block + else actual_block + ) + + if actual_without_id != expected_block: + return False + + return True + + @pytest.fixture def messages() -> list: return [ @@ -141,7 +176,7 @@ def eval_response(callback: BaseFakeCallbackHandler, i: int) -> None: async def test_astream_fallback_to_ainvoke() -> None: - """Test astream uses appropriate implementation.""" + """Test `astream()` uses appropriate implementation.""" class ModelWithGenerate(BaseChatModel): @override @@ -427,11 +462,12 @@ def on_chat_model_start(self, *args: Any, **kwargs: Any) -> Run: def test_trace_images_in_openai_format() -> None: - """Test that images are traced in OpenAI format.""" + """Test that images are traced in OpenAI Chat Completions format.""" llm = ParrotFakeChatModel() messages = [ { "role": "user", + # v0 format "content": [ { "type": "image", @@ -442,7 +478,7 @@ def test_trace_images_in_openai_format() -> None: } ] tracer = FakeChatModelStartTracer() - response = llm.invoke(messages, config={"callbacks": [tracer]}) + llm.invoke(messages, config={"callbacks": [tracer]}) assert tracer.messages == [ [ [ @@ -457,19 +493,51 @@ def test_trace_images_in_openai_format() -> None: ] ] ] - # Test no mutation - assert response.content == [ - { + + +def test_content_block_transformation_v0_to_v1_image() -> None: + """Test that v0 format image content blocks are transformed to v1 format.""" + # Create a message with v0 format image content + image_message = AIMessage( + content=[ + { + "type": "image", + "source_type": "url", + "url": "https://example.com/image.png", + } + ] + ) + + llm = GenericFakeChatModel(messages=iter([image_message]), output_version="v1") + response = llm.invoke("test") + + # With v1 output_version, .content should be transformed + # Check structure, ignoring auto-generated IDs + assert len(response.content) == 1 + content_block = response.content[0] + if isinstance(content_block, dict) and "id" in content_block: + # Remove auto-generated id for comparison + content_without_id = {k: v for k, v in content_block.items() if k != "id"} + expected_content = { + "type": "image", + "url": "https://example.com/image.png", + } + assert content_without_id == expected_content + else: + assert content_block == { "type": "image", - "source_type": "url", "url": "https://example.com/image.png", } - ] -def test_trace_content_blocks_with_no_type_key() -> None: - """Test that we add a ``type`` key to certain content blocks that don't have one.""" - llm = ParrotFakeChatModel() +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_trace_content_blocks_with_no_type_key(output_version: str) -> None: + """Test behavior of content blocks that don't have a `type` key. + + Only for blocks with one key, in which case, the name of the key is used as `type`. + + """ + llm = ParrotFakeChatModel(output_version=output_version) messages = [ { "role": "user", @@ -504,153 +572,368 @@ def test_trace_content_blocks_with_no_type_key() -> None: ] ] ] - # Test no mutation - assert response.content == [ + + if output_version == "v0": + assert response.content == [ + { + "type": "text", + "text": "Hello", + }, + { + "cachePoint": {"type": "default"}, + }, + ] + else: + assert response.content == [ + { + "type": "text", + "text": "Hello", + }, + { + "type": "non_standard", + "value": { + "cachePoint": {"type": "default"}, + }, + }, + ] + + assert response.content_blocks == [ { "type": "text", "text": "Hello", }, { - "cachePoint": {"type": "default"}, + "type": "non_standard", + "value": { + "cachePoint": {"type": "default"}, + }, }, ] def test_extend_support_to_openai_multimodal_formats() -> None: - """Test that chat models normalize OpenAI file and audio inputs.""" - llm = ParrotFakeChatModel() - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Hello"}, - { - "type": "image_url", - "image_url": {"url": "https://example.com/image.png"}, + """Test normalizing OpenAI audio, image, and file inputs to v1.""" + # Audio and file only (chat model default) + messages = HumanMessage( + content=[ + {"type": "text", "text": "Hello"}, + { # audio-base64 + "type": "input_audio", + "input_audio": { + "format": "wav", + "data": "", }, - { - "type": "image_url", - "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZJRg..."}, + }, + { # file-base64 + "type": "file", + "file": { + "filename": "draconomicon.pdf", + "file_data": "", + }, + }, + { # file-id + "type": "file", + "file": {"file_id": ""}, + }, + ] + ) + + expected_content_messages = HumanMessage( + content=[ + {"type": "text", "text": "Hello"}, # TextContentBlock + { # AudioContentBlock + "type": "audio", + "base64": "", + "mime_type": "audio/wav", + }, + { # FileContentBlock + "type": "file", + "base64": "", + "mime_type": "application/pdf", + "extras": {"filename": "draconomicon.pdf"}, + }, + { # ... + "type": "file", + "file_id": "", + }, + ] + ) + + normalized_content = _normalize_messages([messages]) + + # Check structure, ignoring auto-generated IDs + assert len(normalized_content) == 1 + normalized_message = normalized_content[0] + assert len(normalized_message.content) == len(expected_content_messages.content) + + assert _content_blocks_equal_ignore_id( + normalized_message.content, expected_content_messages.content + ) + + messages = HumanMessage( + content=[ + {"type": "text", "text": "Hello"}, + { # image-url + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + }, + { # image-base64 + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZJRg..."}, + }, + { # audio-base64 + "type": "input_audio", + "input_audio": { + "format": "wav", + "data": "data:audio/wav;base64,", + }, + }, + { # file-base64 + "type": "file", + "file": { + "filename": "draconomicon.pdf", + "file_data": "data:application/pdf;base64,", }, + }, + { # file-id + "type": "file", + "file": {"file_id": ""}, + }, + ] + ) + + expected_content_messages = HumanMessage( + content=[ + {"type": "text", "text": "Hello"}, # TextContentBlock + { # Chat Completions Image becomes ImageContentBlock after invoke + "type": "image", + "url": "https://example.com/image.png", + }, + { # ... + "type": "image", + "base64": "data:image/jpeg;base64,/9j/4AAQSkZJRg...", + "mime_type": "image/jpeg", + }, + { # AudioContentBlock + "type": "audio", + "base64": "data:audio/wav;base64,", + "mime_type": "audio/wav", + }, + { # FileContentBlock + "type": "file", + "base64": "data:application/pdf;base64,", + "mime_type": "application/pdf", + "extras": {"filename": "draconomicon.pdf"}, + }, + { # ... + "type": "file", + "file_id": "", + }, + ] + ) + + +def test_normalize_messages_edge_cases() -> None: + # Test behavior of malformed/unrecognized content blocks + + messages = [ + HumanMessage( + content=[ { - "type": "file", - "file": { - "filename": "draconomicon.pdf", - "file_data": "data:application/pdf;base64,", - }, + "type": "input_image", # Responses API type; not handled + "image_url": "uri", }, { - "type": "file", - "file": { - "file_data": "data:application/pdf;base64,", - }, + # Standard OpenAI Chat Completions type but malformed structure + "type": "input_audio", + "input_audio": "uri", # Should be nested in `audio` }, { "type": "file", - "file": {"file_id": ""}, + "file": "uri", # `file` should be a dict for Chat Completions }, { - "type": "input_audio", - "input_audio": {"data": "", "format": "wav"}, + "type": "input_file", # Responses API type; not handled + "file_data": "uri", + "filename": "file-name", }, - ], - }, - ] - expected_content = [ - {"type": "text", "text": "Hello"}, - { - "type": "image_url", - "image_url": {"url": "https://example.com/image.png"}, - }, - { - "type": "image_url", - "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZJRg..."}, - }, - { - "type": "file", - "source_type": "base64", - "data": "", - "mime_type": "application/pdf", - "filename": "draconomicon.pdf", - }, - { - "type": "file", - "source_type": "base64", - "data": "", - "mime_type": "application/pdf", - }, - { - "type": "file", - "file": {"file_id": ""}, - }, - { - "type": "audio", - "source_type": "base64", - "data": "", - "mime_type": "audio/wav", - }, + ] + ) ] - response = llm.invoke(messages) - assert response.content == expected_content - # Test no mutation - assert messages[0]["content"] == [ - {"type": "text", "text": "Hello"}, - { - "type": "image_url", - "image_url": {"url": "https://example.com/image.png"}, - }, - { - "type": "image_url", - "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZJRg..."}, - }, - { - "type": "file", - "file": { - "filename": "draconomicon.pdf", - "file_data": "data:application/pdf;base64,", - }, - }, - { - "type": "file", - "file": { - "file_data": "data:application/pdf;base64,", - }, - }, - { - "type": "file", - "file": {"file_id": ""}, - }, - { - "type": "input_audio", - "input_audio": {"data": "", "format": "wav"}, - }, - ] + assert messages == _normalize_messages(messages) -def test_normalize_messages_edge_cases() -> None: - # Test some blocks that should pass through - messages = [ +def test_normalize_messages_v1_content_blocks_unchanged() -> None: + """Test passing v1 content blocks to `_normalize_messages()` leaves unchanged.""" + input_messages = [ HumanMessage( content=[ { - "type": "file", - "file": "uri", + "type": "text", + "text": "Hello world", }, { - "type": "input_file", - "file_data": "uri", - "filename": "file-name", + "type": "image", + "url": "https://example.com/image.png", + "mime_type": "image/png", }, { - "type": "input_audio", - "input_audio": "uri", + "type": "audio", + "base64": "base64encodedaudiodata", + "mime_type": "audio/wav", }, { - "type": "input_image", - "image_url": "uri", + "type": "file", + "id": "file_123", + }, + { + "type": "reasoning", + "reasoning": "Let me think about this...", }, ] ) ] - assert messages == _normalize_messages(messages) + + result = _normalize_messages(input_messages) + + # Verify the result is identical to the input (message should not be copied) + assert len(result) == 1 + assert result[0] is input_messages[0] + assert result[0].content == input_messages[0].content + + +def test_output_version_invoke(monkeypatch: Any) -> None: + messages = [AIMessage("hello")] + + llm = GenericFakeChatModel(messages=iter(messages), output_version="v1") + response = llm.invoke("hello") + assert response.content == [{"type": "text", "text": "hello"}] + assert response.response_metadata["output_version"] == "v1" + + llm = GenericFakeChatModel(messages=iter(messages)) + response = llm.invoke("hello") + assert response.content == "hello" + + monkeypatch.setenv("LC_OUTPUT_VERSION", "v1") + llm = GenericFakeChatModel(messages=iter(messages)) + response = llm.invoke("hello") + assert response.content == [{"type": "text", "text": "hello"}] + assert response.response_metadata["output_version"] == "v1" + + +# -- v1 output version tests -- + + +async def test_output_version_ainvoke(monkeypatch: Any) -> None: + messages = [AIMessage("hello")] + + # v0 + llm = GenericFakeChatModel(messages=iter(messages)) + response = await llm.ainvoke("hello") + assert response.content == "hello" + + # v1 + llm = GenericFakeChatModel(messages=iter(messages), output_version="v1") + response = await llm.ainvoke("hello") + assert response.content == [{"type": "text", "text": "hello"}] + assert response.response_metadata["output_version"] == "v1" + + # v1 from env var + monkeypatch.setenv("LC_OUTPUT_VERSION", "v1") + llm = GenericFakeChatModel(messages=iter(messages)) + response = await llm.ainvoke("hello") + assert response.content == [{"type": "text", "text": "hello"}] + assert response.response_metadata["output_version"] == "v1" + + +def test_output_version_stream(monkeypatch: Any) -> None: + messages = [AIMessage("foo bar")] + + # v0 + llm = GenericFakeChatModel(messages=iter(messages)) + full = None + for chunk in llm.stream("hello"): + assert isinstance(chunk, AIMessageChunk) + assert isinstance(chunk.content, str) + assert chunk.content + full = chunk if full is None else full + chunk + assert isinstance(full, AIMessageChunk) + assert full.content == "foo bar" + + # v1 + llm = GenericFakeChatModel(messages=iter(messages), output_version="v1") + full_v1: Optional[BaseMessageChunk] = None + for chunk in llm.stream("hello"): + assert isinstance(chunk, AIMessageChunk) + assert isinstance(chunk.content, list) + assert len(chunk.content) == 1 + block = chunk.content[0] + assert isinstance(block, dict) + assert block["type"] == "text" + assert block["text"] + full_v1 = chunk if full_v1 is None else full_v1 + chunk + assert isinstance(full_v1, AIMessageChunk) + assert full_v1.response_metadata["output_version"] == "v1" + + # v1 from env var + monkeypatch.setenv("LC_OUTPUT_VERSION", "v1") + llm = GenericFakeChatModel(messages=iter(messages)) + full_env = None + for chunk in llm.stream("hello"): + assert isinstance(chunk, AIMessageChunk) + assert isinstance(chunk.content, list) + assert len(chunk.content) == 1 + block = chunk.content[0] + assert isinstance(block, dict) + assert block["type"] == "text" + assert block["text"] + full_env = chunk if full_env is None else full_env + chunk + assert isinstance(full_env, AIMessageChunk) + assert full_env.response_metadata["output_version"] == "v1" + + +async def test_output_version_astream(monkeypatch: Any) -> None: + messages = [AIMessage("foo bar")] + + # v0 + llm = GenericFakeChatModel(messages=iter(messages)) + full = None + async for chunk in llm.astream("hello"): + assert isinstance(chunk, AIMessageChunk) + assert isinstance(chunk.content, str) + assert chunk.content + full = chunk if full is None else full + chunk + assert isinstance(full, AIMessageChunk) + assert full.content == "foo bar" + + # v1 + llm = GenericFakeChatModel(messages=iter(messages), output_version="v1") + full_v1: Optional[BaseMessageChunk] = None + async for chunk in llm.astream("hello"): + assert isinstance(chunk, AIMessageChunk) + assert isinstance(chunk.content, list) + assert len(chunk.content) == 1 + block = chunk.content[0] + assert isinstance(block, dict) + assert block["type"] == "text" + assert block["text"] + full_v1 = chunk if full_v1 is None else full_v1 + chunk + assert isinstance(full_v1, AIMessageChunk) + assert full_v1.response_metadata["output_version"] == "v1" + + # v1 from env var + monkeypatch.setenv("LC_OUTPUT_VERSION", "v1") + llm = GenericFakeChatModel(messages=iter(messages)) + full_env = None + async for chunk in llm.astream("hello"): + assert isinstance(chunk, AIMessageChunk) + assert isinstance(chunk.content, list) + assert len(chunk.content) == 1 + block = chunk.content[0] + assert isinstance(block, dict) + assert block["type"] == "text" + assert block["text"] + full_env = chunk if full_env is None else full_env + chunk + assert isinstance(full_env, AIMessageChunk) + assert full_env.response_metadata["output_version"] == "v1" diff --git a/libs/core/tests/unit_tests/language_models/chat_models/test_cache.py b/libs/core/tests/unit_tests/language_models/chat_models/test_cache.py index 39e4babc7821f..7cf428bb3ab15 100644 --- a/libs/core/tests/unit_tests/language_models/chat_models/test_cache.py +++ b/libs/core/tests/unit_tests/language_models/chat_models/test_cache.py @@ -301,8 +301,9 @@ def test_llm_representation_for_serializable() -> None: assert chat._get_llm_string() == ( '{"id": ["tests", "unit_tests", "language_models", "chat_models", ' '"test_cache", "CustomChat"], "kwargs": {"messages": {"id": ' - '["builtins", "list_iterator"], "lc": 1, "type": "not_implemented"}}, "lc": ' - '1, "name": "CustomChat", "type": "constructor"}---[(\'stop\', None)]' + '["builtins", "list_iterator"], "lc": 1, "type": "not_implemented"}, ' + '"output_version": "v0"}, "lc": 1, "name": "CustomChat", "type": ' + "\"constructor\"}---[('stop', None)]" ) diff --git a/libs/core/tests/unit_tests/language_models/chat_models/test_rate_limiting.py b/libs/core/tests/unit_tests/language_models/chat_models/test_rate_limiting.py index c4d6a50f6bedb..0411915c26925 100644 --- a/libs/core/tests/unit_tests/language_models/chat_models/test_rate_limiting.py +++ b/libs/core/tests/unit_tests/language_models/chat_models/test_rate_limiting.py @@ -214,8 +214,8 @@ def test_rate_limit_skips_cache() -> None: assert list(cache._cache) == [ ( '[{"lc": 1, "type": "constructor", "id": ["langchain", "schema", ' - '"messages", ' - '"HumanMessage"], "kwargs": {"content": "foo", "type": "human"}}]', + '"messages", "HumanMessage"], "kwargs": {"content": "foo", ' + '"type": "human"}}]', "[('_type', 'generic-fake-chat-model'), ('stop', None)]", ) ] @@ -241,7 +241,8 @@ def test_serialization_with_rate_limiter() -> None: assert InMemoryRateLimiter.__name__ not in serialized_model -async def test_rate_limit_skips_cache_async() -> None: +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +async def test_rate_limit_skips_cache_async(output_version: str) -> None: """Test that rate limiting does not rate limit cache look ups.""" cache = InMemoryCache() model = GenericFakeChatModel( @@ -250,6 +251,7 @@ async def test_rate_limit_skips_cache_async() -> None: requests_per_second=20, check_every_n_seconds=0.1, max_bucket_size=1 ), cache=cache, + output_version=output_version, ) tic = time.time() diff --git a/libs/core/tests/unit_tests/messages/block_translators/__init__.py b/libs/core/tests/unit_tests/messages/block_translators/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/libs/core/tests/unit_tests/messages/block_translators/test_anthropic.py b/libs/core/tests/unit_tests/messages/block_translators/test_anthropic.py new file mode 100644 index 0000000000000..e0f65657b99ca --- /dev/null +++ b/libs/core/tests/unit_tests/messages/block_translators/test_anthropic.py @@ -0,0 +1,439 @@ +from typing import Optional + +from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage +from langchain_core.messages import content as types + + +def test_convert_to_v1_from_anthropic() -> None: + message = AIMessage( + [ + {"type": "thinking", "thinking": "foo", "signature": "foo_signature"}, + {"type": "text", "text": "Let's call a tool."}, + { + "type": "tool_use", + "id": "abc_123", + "name": "get_weather", + "input": {"location": "San Francisco"}, + }, + { + "type": "text", + "text": "It's sunny.", + "citations": [ + { + "type": "search_result_location", + "cited_text": "The weather is sunny.", + "source": "source_123", + "title": "Document Title", + "search_result_index": 1, + "start_block_index": 0, + "end_block_index": 2, + }, + {"bar": "baz"}, + ], + }, + { + "type": "server_tool_use", + "name": "web_search", + "input": {"query": "web search query"}, + "id": "srvtoolu_abc123", + }, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_abc123", + "content": [ + { + "type": "web_search_result", + "title": "Page Title 1", + "url": "", + "page_age": "January 1, 2025", + "encrypted_content": "", + }, + { + "type": "web_search_result", + "title": "Page Title 2", + "url": "", + "page_age": "January 2, 2025", + "encrypted_content": "", + }, + ], + }, + { + "type": "server_tool_use", + "id": "srvtoolu_def456", + "name": "code_execution", + "input": {"code": "import numpy as np..."}, + }, + { + "type": "code_execution_tool_result", + "tool_use_id": "srvtoolu_def456", + "content": { + "type": "code_execution_result", + "stdout": "Mean: 5.5\nStandard deviation...", + "stderr": "", + "return_code": 0, + }, + }, + {"type": "something_else", "foo": "bar"}, + ], + response_metadata={"model_provider": "anthropic"}, + ) + expected_content: list[types.ContentBlock] = [ + { + "type": "reasoning", + "reasoning": "foo", + "extras": {"signature": "foo_signature"}, + }, + {"type": "text", "text": "Let's call a tool."}, + { + "type": "tool_call", + "id": "abc_123", + "name": "get_weather", + "args": {"location": "San Francisco"}, + }, + { + "type": "text", + "text": "It's sunny.", + "annotations": [ + { + "type": "citation", + "title": "Document Title", + "cited_text": "The weather is sunny.", + "extras": { + "source": "source_123", + "search_result_index": 1, + "start_block_index": 0, + "end_block_index": 2, + }, + }, + {"type": "non_standard_annotation", "value": {"bar": "baz"}}, + ], + }, + { + "type": "web_search_call", + "id": "srvtoolu_abc123", + "query": "web search query", + }, + { + "type": "web_search_result", + "id": "srvtoolu_abc123", + "urls": ["", ""], + "extras": { + "content": [ + { + "type": "web_search_result", + "title": "Page Title 1", + "url": "", + "page_age": "January 1, 2025", + "encrypted_content": "", + }, + { + "type": "web_search_result", + "title": "Page Title 2", + "url": "", + "page_age": "January 2, 2025", + "encrypted_content": "", + }, + ] + }, + }, + { + "type": "code_interpreter_call", + "id": "srvtoolu_def456", + "code": "import numpy as np...", + }, + { + "type": "code_interpreter_result", + "id": "srvtoolu_def456", + "output": [ + { + "type": "code_interpreter_output", + "return_code": 0, + "stdout": "Mean: 5.5\nStandard deviation...", + } + ], + }, + { + "type": "non_standard", + "value": {"type": "something_else", "foo": "bar"}, + }, + ] + assert message.content_blocks == expected_content + + # Check no mutation + assert message.content != expected_content + + +def test_convert_to_v1_from_anthropic_chunk() -> None: + chunks = [ + AIMessageChunk( + content=[{"text": "Looking ", "type": "text", "index": 0}], + response_metadata={"model_provider": "anthropic"}, + ), + AIMessageChunk( + content=[{"text": "now.", "type": "text", "index": 0}], + response_metadata={"model_provider": "anthropic"}, + ), + AIMessageChunk( + content=[ + { + "type": "tool_use", + "name": "get_weather", + "input": {}, + "id": "toolu_abc123", + "index": 1, + } + ], + tool_call_chunks=[ + { + "type": "tool_call_chunk", + "name": "get_weather", + "args": "", + "id": "toolu_abc123", + "index": 1, + } + ], + response_metadata={"model_provider": "anthropic"}, + ), + AIMessageChunk( + content=[{"type": "input_json_delta", "partial_json": "", "index": 1}], + tool_call_chunks=[ + { + "name": None, + "args": "", + "id": None, + "index": 1, + "type": "tool_call_chunk", + } + ], + response_metadata={"model_provider": "anthropic"}, + ), + AIMessageChunk( + content=[ + {"type": "input_json_delta", "partial_json": '{"loca', "index": 1} + ], + tool_call_chunks=[ + { + "name": None, + "args": '{"loca', + "id": None, + "index": 1, + "type": "tool_call_chunk", + } + ], + response_metadata={"model_provider": "anthropic"}, + ), + AIMessageChunk( + content=[ + {"type": "input_json_delta", "partial_json": 'tion": "San ', "index": 1} + ], + tool_call_chunks=[ + { + "name": None, + "args": 'tion": "San ', + "id": None, + "index": 1, + "type": "tool_call_chunk", + } + ], + response_metadata={"model_provider": "anthropic"}, + ), + AIMessageChunk( + content=[ + {"type": "input_json_delta", "partial_json": 'Francisco"}', "index": 1} + ], + tool_call_chunks=[ + { + "name": None, + "args": 'Francisco"}', + "id": None, + "index": 1, + "type": "tool_call_chunk", + } + ], + response_metadata={"model_provider": "anthropic"}, + ), + ] + expected_contents: list[types.ContentBlock] = [ + {"type": "text", "text": "Looking ", "index": 0}, + {"type": "text", "text": "now.", "index": 0}, + { + "type": "tool_call_chunk", + "name": "get_weather", + "args": "", + "id": "toolu_abc123", + "index": 1, + }, + {"name": None, "args": "", "id": None, "index": 1, "type": "tool_call_chunk"}, + { + "name": None, + "args": '{"loca', + "id": None, + "index": 1, + "type": "tool_call_chunk", + }, + { + "name": None, + "args": 'tion": "San ', + "id": None, + "index": 1, + "type": "tool_call_chunk", + }, + { + "name": None, + "args": 'Francisco"}', + "id": None, + "index": 1, + "type": "tool_call_chunk", + }, + ] + for chunk, expected in zip(chunks, expected_contents): + assert chunk.content_blocks == [expected] + + full: Optional[AIMessageChunk] = None + for chunk in chunks: + full = chunk if full is None else full + chunk + assert isinstance(full, AIMessageChunk) + + expected_content = [ + {"type": "text", "text": "Looking now.", "index": 0}, + { + "type": "tool_use", + "name": "get_weather", + "partial_json": '{"location": "San Francisco"}', + "input": {}, + "id": "toolu_abc123", + "index": 1, + }, + ] + assert full.content == expected_content + + expected_content_blocks = [ + {"type": "text", "text": "Looking now.", "index": 0}, + { + "type": "tool_call_chunk", + "name": "get_weather", + "args": '{"location": "San Francisco"}', + "id": "toolu_abc123", + "index": 1, + }, + ] + assert full.content_blocks == expected_content_blocks + + +def test_convert_to_v1_from_anthropic_input() -> None: + message = HumanMessage( + [ + {"type": "text", "text": "foo"}, + { + "type": "document", + "source": { + "type": "base64", + "data": "", + "media_type": "application/pdf", + }, + }, + { + "type": "document", + "source": { + "type": "url", + "url": "", + }, + }, + { + "type": "document", + "source": { + "type": "content", + "content": [ + {"type": "text", "text": "The grass is green"}, + {"type": "text", "text": "The sky is blue"}, + ], + }, + "citations": {"enabled": True}, + }, + { + "type": "document", + "source": { + "type": "text", + "data": "", + "media_type": "text/plain", + }, + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": "", + }, + }, + { + "type": "image", + "source": { + "type": "url", + "url": "", + }, + }, + { + "type": "image", + "source": { + "type": "file", + "file_id": "", + }, + }, + { + "type": "document", + "source": {"type": "file", "file_id": ""}, + }, + ] + ) + + expected: list[types.ContentBlock] = [ + {"type": "text", "text": "foo"}, + { + "type": "file", + "base64": "", + "mime_type": "application/pdf", + }, + { + "type": "file", + "url": "", + }, + { + "type": "non_standard", + "value": { + "type": "document", + "source": { + "type": "content", + "content": [ + {"type": "text", "text": "The grass is green"}, + {"type": "text", "text": "The sky is blue"}, + ], + }, + "citations": {"enabled": True}, + }, + }, + { + "type": "text-plain", + "text": "", + "mime_type": "text/plain", + }, + { + "type": "image", + "base64": "", + "mime_type": "image/jpeg", + }, + { + "type": "image", + "url": "", + }, + { + "type": "image", + "id": "", + }, + { + "type": "file", + "id": "", + }, + ] + + assert message.content_blocks == expected diff --git a/libs/core/tests/unit_tests/messages/block_translators/test_langchain_v0.py b/libs/core/tests/unit_tests/messages/block_translators/test_langchain_v0.py new file mode 100644 index 0000000000000..c586f134075de --- /dev/null +++ b/libs/core/tests/unit_tests/messages/block_translators/test_langchain_v0.py @@ -0,0 +1,79 @@ +from langchain_core.messages import HumanMessage +from langchain_core.messages import content as types +from tests.unit_tests.language_models.chat_models.test_base import ( + _content_blocks_equal_ignore_id, +) + + +def test_convert_to_v1_from_openai_input() -> None: + message = HumanMessage( + content=[ + {"type": "text", "text": "Hello"}, + { + "type": "image", + "source_type": "url", + "url": "https://example.com/image.png", + }, + { + "type": "image", + "source_type": "base64", + "data": "", + "mime_type": "image/png", + }, + { + "type": "file", + "source_type": "url", + "url": "", + }, + { + "type": "file", + "source_type": "base64", + "data": "", + "mime_type": "application/pdf", + }, + { + "type": "audio", + "source_type": "base64", + "data": "", + "mime_type": "audio/mpeg", + }, + { + "type": "file", + "source_type": "id", + "id": "", + }, + ] + ) + + expected: list[types.ContentBlock] = [ + {"type": "text", "text": "Hello"}, + { + "type": "image", + "url": "https://example.com/image.png", + }, + { + "type": "image", + "base64": "", + "mime_type": "image/png", + }, + { + "type": "file", + "url": "", + }, + { + "type": "file", + "base64": "", + "mime_type": "application/pdf", + }, + { + "type": "audio", + "base64": "", + "mime_type": "audio/mpeg", + }, + { + "type": "file", + "file_id": "", + }, + ] + + assert _content_blocks_equal_ignore_id(message.content_blocks, expected) diff --git a/libs/core/tests/unit_tests/messages/block_translators/test_openai.py b/libs/core/tests/unit_tests/messages/block_translators/test_openai.py new file mode 100644 index 0000000000000..2ed2086ea4443 --- /dev/null +++ b/libs/core/tests/unit_tests/messages/block_translators/test_openai.py @@ -0,0 +1,295 @@ +from typing import Optional + +from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage +from langchain_core.messages import content as types +from tests.unit_tests.language_models.chat_models.test_base import ( + _content_blocks_equal_ignore_id, +) + + +def test_convert_to_v1_from_responses() -> None: + message = AIMessage( + [ + {"type": "reasoning", "id": "abc123", "summary": []}, + { + "type": "reasoning", + "id": "abc234", + "summary": [ + {"type": "summary_text", "text": "foo bar"}, + {"type": "summary_text", "text": "baz"}, + ], + }, + { + "type": "function_call", + "call_id": "call_123", + "name": "get_weather", + "arguments": '{"location": "San Francisco"}', + }, + { + "type": "function_call", + "call_id": "call_234", + "name": "get_weather_2", + "arguments": '{"location": "New York"}', + "id": "fc_123", + }, + {"type": "text", "text": "Hello "}, + { + "type": "text", + "text": "world", + "annotations": [ + {"type": "url_citation", "url": "https://example.com"}, + { + "type": "file_citation", + "filename": "my doc", + "index": 1, + "file_id": "file_123", + }, + {"bar": "baz"}, + ], + }, + {"type": "image_generation_call", "id": "ig_123", "result": "..."}, + {"type": "something_else", "foo": "bar"}, + ], + tool_calls=[ + { + "type": "tool_call", + "id": "call_123", + "name": "get_weather", + "args": {"location": "San Francisco"}, + }, + { + "type": "tool_call", + "id": "call_234", + "name": "get_weather_2", + "args": {"location": "New York"}, + }, + ], + response_metadata={"model_provider": "openai"}, + ) + expected_content: list[types.ContentBlock] = [ + {"type": "reasoning", "id": "abc123"}, + {"type": "reasoning", "id": "abc234", "reasoning": "foo bar"}, + {"type": "reasoning", "id": "abc234", "reasoning": "baz"}, + { + "type": "tool_call", + "id": "call_123", + "name": "get_weather", + "args": {"location": "San Francisco"}, + }, + { + "type": "tool_call", + "id": "call_234", + "name": "get_weather_2", + "args": {"location": "New York"}, + "extras": {"item_id": "fc_123"}, + }, + {"type": "text", "text": "Hello "}, + { + "type": "text", + "text": "world", + "annotations": [ + {"type": "citation", "url": "https://example.com"}, + { + "type": "citation", + "title": "my doc", + "extras": {"file_id": "file_123", "index": 1}, + }, + {"type": "non_standard_annotation", "value": {"bar": "baz"}}, + ], + }, + {"type": "image", "base64": "...", "id": "ig_123"}, + { + "type": "non_standard", + "value": {"type": "something_else", "foo": "bar"}, + }, + ] + assert message.content_blocks == expected_content + + # Check no mutation + assert message.content != expected_content + + +def test_convert_to_v1_from_responses_chunk() -> None: + chunks = [ + AIMessageChunk( + content=[{"type": "reasoning", "id": "abc123", "summary": [], "index": 0}], + response_metadata={"model_provider": "openai"}, + ), + AIMessageChunk( + content=[ + { + "type": "reasoning", + "id": "abc234", + "summary": [ + {"type": "summary_text", "text": "foo ", "index": 0}, + ], + "index": 1, + } + ], + response_metadata={"model_provider": "openai"}, + ), + AIMessageChunk( + content=[ + { + "type": "reasoning", + "id": "abc234", + "summary": [ + {"type": "summary_text", "text": "bar", "index": 0}, + ], + "index": 1, + } + ], + response_metadata={"model_provider": "openai"}, + ), + AIMessageChunk( + content=[ + { + "type": "reasoning", + "id": "abc234", + "summary": [ + {"type": "summary_text", "text": "baz", "index": 1}, + ], + "index": 1, + } + ], + response_metadata={"model_provider": "openai"}, + ), + ] + expected_chunks = [ + AIMessageChunk( + content=[{"type": "reasoning", "id": "abc123", "index": "lc_rs_305f30"}], + response_metadata={"model_provider": "openai"}, + ), + AIMessageChunk( + content=[ + { + "type": "reasoning", + "id": "abc234", + "reasoning": "foo ", + "index": "lc_rs_315f30", + } + ], + response_metadata={"model_provider": "openai"}, + ), + AIMessageChunk( + content=[ + { + "type": "reasoning", + "id": "abc234", + "reasoning": "bar", + "index": "lc_rs_315f30", + } + ], + response_metadata={"model_provider": "openai"}, + ), + AIMessageChunk( + content=[ + { + "type": "reasoning", + "id": "abc234", + "reasoning": "baz", + "index": "lc_rs_315f31", + } + ], + response_metadata={"model_provider": "openai"}, + ), + ] + for chunk, expected in zip(chunks, expected_chunks): + assert chunk.content_blocks == expected.content_blocks + + full: Optional[AIMessageChunk] = None + for chunk in chunks: + full = chunk if full is None else full + chunk + assert isinstance(full, AIMessageChunk) + + expected_content = [ + {"type": "reasoning", "id": "abc123", "summary": [], "index": 0}, + { + "type": "reasoning", + "id": "abc234", + "summary": [ + {"type": "summary_text", "text": "foo bar", "index": 0}, + {"type": "summary_text", "text": "baz", "index": 1}, + ], + "index": 1, + }, + ] + assert full.content == expected_content + + expected_content_blocks = [ + {"type": "reasoning", "id": "abc123", "index": "lc_rs_305f30"}, + { + "type": "reasoning", + "id": "abc234", + "reasoning": "foo bar", + "index": "lc_rs_315f30", + }, + { + "type": "reasoning", + "id": "abc234", + "reasoning": "baz", + "index": "lc_rs_315f31", + }, + ] + assert full.content_blocks == expected_content_blocks + + +def test_convert_to_v1_from_openai_input() -> None: + message = HumanMessage( + content=[ + {"type": "text", "text": "Hello"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + }, + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZJRg..."}, + }, + { + "type": "input_audio", + "input_audio": { + "format": "wav", + "data": "", + }, + }, + { + "type": "file", + "file": { + "filename": "draconomicon.pdf", + "file_data": "", + }, + }, + { + "type": "file", + "file": {"file_id": ""}, + }, + ] + ) + + expected: list[types.ContentBlock] = [ + {"type": "text", "text": "Hello"}, + { + "type": "image", + "url": "https://example.com/image.png", + }, + { + "type": "image", + "base64": "/9j/4AAQSkZJRg...", + "mime_type": "image/jpeg", + }, + { + "type": "audio", + "base64": "", + "mime_type": "audio/wav", + }, + { + "type": "file", + "base64": "", + "mime_type": "application/pdf", + "extras": {"filename": "draconomicon.pdf"}, + }, + {"type": "file", "file_id": ""}, + ] + + assert _content_blocks_equal_ignore_id(message.content_blocks, expected) diff --git a/libs/core/tests/unit_tests/messages/block_translators/test_registration.py b/libs/core/tests/unit_tests/messages/block_translators/test_registration.py new file mode 100644 index 0000000000000..74c16d30a248a --- /dev/null +++ b/libs/core/tests/unit_tests/messages/block_translators/test_registration.py @@ -0,0 +1,29 @@ +import pkgutil +from pathlib import Path + +import pytest + +from langchain_core.messages.block_translators import PROVIDER_TRANSLATORS + + +def test_all_providers_registered() -> None: + """Test that all block translators implemented in langchain-core are registered. + + If this test fails, it is likely that a block translator is implemented but not + registered on import. Check that the provider is included in + ``langchain_core.messages.block_translators.__init__._register_translators``. + """ + package_path = ( + Path(__file__).parents[4] / "langchain_core" / "messages" / "block_translators" + ) + + for module_info in pkgutil.iter_modules([str(package_path)]): + module_name = module_info.name + + # Skip the __init__ module, any private modules, and ``langchain_v0``, which is + # only used to parse v0 multimodal inputs. + if module_name.startswith("_") or module_name == "langchain_v0": + continue + + if module_name not in PROVIDER_TRANSLATORS: + pytest.fail(f"Block translator not registered: {module_name}") diff --git a/libs/core/tests/unit_tests/messages/test_ai.py b/libs/core/tests/unit_tests/messages/test_ai.py index d36d034712817..a0edf0b5714bc 100644 --- a/libs/core/tests/unit_tests/messages/test_ai.py +++ b/libs/core/tests/unit_tests/messages/test_ai.py @@ -1,5 +1,10 @@ +from typing import Union, cast + +import pytest + from langchain_core.load import dumpd, load from langchain_core.messages import AIMessage, AIMessageChunk +from langchain_core.messages import content as types from langchain_core.messages.ai import ( InputTokenDetails, OutputTokenDetails, @@ -196,3 +201,215 @@ def test_add_ai_message_chunks_usage() -> None: output_token_details=OutputTokenDetails(audio=1, reasoning=2), ), ) + + +def test_init_tool_calls() -> None: + # Test we add "type" key on init + msg = AIMessage("", tool_calls=[{"name": "foo", "args": {"a": "b"}, "id": "abc"}]) + assert len(msg.tool_calls) == 1 + assert msg.tool_calls[0]["type"] == "tool_call" + + # Test we can assign without adding type key + msg.tool_calls = [{"name": "bar", "args": {"c": "d"}, "id": "def"}] + + +def test_content_blocks() -> None: + message = AIMessage( + "", + tool_calls=[ + {"type": "tool_call", "name": "foo", "args": {"a": "b"}, "id": "abc_123"} + ], + ) + assert len(message.content_blocks) == 1 + assert message.content_blocks[0]["type"] == "tool_call" + assert message.content_blocks == [ + {"type": "tool_call", "id": "abc_123", "name": "foo", "args": {"a": "b"}} + ] + assert message.content == "" + + message = AIMessage( + "foo", + tool_calls=[ + {"type": "tool_call", "name": "foo", "args": {"a": "b"}, "id": "abc_123"} + ], + ) + assert len(message.content_blocks) == 2 + assert message.content_blocks[0]["type"] == "text" + assert message.content_blocks[1]["type"] == "tool_call" + assert message.content_blocks == [ + {"type": "text", "text": "foo"}, + {"type": "tool_call", "id": "abc_123", "name": "foo", "args": {"a": "b"}}, + ] + assert message.content == "foo" + + # With standard blocks + standard_content: list[types.ContentBlock] = [ + {"type": "reasoning", "reasoning": "foo"}, + {"type": "text", "text": "bar"}, + { + "type": "text", + "text": "baz", + "annotations": [{"type": "citation", "url": "http://example.com"}], + }, + { + "type": "image", + "url": "http://example.com/image.png", + "extras": {"foo": "bar"}, + }, + { + "type": "non_standard", + "value": {"custom_key": "custom_value", "another_key": 123}, + }, + { + "type": "tool_call", + "name": "foo", + "args": {"a": "b"}, + "id": "abc_123", + }, + ] + missing_tool_call: types.ToolCall = { + "type": "tool_call", + "name": "bar", + "args": {"c": "d"}, + "id": "abc_234", + } + message = AIMessage( + content_blocks=standard_content, + tool_calls=[ + {"type": "tool_call", "name": "foo", "args": {"a": "b"}, "id": "abc_123"}, + missing_tool_call, + ], + ) + assert message.content_blocks == [*standard_content, missing_tool_call] + + # Check we auto-populate tool_calls + standard_content = [ + {"type": "text", "text": "foo"}, + { + "type": "tool_call", + "name": "foo", + "args": {"a": "b"}, + "id": "abc_123", + }, + missing_tool_call, + ] + message = AIMessage(content_blocks=standard_content) + assert message.tool_calls == [ + {"type": "tool_call", "name": "foo", "args": {"a": "b"}, "id": "abc_123"}, + missing_tool_call, + ] + + # Chunks + message = AIMessageChunk( + content="", + tool_call_chunks=[ + { + "type": "tool_call_chunk", + "name": "foo", + "args": "", + "id": "abc_123", + "index": 0, + } + ], + ) + assert len(message.content_blocks) == 1 + assert message.content_blocks[0]["type"] == "tool_call_chunk" + assert message.content_blocks == [ + { + "type": "tool_call_chunk", + "name": "foo", + "args": "", + "id": "abc_123", + "index": 0, + } + ] + assert message.content == "" + + # Non-standard + standard_content_1: list[types.ContentBlock] = [ + {"type": "non_standard", "index": 0, "value": {"foo": "bar "}} + ] + standard_content_2: list[types.ContentBlock] = [ + {"type": "non_standard", "index": 0, "value": {"foo": "baz"}} + ] + chunk_1 = AIMessageChunk( + content=cast("Union[str, list[Union[str, dict]]]", standard_content_1) + ) + chunk_2 = AIMessageChunk( + content=cast("Union[str, list[Union[str, dict]]]", standard_content_2) + ) + merged_chunk = chunk_1 + chunk_2 + assert merged_chunk.content == [ + {"type": "non_standard", "index": 0, "value": {"foo": "bar baz"}}, + ] + + # Test non-standard + non-standard + chunk_1 = AIMessageChunk( + content=[ + { + "type": "non_standard", + "index": 0, + "value": {"type": "non_standard_tool", "foo": "bar"}, + } + ] + ) + chunk_2 = AIMessageChunk( + content=[ + { + "type": "non_standard", + "index": 0, + "value": {"type": "input_json_delta", "partial_json": "a"}, + } + ] + ) + chunk_3 = AIMessageChunk( + content=[ + { + "type": "non_standard", + "index": 0, + "value": {"type": "input_json_delta", "partial_json": "b"}, + } + ] + ) + merged_chunk = chunk_1 + chunk_2 + chunk_3 + assert merged_chunk.content == [ + { + "type": "non_standard", + "index": 0, + "value": {"type": "non_standard_tool", "foo": "bar", "partial_json": "ab"}, + } + ] + + # Test standard + non-standard with same index + standard_content_1 = [ + {"type": "web_search_call", "id": "ws_123", "query": "web query", "index": 0} + ] + standard_content_2 = [{"type": "non_standard", "value": {"foo": "bar"}, "index": 0}] + chunk_1 = AIMessageChunk( + content=cast("Union[str, list[Union[str, dict]]]", standard_content_1) + ) + chunk_2 = AIMessageChunk( + content=cast("Union[str, list[Union[str, dict]]]", standard_content_2) + ) + merged_chunk = chunk_1 + chunk_2 + assert merged_chunk.content == [ + { + "type": "web_search_call", + "id": "ws_123", + "query": "web query", + "index": 0, + "extras": {"foo": "bar"}, + } + ] + + +def test_provider_warns() -> None: + # Test that major providers warn if content block standardization is not yet + # implemented. + # This test should be removed when all major providers support content block + # standardization. + message = AIMessage("Hello.", response_metadata={"model_provider": "groq"}) + with pytest.warns(match="not yet fully supported for Groq"): + content_blocks = message.content_blocks + + assert content_blocks == [{"type": "text", "text": "Hello."}] diff --git a/libs/core/tests/unit_tests/messages/test_imports.py b/libs/core/tests/unit_tests/messages/test_imports.py index ff9fbf92fc77e..bf438b0cd8eac 100644 --- a/libs/core/tests/unit_tests/messages/test_imports.py +++ b/libs/core/tests/unit_tests/messages/test_imports.py @@ -5,26 +5,51 @@ "_message_from_dict", "AIMessage", "AIMessageChunk", + "Annotation", "AnyMessage", + "AudioContentBlock", "BaseMessage", "BaseMessageChunk", + "ContentBlock", "ChatMessage", "ChatMessageChunk", + "Citation", + "CodeInterpreterCall", + "CodeInterpreterOutput", + "CodeInterpreterResult", + "DataContentBlock", + "FileContentBlock", "FunctionMessage", "FunctionMessageChunk", "HumanMessage", "HumanMessageChunk", + "ImageContentBlock", "InvalidToolCall", + "LC_AUTO_PREFIX", + "LC_ID_PREFIX", + "NonStandardAnnotation", + "NonStandardContentBlock", + "PlainTextContentBlock", "SystemMessage", "SystemMessageChunk", + "TextContentBlock", "ToolCall", "ToolCallChunk", "ToolMessage", "ToolMessageChunk", + "VideoContentBlock", + "WebSearchCall", + "WebSearchResult", + "ReasoningContentBlock", "RemoveMessage", "convert_to_messages", + "ensure_id", "get_buffer_string", "is_data_content_block", + "is_reasoning_block", + "is_text_block", + "is_tool_call_block", + "is_tool_call_chunk", "merge_content", "message_chunk_to_message", "message_to_dict", diff --git a/libs/core/tests/unit_tests/messages/test_utils.py b/libs/core/tests/unit_tests/messages/test_utils.py index bedd518589ea0..d655fd13bdbf5 100644 --- a/libs/core/tests/unit_tests/messages/test_utils.py +++ b/libs/core/tests/unit_tests/messages/test_utils.py @@ -1215,13 +1215,14 @@ def test_convert_to_openai_messages_developer() -> None: def test_convert_to_openai_messages_multimodal() -> None: + """v0 and v1 content to OpenAI messages conversion.""" messages = [ HumanMessage( content=[ + # Prior v0 blocks {"type": "text", "text": "Text message"}, { "type": "image", - "source_type": "url", "url": "https://example.com/test.png", }, { @@ -1238,6 +1239,7 @@ def test_convert_to_openai_messages_multimodal() -> None: "filename": "test.pdf", }, { + # OpenAI Chat Completions file format "type": "file", "file": { "filename": "draconomicon.pdf", @@ -1262,22 +1264,47 @@ def test_convert_to_openai_messages_multimodal() -> None: "format": "wav", }, }, + # v1 Additions + { + "type": "image", + "source_type": "url", # backward compatibility v0 block field + "url": "https://example.com/test.png", + }, + { + "type": "image", + "base64": "", + "mime_type": "image/png", + }, + { + "type": "file", + "base64": "", + "mime_type": "application/pdf", + "filename": "test.pdf", # backward compatibility v0 block field + }, + { + "type": "file", + "file_id": "file-abc123", + }, + { + "type": "audio", + "base64": "", + "mime_type": "audio/wav", + }, ] ) ] result = convert_to_openai_messages(messages, text_format="block") assert len(result) == 1 message = result[0] - assert len(message["content"]) == 8 + assert len(message["content"]) == 13 - # Test adding filename + # Test auto-adding filename messages = [ HumanMessage( content=[ { "type": "file", - "source_type": "base64", - "data": "", + "base64": "", "mime_type": "application/pdf", }, ] @@ -1290,6 +1317,7 @@ def test_convert_to_openai_messages_multimodal() -> None: assert len(message["content"]) == 1 block = message["content"][0] assert block == { + # OpenAI Chat Completions file format "type": "file", "file": { "file_data": "data:application/pdf;base64,", diff --git a/libs/core/tests/unit_tests/prompts/__snapshots__/test_chat.ambr b/libs/core/tests/unit_tests/prompts/__snapshots__/test_chat.ambr index 7c07416fe5d9c..80ab67312f33a 100644 --- a/libs/core/tests/unit_tests/prompts/__snapshots__/test_chat.ambr +++ b/libs/core/tests/unit_tests/prompts/__snapshots__/test_chat.ambr @@ -382,10 +382,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - FunctionMessage are an older version of the ToolMessage schema, and - do not contain the tool_call_id field. + ``FunctionMessage`` are an older version of the ``ToolMessage`` schema, and + do not contain the ``tool_call_id`` field. - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -517,7 +517,7 @@ 'description': ''' Message from a human. - HumanMessages are messages that are passed in from a human to the model. + ``HumanMessage``s are messages that are passed in from a human to the model. Example: @@ -692,7 +692,6 @@ Does *not* need to sum to full input token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -726,7 +725,7 @@ 'description': ''' Allowance for errors made by LLM. - Here we add an `error` key to surface errors made during generation + Here we add an ``error`` key to surface errors made during generation (e.g., invalid JSON arguments.) ''', 'properties': dict({ @@ -752,6 +751,10 @@ ]), 'title': 'Error', }), + 'extras': dict({ + 'title': 'Extras', + 'type': 'object', + }), 'id': dict({ 'anyOf': list([ dict({ @@ -763,6 +766,17 @@ ]), 'title': 'Id', }), + 'index': dict({ + 'anyOf': list([ + dict({ + 'type': 'integer', + }), + dict({ + 'type': 'string', + }), + ]), + 'title': 'Index', + }), 'name': dict({ 'anyOf': list([ dict({ @@ -781,9 +795,10 @@ }), }), 'required': list([ + 'type', + 'id', 'name', 'args', - 'id', 'error', ]), 'title': 'InvalidToolCall', @@ -796,7 +811,6 @@ Does *not* need to sum to full output token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -996,8 +1010,8 @@ "id": "123" } - This represents a request to call the tool named "foo" with arguments {"a": 1} - and an identifier of "123". + This represents a request to call the tool named ``'foo'`` with arguments + ``{"a": 1}`` and an identifier of ``'123'``. ''', 'properties': dict({ 'args': dict({ @@ -1037,9 +1051,9 @@ 'description': ''' A chunk of a tool call (e.g., as part of a stream). - When merging ToolCallChunks (e.g., via AIMessageChunk.__add__), + When merging ``ToolCallChunk``s (e.g., via ``AIMessageChunk.__add__``), all string attributes are concatenated. Chunks are only merged if their - values of `index` are equal and not None. + values of ``index`` are equal and not None. Example: @@ -1118,10 +1132,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - ToolMessages contain the result of a tool invocation. Typically, the result - is encoded inside the `content` field. + ``ToolMessage``s contain the result of a tool invocation. Typically, the result + is encoded inside the ``content`` field. - Example: A ToolMessage representing a result of 42 from a tool call with id + Example: A ``ToolMessage`` representing a result of ``42`` from a tool call with id .. code-block:: python @@ -1130,7 +1144,7 @@ ToolMessage(content='42', tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL') - Example: A ToolMessage where only part of the tool output is sent to the model + Example: A ``ToolMessage`` where only part of the tool output is sent to the model and the full output is passed in to artifact. .. versionadded:: 0.2.17 @@ -1151,7 +1165,7 @@ tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL', ) - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -1323,7 +1337,6 @@ This is a standard representation of token usage that is consistent across models. Example: - .. code-block:: python { @@ -1814,10 +1827,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - FunctionMessage are an older version of the ToolMessage schema, and - do not contain the tool_call_id field. + ``FunctionMessage`` are an older version of the ``ToolMessage`` schema, and + do not contain the ``tool_call_id`` field. - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -1949,7 +1962,7 @@ 'description': ''' Message from a human. - HumanMessages are messages that are passed in from a human to the model. + ``HumanMessage``s are messages that are passed in from a human to the model. Example: @@ -2124,7 +2137,6 @@ Does *not* need to sum to full input token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -2158,7 +2170,7 @@ 'description': ''' Allowance for errors made by LLM. - Here we add an `error` key to surface errors made during generation + Here we add an ``error`` key to surface errors made during generation (e.g., invalid JSON arguments.) ''', 'properties': dict({ @@ -2184,6 +2196,10 @@ ]), 'title': 'Error', }), + 'extras': dict({ + 'title': 'Extras', + 'type': 'object', + }), 'id': dict({ 'anyOf': list([ dict({ @@ -2195,6 +2211,17 @@ ]), 'title': 'Id', }), + 'index': dict({ + 'anyOf': list([ + dict({ + 'type': 'integer', + }), + dict({ + 'type': 'string', + }), + ]), + 'title': 'Index', + }), 'name': dict({ 'anyOf': list([ dict({ @@ -2213,9 +2240,10 @@ }), }), 'required': list([ + 'type', + 'id', 'name', 'args', - 'id', 'error', ]), 'title': 'InvalidToolCall', @@ -2228,7 +2256,6 @@ Does *not* need to sum to full output token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -2428,8 +2455,8 @@ "id": "123" } - This represents a request to call the tool named "foo" with arguments {"a": 1} - and an identifier of "123". + This represents a request to call the tool named ``'foo'`` with arguments + ``{"a": 1}`` and an identifier of ``'123'``. ''', 'properties': dict({ 'args': dict({ @@ -2469,9 +2496,9 @@ 'description': ''' A chunk of a tool call (e.g., as part of a stream). - When merging ToolCallChunks (e.g., via AIMessageChunk.__add__), + When merging ``ToolCallChunk``s (e.g., via ``AIMessageChunk.__add__``), all string attributes are concatenated. Chunks are only merged if their - values of `index` are equal and not None. + values of ``index`` are equal and not None. Example: @@ -2550,10 +2577,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - ToolMessages contain the result of a tool invocation. Typically, the result - is encoded inside the `content` field. + ``ToolMessage``s contain the result of a tool invocation. Typically, the result + is encoded inside the ``content`` field. - Example: A ToolMessage representing a result of 42 from a tool call with id + Example: A ``ToolMessage`` representing a result of ``42`` from a tool call with id .. code-block:: python @@ -2562,7 +2589,7 @@ ToolMessage(content='42', tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL') - Example: A ToolMessage where only part of the tool output is sent to the model + Example: A ``ToolMessage`` where only part of the tool output is sent to the model and the full output is passed in to artifact. .. versionadded:: 0.2.17 @@ -2583,7 +2610,7 @@ tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL', ) - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -2755,7 +2782,6 @@ This is a standard representation of token usage that is consistent across models. Example: - .. code-block:: python { diff --git a/libs/core/tests/unit_tests/runnables/__snapshots__/test_graph.ambr b/libs/core/tests/unit_tests/runnables/__snapshots__/test_graph.ambr index a788c425fced3..8ed1ab684bec0 100644 --- a/libs/core/tests/unit_tests/runnables/__snapshots__/test_graph.ambr +++ b/libs/core/tests/unit_tests/runnables/__snapshots__/test_graph.ambr @@ -785,10 +785,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - FunctionMessage are an older version of the ToolMessage schema, and - do not contain the tool_call_id field. + ``FunctionMessage`` are an older version of the ``ToolMessage`` schema, and + do not contain the ``tool_call_id`` field. - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -920,7 +920,7 @@ 'description': ''' Message from a human. - HumanMessages are messages that are passed in from a human to the model. + ``HumanMessage``s are messages that are passed in from a human to the model. Example: @@ -1095,7 +1095,6 @@ Does *not* need to sum to full input token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -1129,7 +1128,7 @@ 'description': ''' Allowance for errors made by LLM. - Here we add an `error` key to surface errors made during generation + Here we add an ``error`` key to surface errors made during generation (e.g., invalid JSON arguments.) ''', 'properties': dict({ @@ -1155,6 +1154,10 @@ ]), 'title': 'Error', }), + 'extras': dict({ + 'title': 'Extras', + 'type': 'object', + }), 'id': dict({ 'anyOf': list([ dict({ @@ -1166,6 +1169,17 @@ ]), 'title': 'Id', }), + 'index': dict({ + 'anyOf': list([ + dict({ + 'type': 'integer', + }), + dict({ + 'type': 'string', + }), + ]), + 'title': 'Index', + }), 'name': dict({ 'anyOf': list([ dict({ @@ -1184,9 +1198,10 @@ }), }), 'required': list([ + 'type', + 'id', 'name', 'args', - 'id', 'error', ]), 'title': 'InvalidToolCall', @@ -1199,7 +1214,6 @@ Does *not* need to sum to full output token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -1399,8 +1413,8 @@ "id": "123" } - This represents a request to call the tool named "foo" with arguments {"a": 1} - and an identifier of "123". + This represents a request to call the tool named ``'foo'`` with arguments + ``{"a": 1}`` and an identifier of ``'123'``. ''', 'properties': dict({ 'args': dict({ @@ -1440,9 +1454,9 @@ 'description': ''' A chunk of a tool call (e.g., as part of a stream). - When merging ToolCallChunks (e.g., via AIMessageChunk.__add__), + When merging ``ToolCallChunk``s (e.g., via ``AIMessageChunk.__add__``), all string attributes are concatenated. Chunks are only merged if their - values of `index` are equal and not None. + values of ``index`` are equal and not None. Example: @@ -1521,10 +1535,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - ToolMessages contain the result of a tool invocation. Typically, the result - is encoded inside the `content` field. + ``ToolMessage``s contain the result of a tool invocation. Typically, the result + is encoded inside the ``content`` field. - Example: A ToolMessage representing a result of 42 from a tool call with id + Example: A ``ToolMessage`` representing a result of ``42`` from a tool call with id .. code-block:: python @@ -1533,7 +1547,7 @@ ToolMessage(content='42', tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL') - Example: A ToolMessage where only part of the tool output is sent to the model + Example: A ``ToolMessage`` where only part of the tool output is sent to the model and the full output is passed in to artifact. .. versionadded:: 0.2.17 @@ -1554,7 +1568,7 @@ tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL', ) - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -1726,7 +1740,6 @@ This is a standard representation of token usage that is consistent across models. Example: - .. code-block:: python { diff --git a/libs/core/tests/unit_tests/runnables/__snapshots__/test_runnable.ambr b/libs/core/tests/unit_tests/runnables/__snapshots__/test_runnable.ambr index 079e490906158..87d9a140a9a99 100644 --- a/libs/core/tests/unit_tests/runnables/__snapshots__/test_runnable.ambr +++ b/libs/core/tests/unit_tests/runnables/__snapshots__/test_runnable.ambr @@ -2334,10 +2334,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - FunctionMessage are an older version of the ToolMessage schema, and - do not contain the tool_call_id field. + ``FunctionMessage`` are an older version of the ``ToolMessage`` schema, and + do not contain the ``tool_call_id`` field. - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -2467,7 +2467,7 @@ 'description': ''' Message from a human. - HumanMessages are messages that are passed in from a human to the model. + ``HumanMessage``s are messages that are passed in from a human to the model. Example: @@ -2640,7 +2640,6 @@ Does *not* need to sum to full input token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -2674,7 +2673,7 @@ 'description': ''' Allowance for errors made by LLM. - Here we add an `error` key to surface errors made during generation + Here we add an ``error`` key to surface errors made during generation (e.g., invalid JSON arguments.) ''', 'properties': dict({ @@ -2743,7 +2742,6 @@ Does *not* need to sum to full output token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -2941,8 +2939,8 @@ "id": "123" } - This represents a request to call the tool named "foo" with arguments {"a": 1} - and an identifier of "123". + This represents a request to call the tool named ``'foo'`` with arguments + ``{"a": 1}`` and an identifier of ``'123'``. ''', 'properties': dict({ 'args': dict({ @@ -2981,9 +2979,9 @@ 'description': ''' A chunk of a tool call (e.g., as part of a stream). - When merging ToolCallChunks (e.g., via AIMessageChunk.__add__), + When merging ``ToolCallChunk``s (e.g., via ``AIMessageChunk.__add__``), all string attributes are concatenated. Chunks are only merged if their - values of `index` are equal and not None. + values of ``index`` are equal and not None. Example: @@ -3061,10 +3059,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - ToolMessages contain the result of a tool invocation. Typically, the result - is encoded inside the `content` field. + ``ToolMessage``s contain the result of a tool invocation. Typically, the result + is encoded inside the ``content`` field. - Example: A ToolMessage representing a result of 42 from a tool call with id + Example: A ``ToolMessage`` representing a result of ``42`` from a tool call with id .. code-block:: python @@ -3073,7 +3071,7 @@ ToolMessage(content='42', tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL') - Example: A ToolMessage where only part of the tool output is sent to the model + Example: A ``ToolMessage`` where only part of the tool output is sent to the model and the full output is passed in to artifact. .. versionadded:: 0.2.17 @@ -3094,7 +3092,7 @@ tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL', ) - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -3264,7 +3262,6 @@ This is a standard representation of token usage that is consistent across models. Example: - .. code-block:: python { @@ -3810,10 +3807,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - FunctionMessage are an older version of the ToolMessage schema, and - do not contain the tool_call_id field. + ``FunctionMessage`` are an older version of the ``ToolMessage`` schema, and + do not contain the ``tool_call_id`` field. - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -3943,7 +3940,7 @@ 'description': ''' Message from a human. - HumanMessages are messages that are passed in from a human to the model. + ``HumanMessage``s are messages that are passed in from a human to the model. Example: @@ -4116,7 +4113,6 @@ Does *not* need to sum to full input token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -4150,7 +4146,7 @@ 'description': ''' Allowance for errors made by LLM. - Here we add an `error` key to surface errors made during generation + Here we add an ``error`` key to surface errors made during generation (e.g., invalid JSON arguments.) ''', 'properties': dict({ @@ -4219,7 +4215,6 @@ Does *not* need to sum to full output token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -4436,8 +4431,8 @@ "id": "123" } - This represents a request to call the tool named "foo" with arguments {"a": 1} - and an identifier of "123". + This represents a request to call the tool named ``'foo'`` with arguments + ``{"a": 1}`` and an identifier of ``'123'``. ''', 'properties': dict({ 'args': dict({ @@ -4476,9 +4471,9 @@ 'description': ''' A chunk of a tool call (e.g., as part of a stream). - When merging ToolCallChunks (e.g., via AIMessageChunk.__add__), + When merging ``ToolCallChunk``s (e.g., via ``AIMessageChunk.__add__``), all string attributes are concatenated. Chunks are only merged if their - values of `index` are equal and not None. + values of ``index`` are equal and not None. Example: @@ -4556,10 +4551,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - ToolMessages contain the result of a tool invocation. Typically, the result - is encoded inside the `content` field. + ``ToolMessage``s contain the result of a tool invocation. Typically, the result + is encoded inside the ``content`` field. - Example: A ToolMessage representing a result of 42 from a tool call with id + Example: A ``ToolMessage`` representing a result of ``42`` from a tool call with id .. code-block:: python @@ -4568,7 +4563,7 @@ ToolMessage(content='42', tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL') - Example: A ToolMessage where only part of the tool output is sent to the model + Example: A ``ToolMessage`` where only part of the tool output is sent to the model and the full output is passed in to artifact. .. versionadded:: 0.2.17 @@ -4589,7 +4584,7 @@ tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL', ) - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -4759,7 +4754,6 @@ This is a standard representation of token usage that is consistent across models. Example: - .. code-block:: python { @@ -5317,10 +5311,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - FunctionMessage are an older version of the ToolMessage schema, and - do not contain the tool_call_id field. + ``FunctionMessage`` are an older version of the ``ToolMessage`` schema, and + do not contain the ``tool_call_id`` field. - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -5450,7 +5444,7 @@ 'description': ''' Message from a human. - HumanMessages are messages that are passed in from a human to the model. + ``HumanMessage``s are messages that are passed in from a human to the model. Example: @@ -5623,7 +5617,6 @@ Does *not* need to sum to full input token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -5657,7 +5650,7 @@ 'description': ''' Allowance for errors made by LLM. - Here we add an `error` key to surface errors made during generation + Here we add an ``error`` key to surface errors made during generation (e.g., invalid JSON arguments.) ''', 'properties': dict({ @@ -5726,7 +5719,6 @@ Does *not* need to sum to full output token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -5943,8 +5935,8 @@ "id": "123" } - This represents a request to call the tool named "foo" with arguments {"a": 1} - and an identifier of "123". + This represents a request to call the tool named ``'foo'`` with arguments + ``{"a": 1}`` and an identifier of ``'123'``. ''', 'properties': dict({ 'args': dict({ @@ -5983,9 +5975,9 @@ 'description': ''' A chunk of a tool call (e.g., as part of a stream). - When merging ToolCallChunks (e.g., via AIMessageChunk.__add__), + When merging ``ToolCallChunk``s (e.g., via ``AIMessageChunk.__add__``), all string attributes are concatenated. Chunks are only merged if their - values of `index` are equal and not None. + values of ``index`` are equal and not None. Example: @@ -6063,10 +6055,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - ToolMessages contain the result of a tool invocation. Typically, the result - is encoded inside the `content` field. + ``ToolMessage``s contain the result of a tool invocation. Typically, the result + is encoded inside the ``content`` field. - Example: A ToolMessage representing a result of 42 from a tool call with id + Example: A ``ToolMessage`` representing a result of ``42`` from a tool call with id .. code-block:: python @@ -6075,7 +6067,7 @@ ToolMessage(content='42', tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL') - Example: A ToolMessage where only part of the tool output is sent to the model + Example: A ``ToolMessage`` where only part of the tool output is sent to the model and the full output is passed in to artifact. .. versionadded:: 0.2.17 @@ -6096,7 +6088,7 @@ tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL', ) - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -6266,7 +6258,6 @@ This is a standard representation of token usage that is consistent across models. Example: - .. code-block:: python { @@ -6699,10 +6690,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - FunctionMessage are an older version of the ToolMessage schema, and - do not contain the tool_call_id field. + ``FunctionMessage`` are an older version of the ``ToolMessage`` schema, and + do not contain the ``tool_call_id`` field. - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -6832,7 +6823,7 @@ 'description': ''' Message from a human. - HumanMessages are messages that are passed in from a human to the model. + ``HumanMessage``s are messages that are passed in from a human to the model. Example: @@ -7005,7 +6996,6 @@ Does *not* need to sum to full input token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -7039,7 +7029,7 @@ 'description': ''' Allowance for errors made by LLM. - Here we add an `error` key to surface errors made during generation + Here we add an ``error`` key to surface errors made during generation (e.g., invalid JSON arguments.) ''', 'properties': dict({ @@ -7108,7 +7098,6 @@ Does *not* need to sum to full output token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -7306,8 +7295,8 @@ "id": "123" } - This represents a request to call the tool named "foo" with arguments {"a": 1} - and an identifier of "123". + This represents a request to call the tool named ``'foo'`` with arguments + ``{"a": 1}`` and an identifier of ``'123'``. ''', 'properties': dict({ 'args': dict({ @@ -7346,9 +7335,9 @@ 'description': ''' A chunk of a tool call (e.g., as part of a stream). - When merging ToolCallChunks (e.g., via AIMessageChunk.__add__), + When merging ``ToolCallChunk``s (e.g., via ``AIMessageChunk.__add__``), all string attributes are concatenated. Chunks are only merged if their - values of `index` are equal and not None. + values of ``index`` are equal and not None. Example: @@ -7426,10 +7415,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - ToolMessages contain the result of a tool invocation. Typically, the result - is encoded inside the `content` field. + ``ToolMessage``s contain the result of a tool invocation. Typically, the result + is encoded inside the ``content`` field. - Example: A ToolMessage representing a result of 42 from a tool call with id + Example: A ``ToolMessage`` representing a result of ``42`` from a tool call with id .. code-block:: python @@ -7438,7 +7427,7 @@ ToolMessage(content='42', tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL') - Example: A ToolMessage where only part of the tool output is sent to the model + Example: A ``ToolMessage`` where only part of the tool output is sent to the model and the full output is passed in to artifact. .. versionadded:: 0.2.17 @@ -7459,7 +7448,7 @@ tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL', ) - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -7629,7 +7618,6 @@ This is a standard representation of token usage that is consistent across models. Example: - .. code-block:: python { @@ -8217,10 +8205,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - FunctionMessage are an older version of the ToolMessage schema, and - do not contain the tool_call_id field. + ``FunctionMessage`` are an older version of the ``ToolMessage`` schema, and + do not contain the ``tool_call_id`` field. - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -8350,7 +8338,7 @@ 'description': ''' Message from a human. - HumanMessages are messages that are passed in from a human to the model. + ``HumanMessage``s are messages that are passed in from a human to the model. Example: @@ -8523,7 +8511,6 @@ Does *not* need to sum to full input token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -8557,7 +8544,7 @@ 'description': ''' Allowance for errors made by LLM. - Here we add an `error` key to surface errors made during generation + Here we add an ``error`` key to surface errors made during generation (e.g., invalid JSON arguments.) ''', 'properties': dict({ @@ -8626,7 +8613,6 @@ Does *not* need to sum to full output token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -8843,8 +8829,8 @@ "id": "123" } - This represents a request to call the tool named "foo" with arguments {"a": 1} - and an identifier of "123". + This represents a request to call the tool named ``'foo'`` with arguments + ``{"a": 1}`` and an identifier of ``'123'``. ''', 'properties': dict({ 'args': dict({ @@ -8883,9 +8869,9 @@ 'description': ''' A chunk of a tool call (e.g., as part of a stream). - When merging ToolCallChunks (e.g., via AIMessageChunk.__add__), + When merging ``ToolCallChunk``s (e.g., via ``AIMessageChunk.__add__``), all string attributes are concatenated. Chunks are only merged if their - values of `index` are equal and not None. + values of ``index`` are equal and not None. Example: @@ -8963,10 +8949,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - ToolMessages contain the result of a tool invocation. Typically, the result - is encoded inside the `content` field. + ``ToolMessage``s contain the result of a tool invocation. Typically, the result + is encoded inside the ``content`` field. - Example: A ToolMessage representing a result of 42 from a tool call with id + Example: A ``ToolMessage`` representing a result of ``42`` from a tool call with id .. code-block:: python @@ -8975,7 +8961,7 @@ ToolMessage(content='42', tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL') - Example: A ToolMessage where only part of the tool output is sent to the model + Example: A ``ToolMessage`` where only part of the tool output is sent to the model and the full output is passed in to artifact. .. versionadded:: 0.2.17 @@ -8996,7 +8982,7 @@ tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL', ) - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -9166,7 +9152,6 @@ This is a standard representation of token usage that is consistent across models. Example: - .. code-block:: python { @@ -9644,10 +9629,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - FunctionMessage are an older version of the ToolMessage schema, and - do not contain the tool_call_id field. + ``FunctionMessage`` are an older version of the ``ToolMessage`` schema, and + do not contain the ``tool_call_id`` field. - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -9777,7 +9762,7 @@ 'description': ''' Message from a human. - HumanMessages are messages that are passed in from a human to the model. + ``HumanMessage``s are messages that are passed in from a human to the model. Example: @@ -9950,7 +9935,6 @@ Does *not* need to sum to full input token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -9984,7 +9968,7 @@ 'description': ''' Allowance for errors made by LLM. - Here we add an `error` key to surface errors made during generation + Here we add an ``error`` key to surface errors made during generation (e.g., invalid JSON arguments.) ''', 'properties': dict({ @@ -10053,7 +10037,6 @@ Does *not* need to sum to full output token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -10251,8 +10234,8 @@ "id": "123" } - This represents a request to call the tool named "foo" with arguments {"a": 1} - and an identifier of "123". + This represents a request to call the tool named ``'foo'`` with arguments + ``{"a": 1}`` and an identifier of ``'123'``. ''', 'properties': dict({ 'args': dict({ @@ -10291,9 +10274,9 @@ 'description': ''' A chunk of a tool call (e.g., as part of a stream). - When merging ToolCallChunks (e.g., via AIMessageChunk.__add__), + When merging ``ToolCallChunk``s (e.g., via ``AIMessageChunk.__add__``), all string attributes are concatenated. Chunks are only merged if their - values of `index` are equal and not None. + values of ``index`` are equal and not None. Example: @@ -10371,10 +10354,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - ToolMessages contain the result of a tool invocation. Typically, the result - is encoded inside the `content` field. + ``ToolMessage``s contain the result of a tool invocation. Typically, the result + is encoded inside the ``content`` field. - Example: A ToolMessage representing a result of 42 from a tool call with id + Example: A ``ToolMessage`` representing a result of ``42`` from a tool call with id .. code-block:: python @@ -10383,7 +10366,7 @@ ToolMessage(content='42', tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL') - Example: A ToolMessage where only part of the tool output is sent to the model + Example: A ``ToolMessage`` where only part of the tool output is sent to the model and the full output is passed in to artifact. .. versionadded:: 0.2.17 @@ -10404,7 +10387,7 @@ tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL', ) - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -10574,7 +10557,6 @@ This is a standard representation of token usage that is consistent across models. Example: - .. code-block:: python { @@ -11070,10 +11052,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - FunctionMessage are an older version of the ToolMessage schema, and - do not contain the tool_call_id field. + ``FunctionMessage`` are an older version of the ``ToolMessage`` schema, and + do not contain the ``tool_call_id`` field. - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -11203,7 +11185,7 @@ 'description': ''' Message from a human. - HumanMessages are messages that are passed in from a human to the model. + ``HumanMessage``s are messages that are passed in from a human to the model. Example: @@ -11376,7 +11358,6 @@ Does *not* need to sum to full input token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -11410,7 +11391,7 @@ 'description': ''' Allowance for errors made by LLM. - Here we add an `error` key to surface errors made during generation + Here we add an ``error`` key to surface errors made during generation (e.g., invalid JSON arguments.) ''', 'properties': dict({ @@ -11479,7 +11460,6 @@ Does *not* need to sum to full output token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -11707,8 +11687,8 @@ "id": "123" } - This represents a request to call the tool named "foo" with arguments {"a": 1} - and an identifier of "123". + This represents a request to call the tool named ``'foo'`` with arguments + ``{"a": 1}`` and an identifier of ``'123'``. ''', 'properties': dict({ 'args': dict({ @@ -11747,9 +11727,9 @@ 'description': ''' A chunk of a tool call (e.g., as part of a stream). - When merging ToolCallChunks (e.g., via AIMessageChunk.__add__), + When merging ``ToolCallChunk``s (e.g., via ``AIMessageChunk.__add__``), all string attributes are concatenated. Chunks are only merged if their - values of `index` are equal and not None. + values of ``index`` are equal and not None. Example: @@ -11827,10 +11807,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - ToolMessages contain the result of a tool invocation. Typically, the result - is encoded inside the `content` field. + ``ToolMessage``s contain the result of a tool invocation. Typically, the result + is encoded inside the ``content`` field. - Example: A ToolMessage representing a result of 42 from a tool call with id + Example: A ``ToolMessage`` representing a result of ``42`` from a tool call with id .. code-block:: python @@ -11839,7 +11819,7 @@ ToolMessage(content='42', tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL') - Example: A ToolMessage where only part of the tool output is sent to the model + Example: A ``ToolMessage`` where only part of the tool output is sent to the model and the full output is passed in to artifact. .. versionadded:: 0.2.17 @@ -11860,7 +11840,7 @@ tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL', ) - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -12030,7 +12010,6 @@ This is a standard representation of token usage that is consistent across models. Example: - .. code-block:: python { @@ -12538,10 +12517,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - FunctionMessage are an older version of the ToolMessage schema, and - do not contain the tool_call_id field. + ``FunctionMessage`` are an older version of the ``ToolMessage`` schema, and + do not contain the ``tool_call_id`` field. - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -12671,7 +12650,7 @@ 'description': ''' Message from a human. - HumanMessages are messages that are passed in from a human to the model. + ``HumanMessage``s are messages that are passed in from a human to the model. Example: @@ -12844,7 +12823,6 @@ Does *not* need to sum to full input token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -12878,7 +12856,7 @@ 'description': ''' Allowance for errors made by LLM. - Here we add an `error` key to surface errors made during generation + Here we add an ``error`` key to surface errors made during generation (e.g., invalid JSON arguments.) ''', 'properties': dict({ @@ -12947,7 +12925,6 @@ Does *not* need to sum to full output token count. Does *not* need to have all keys. Example: - .. code-block:: python { @@ -13164,8 +13141,8 @@ "id": "123" } - This represents a request to call the tool named "foo" with arguments {"a": 1} - and an identifier of "123". + This represents a request to call the tool named ``'foo'`` with arguments + ``{"a": 1}`` and an identifier of ``'123'``. ''', 'properties': dict({ 'args': dict({ @@ -13204,9 +13181,9 @@ 'description': ''' A chunk of a tool call (e.g., as part of a stream). - When merging ToolCallChunks (e.g., via AIMessageChunk.__add__), + When merging ``ToolCallChunk``s (e.g., via ``AIMessageChunk.__add__``), all string attributes are concatenated. Chunks are only merged if their - values of `index` are equal and not None. + values of ``index`` are equal and not None. Example: @@ -13284,10 +13261,10 @@ 'description': ''' Message for passing the result of executing a tool back to a model. - ToolMessages contain the result of a tool invocation. Typically, the result - is encoded inside the `content` field. + ``ToolMessage``s contain the result of a tool invocation. Typically, the result + is encoded inside the ``content`` field. - Example: A ToolMessage representing a result of 42 from a tool call with id + Example: A ``ToolMessage`` representing a result of ``42`` from a tool call with id .. code-block:: python @@ -13296,7 +13273,7 @@ ToolMessage(content='42', tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL') - Example: A ToolMessage where only part of the tool output is sent to the model + Example: A ``ToolMessage`` where only part of the tool output is sent to the model and the full output is passed in to artifact. .. versionadded:: 0.2.17 @@ -13317,7 +13294,7 @@ tool_call_id='call_Jja7J89XsjrOLA5r!MEOW!SL', ) - The tool_call_id field is used to associate the tool call request with the + The ``tool_call_id`` field is used to associate the tool call request with the tool call response. This is useful in situations where a chat model is able to request multiple tool calls in parallel. ''', @@ -13487,7 +13464,6 @@ This is a standard representation of token usage that is consistent across models. Example: - .. code-block:: python { diff --git a/libs/core/tests/unit_tests/stubs.py b/libs/core/tests/unit_tests/stubs.py index 5cd45afb41f48..57759ec9d47ff 100644 --- a/libs/core/tests/unit_tests/stubs.py +++ b/libs/core/tests/unit_tests/stubs.py @@ -15,34 +15,35 @@ def __eq__(self, other: object) -> bool: # The code below creates version of pydantic models # that will work in unit tests with AnyStr as id field + # Please note that the `id` field is assigned AFTER the model is created # to workaround an issue with pydantic ignoring the __eq__ method on # subclassed strings. def _any_id_document(**kwargs: Any) -> Document: - """Create a document with an id field.""" + """Create a `Document` with an id field.""" message = Document(**kwargs) message.id = AnyStr() return message def _any_id_ai_message(**kwargs: Any) -> AIMessage: - """Create ai message with an any id field.""" + """Create an `AIMessage` with an any id field.""" message = AIMessage(**kwargs) message.id = AnyStr() return message def _any_id_ai_message_chunk(**kwargs: Any) -> AIMessageChunk: - """Create ai message with an any id field.""" + """Create an `AIMessageChunk` with an any id field.""" message = AIMessageChunk(**kwargs) message.id = AnyStr() return message def _any_id_human_message(**kwargs: Any) -> HumanMessage: - """Create a human with an any id field.""" + """Create a `HumanMessage` with an any id field.""" message = HumanMessage(**kwargs) message.id = AnyStr() return message diff --git a/libs/core/tests/unit_tests/test_messages.py b/libs/core/tests/unit_tests/test_messages.py index 807f52ae10d55..3de287a287c1e 100644 --- a/libs/core/tests/unit_tests/test_messages.py +++ b/libs/core/tests/unit_tests/test_messages.py @@ -3,6 +3,7 @@ from typing import Optional, Union import pytest +from typing_extensions import get_args from langchain_core.documents import Document from langchain_core.load import dumpd, load @@ -30,6 +31,7 @@ messages_from_dict, messages_to_dict, ) +from langchain_core.messages.content import KNOWN_BLOCK_TYPES, ContentBlock from langchain_core.messages.tool import invalid_tool_call as create_invalid_tool_call from langchain_core.messages.tool import tool_call as create_tool_call from langchain_core.messages.tool import tool_call_chunk as create_tool_call_chunk @@ -178,21 +180,23 @@ def test_message_chunks() -> None: assert AIMessageChunk(content="") + left == left assert right + AIMessageChunk(content="") == right + default_id = "lc_run--abc123" + meaningful_id = "msg_def456" + # Test ID order of precedence - null_id = AIMessageChunk(content="", id=None) - default_id = AIMessageChunk( - content="", id="run-abc123" + null_id_chunk = AIMessageChunk(content="", id=None) + default_id_chunk = AIMessageChunk( + content="", id=default_id ) # LangChain-assigned run ID - meaningful_id = AIMessageChunk(content="", id="msg_def456") # provider-assigned ID - - assert (null_id + default_id).id == "run-abc123" - assert (default_id + null_id).id == "run-abc123" + provider_chunk = AIMessageChunk( + content="", id=meaningful_id + ) # provided ID (either by user or provider) - assert (null_id + meaningful_id).id == "msg_def456" - assert (meaningful_id + null_id).id == "msg_def456" + assert (null_id_chunk + default_id_chunk).id == default_id + assert (null_id_chunk + provider_chunk).id == meaningful_id - assert (default_id + meaningful_id).id == "msg_def456" - assert (meaningful_id + default_id).id == "msg_def456" + # Provider assigned IDs have highest precedence + assert (default_id_chunk + provider_chunk).id == meaningful_id def test_chat_message_chunks() -> None: @@ -207,7 +211,7 @@ def test_chat_message_chunks() -> None: ): ChatMessageChunk(role="User", content="I am") + ChatMessageChunk( role="Assistant", content=" indeed." - ) + ) # type: ignore[reportUnusedExpression, unused-ignore] assert ChatMessageChunk(role="User", content="I am") + AIMessageChunk( content=" indeed." @@ -316,7 +320,7 @@ def test_function_message_chunks() -> None: ): FunctionMessageChunk(name="hello", content="I am") + FunctionMessageChunk( name="bye", content=" indeed." - ) + ) # type: ignore[reportUnusedExpression, unused-ignore] def test_ai_message_chunks() -> None: @@ -332,7 +336,7 @@ def test_ai_message_chunks() -> None: ): AIMessageChunk(example=True, content="I am") + AIMessageChunk( example=False, content=" indeed." - ) + ) # type: ignore[reportUnusedExpression, unused-ignore] class TestGetBufferString(unittest.TestCase): @@ -1038,12 +1042,13 @@ def test_tool_message_content() -> None: ToolMessage(["foo"], tool_call_id="1") ToolMessage([{"foo": "bar"}], tool_call_id="1") - assert ToolMessage(("a", "b", "c"), tool_call_id="1").content == ["a", "b", "c"] # type: ignore[arg-type] - assert ToolMessage(5, tool_call_id="1").content == "5" # type: ignore[arg-type] - assert ToolMessage(5.1, tool_call_id="1").content == "5.1" # type: ignore[arg-type] - assert ToolMessage({"foo": "bar"}, tool_call_id="1").content == "{'foo': 'bar'}" # type: ignore[arg-type] + # Ignoring since we're testing that tuples get converted to lists in `coerce_args` + assert ToolMessage(("a", "b", "c"), tool_call_id="1").content == ["a", "b", "c"] # type: ignore[call-overload] + assert ToolMessage(5, tool_call_id="1").content == "5" # type: ignore[call-overload] + assert ToolMessage(5.1, tool_call_id="1").content == "5.1" # type: ignore[call-overload] + assert ToolMessage({"foo": "bar"}, tool_call_id="1").content == "{'foo': 'bar'}" # type: ignore[call-overload] assert ( - ToolMessage(Document("foo"), tool_call_id="1").content == "page_content='foo'" # type: ignore[arg-type] + ToolMessage(Document("foo"), tool_call_id="1").content == "page_content='foo'" # type: ignore[call-overload] ) @@ -1113,92 +1118,191 @@ def test_message_text() -> None: def test_is_data_content_block() -> None: + # Test all DataContentBlock types with various data fields + + # Image blocks + assert is_data_content_block({"type": "image", "url": "https://..."}) assert is_data_content_block( - { - "type": "image", - "source_type": "url", - "url": "https://...", - } + {"type": "image", "base64": "", "mime_type": "image/jpeg"} + ) + + # Video blocks + assert is_data_content_block({"type": "video", "url": "https://video.mp4"}) + assert is_data_content_block( + {"type": "video", "base64": "", "mime_type": "video/mp4"} + ) + assert is_data_content_block({"type": "video", "file_id": "vid_123"}) + + # Audio blocks + assert is_data_content_block({"type": "audio", "url": "https://audio.mp3"}) + assert is_data_content_block( + {"type": "audio", "base64": "", "mime_type": "audio/mp3"} + ) + assert is_data_content_block({"type": "audio", "file_id": "aud_123"}) + + # Plain text blocks + assert is_data_content_block({"type": "text-plain", "text": "document content"}) + assert is_data_content_block({"type": "text-plain", "url": "https://doc.txt"}) + assert is_data_content_block({"type": "text-plain", "file_id": "txt_123"}) + + # File blocks + assert is_data_content_block({"type": "file", "url": "https://file.pdf"}) + assert is_data_content_block( + {"type": "file", "base64": "", "mime_type": "application/pdf"} ) + assert is_data_content_block({"type": "file", "file_id": "file_123"}) + + # Blocks with additional metadata (should still be valid) assert is_data_content_block( { "type": "image", - "source_type": "base64", - "data": "", + "base64": "", "mime_type": "image/jpeg", + "cache_control": {"type": "ephemeral"}, } ) assert is_data_content_block( { "type": "image", - "source_type": "base64", - "data": "", + "base64": "", "mime_type": "image/jpeg", - "cache_control": {"type": "ephemeral"}, + "metadata": {"cache_control": {"type": "ephemeral"}}, } ) assert is_data_content_block( { "type": "image", - "source_type": "base64", - "data": "", + "base64": "", "mime_type": "image/jpeg", - "metadata": {"cache_control": {"type": "ephemeral"}}, + "extras": "hi", } ) + # Invalid cases - wrong type + assert not is_data_content_block({"type": "text", "text": "foo"}) assert not is_data_content_block( { - "type": "text", - "text": "foo", - } + "type": "image_url", + "image_url": {"url": "https://..."}, + } # This is OpenAI Chat Completions ) - assert not is_data_content_block( + assert not is_data_content_block({"type": "tool_call", "name": "func", "args": {}}) + assert not is_data_content_block({"type": "invalid", "url": "something"}) + + # Invalid cases - valid type but no data or `source_type` fields + assert not is_data_content_block({"type": "image"}) + assert not is_data_content_block({"type": "video", "mime_type": "video/mp4"}) + assert not is_data_content_block({"type": "audio", "extras": {"key": "value"}}) + + # Invalid cases - valid type but wrong data field name + assert not is_data_content_block({"type": "image", "source": ""}) + assert not is_data_content_block({"type": "video", "data": "video_data"}) + + # Edge cases - empty or missing values + assert not is_data_content_block({}) + assert not is_data_content_block({"url": "https://..."}) # missing type + + +def test_convert_to_openai_image_block() -> None: + for input_block in [ { + "type": "image", + "url": "https://...", + "cache_control": {"type": "ephemeral"}, + }, + { + "type": "image", + "source_type": "url", + "url": "https://...", + "cache_control": {"type": "ephemeral"}, + }, + ]: + expected = { "type": "image_url", "image_url": {"url": "https://..."}, } - ) - assert not is_data_content_block( + result = convert_to_openai_image_block(input_block) + assert result == expected + + for input_block in [ { "type": "image", - "source_type": "base64", - } - ) - assert not is_data_content_block( + "base64": "", + "mime_type": "image/jpeg", + "cache_control": {"type": "ephemeral"}, + }, { "type": "image", - "source": "", + "source_type": "base64", + "data": "", + "mime_type": "image/jpeg", + "cache_control": {"type": "ephemeral"}, + }, + ]: + expected = { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,", + }, } - ) + result = convert_to_openai_image_block(input_block) + assert result == expected -def test_convert_to_openai_image_block() -> None: - input_block = { - "type": "image", - "source_type": "url", - "url": "https://...", - "cache_control": {"type": "ephemeral"}, - } +def test_known_block_types() -> None: expected = { - "type": "image_url", - "image_url": {"url": "https://..."}, - } - result = convert_to_openai_image_block(input_block) - assert result == expected - - input_block = { - "type": "image", - "source_type": "base64", - "data": "", - "mime_type": "image/jpeg", - "cache_control": {"type": "ephemeral"}, + bt + for bt in get_args(ContentBlock) + for bt in get_args(bt.__annotations__["type"]) } + # Normalize any Literal[...] types in block types to their string values. + # This ensures all entries are plain strings, not Literal objects. expected = { - "type": "image_url", - "image_url": { - "url": "data:image/jpeg;base64,", - }, + t + if isinstance(t, str) + else t.__args__[0] + if hasattr(t, "__args__") and len(t.__args__) == 1 + else t + for t in expected } - result = convert_to_openai_image_block(input_block) - assert result == expected + assert expected == KNOWN_BLOCK_TYPES + + +def test_typed_init() -> None: + ai_message = AIMessage(content_blocks=[{"type": "text", "text": "Hello"}]) + assert ai_message.content == [{"type": "text", "text": "Hello"}] + assert ai_message.content_blocks == ai_message.content + + human_message = HumanMessage(content_blocks=[{"type": "text", "text": "Hello"}]) + assert human_message.content == [{"type": "text", "text": "Hello"}] + assert human_message.content_blocks == human_message.content + + system_message = SystemMessage(content_blocks=[{"type": "text", "text": "Hello"}]) + assert system_message.content == [{"type": "text", "text": "Hello"}] + assert system_message.content_blocks == system_message.content + + tool_message = ToolMessage( + content_blocks=[{"type": "text", "text": "Hello"}], + tool_call_id="abc123", + ) + assert tool_message.content == [{"type": "text", "text": "Hello"}] + assert tool_message.content_blocks == tool_message.content + + for message_class in [AIMessage, HumanMessage, SystemMessage]: + message = message_class("Hello") + assert message.content == "Hello" + assert message.content_blocks == [{"type": "text", "text": "Hello"}] + + message = message_class(content="Hello") + assert message.content == "Hello" + assert message.content_blocks == [{"type": "text", "text": "Hello"}] + + # Test we get type errors for malformed blocks (type checker will complain if + # below type-ignores are unused). + _ = AIMessage(content_blocks=[{"type": "text", "bad": "Hello"}]) # type: ignore[list-item] + _ = HumanMessage(content_blocks=[{"type": "text", "bad": "Hello"}]) # type: ignore[list-item] + _ = SystemMessage(content_blocks=[{"type": "text", "bad": "Hello"}]) # type: ignore[list-item] + _ = ToolMessage( + content_blocks=[{"type": "text", "bad": "Hello"}], # type: ignore[list-item] + tool_call_id="abc123", + ) diff --git a/libs/core/tests/unit_tests/test_tools.py b/libs/core/tests/unit_tests/test_tools.py index 72c6a5a387cfb..af0cdec45ce21 100644 --- a/libs/core/tests/unit_tests/test_tools.py +++ b/libs/core/tests/unit_tests/test_tools.py @@ -2281,7 +2281,7 @@ def test_tool_injected_tool_call_id() -> None: @tool def foo(x: int, tool_call_id: Annotated[str, InjectedToolCallId]) -> ToolMessage: """Foo.""" - return ToolMessage(x, tool_call_id=tool_call_id) # type: ignore[arg-type] + return ToolMessage(x, tool_call_id=tool_call_id) # type: ignore[call-overload] assert foo.invoke( { @@ -2290,7 +2290,7 @@ def foo(x: int, tool_call_id: Annotated[str, InjectedToolCallId]) -> ToolMessage "name": "foo", "id": "bar", } - ) == ToolMessage(0, tool_call_id="bar") # type: ignore[arg-type] + ) == ToolMessage(0, tool_call_id="bar") # type: ignore[call-overload] with pytest.raises( ValueError, @@ -2302,7 +2302,7 @@ def foo(x: int, tool_call_id: Annotated[str, InjectedToolCallId]) -> ToolMessage @tool def foo2(x: int, tool_call_id: Annotated[str, InjectedToolCallId()]) -> ToolMessage: """Foo.""" - return ToolMessage(x, tool_call_id=tool_call_id) # type: ignore[arg-type] + return ToolMessage(x, tool_call_id=tool_call_id) # type: ignore[call-overload] assert foo2.invoke( { @@ -2311,14 +2311,14 @@ def foo2(x: int, tool_call_id: Annotated[str, InjectedToolCallId()]) -> ToolMess "name": "foo", "id": "bar", } - ) == ToolMessage(0, tool_call_id="bar") # type: ignore[arg-type] + ) == ToolMessage(0, tool_call_id="bar") # type: ignore[call-overload] def test_tool_uninjected_tool_call_id() -> None: @tool def foo(x: int, tool_call_id: str) -> ToolMessage: """Foo.""" - return ToolMessage(x, tool_call_id=tool_call_id) # type: ignore[arg-type] + return ToolMessage(x, tool_call_id=tool_call_id) # type: ignore[call-overload] with pytest.raises(ValueError, match="1 validation error for foo"): foo.invoke({"type": "tool_call", "args": {"x": 0}, "name": "foo", "id": "bar"}) @@ -2330,7 +2330,7 @@ def foo(x: int, tool_call_id: str) -> ToolMessage: "name": "foo", "id": "bar", } - ) == ToolMessage(0, tool_call_id="zap") # type: ignore[arg-type] + ) == ToolMessage(0, tool_call_id="zap") # type: ignore[call-overload] def test_tool_return_output_mixin() -> None: diff --git a/libs/langchain/langchain/agents/output_parsers/tools.py b/libs/langchain/langchain/agents/output_parsers/tools.py index b7ec8a47aa19a..1cf1ed8680f03 100644 --- a/libs/langchain/langchain/agents/output_parsers/tools.py +++ b/libs/langchain/langchain/agents/output_parsers/tools.py @@ -47,7 +47,12 @@ def parse_ai_message_to_tool_action( try: args = json.loads(function["arguments"] or "{}") tool_calls.append( - ToolCall(name=function_name, args=args, id=tool_call["id"]), + ToolCall( + type="tool_call", + name=function_name, + args=args, + id=tool_call["id"], + ), ) except JSONDecodeError as e: msg = ( diff --git a/libs/langchain/tests/unit_tests/chat_models/test_base.py b/libs/langchain/tests/unit_tests/chat_models/test_base.py index e8e5a43c55076..6c8df01254070 100644 --- a/libs/langchain/tests/unit_tests/chat_models/test_base.py +++ b/libs/langchain/tests/unit_tests/chat_models/test_base.py @@ -277,6 +277,7 @@ def test_configurable_with_default() -> None: "model_kwargs": {}, "streaming": False, "stream_usage": True, + "output_version": "v0", }, "kwargs": { "tools": [{"name": "foo", "description": "foo", "input_schema": {}}], diff --git a/libs/langchain_v1/tests/unit_tests/chat_models/test_chat_models.py b/libs/langchain_v1/tests/unit_tests/chat_models/test_chat_models.py index 3f4a05bcf4493..369b74bc7e507 100644 --- a/libs/langchain_v1/tests/unit_tests/chat_models/test_chat_models.py +++ b/libs/langchain_v1/tests/unit_tests/chat_models/test_chat_models.py @@ -277,6 +277,7 @@ def test_configurable_with_default() -> None: "model_kwargs": {}, "streaming": False, "stream_usage": True, + "output_version": "v0", }, "kwargs": { "tools": [{"name": "foo", "description": "foo", "input_schema": {}}], diff --git a/libs/partners/anthropic/langchain_anthropic/_compat.py b/libs/partners/anthropic/langchain_anthropic/_compat.py new file mode 100644 index 0000000000000..3b90416232406 --- /dev/null +++ b/libs/partners/anthropic/langchain_anthropic/_compat.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +import json +from typing import Any, Optional, cast + +from langchain_core.messages import content as types + + +def _convert_annotation_from_v1(annotation: types.Annotation) -> dict[str, Any]: + """Right-inverse of _convert_citation_to_v1.""" + if annotation["type"] == "non_standard_annotation": + return annotation["value"] + + if annotation["type"] == "citation": + if "url" in annotation: + # web_search_result_location + out: dict[str, Any] = {} + if cited_text := annotation.get("cited_text"): + out["cited_text"] = cited_text + if "encrypted_index" in annotation.get("extras", {}): + out["encrypted_index"] = annotation["extras"]["encrypted_index"] + if "title" in annotation: + out["title"] = annotation["title"] + out["type"] = "web_search_result_location" + if "url" in annotation: + out["url"] = annotation["url"] + + for key, value in annotation.get("extras", {}).items(): + if key not in out: + out[key] = value + + return out + + if "start_char_index" in annotation.get("extras", {}): + # char_location + out = {"type": "char_location"} + for field in ["cited_text"]: + if value := annotation.get(field): + out[field] = value + if title := annotation.get("title"): + out["document_title"] = title + + for key, value in annotation.get("extras", {}).items(): + out[key] = value + + return out + + if "search_result_index" in annotation.get("extras", {}): + # search_result_location + out = {"type": "search_result_location"} + for field in ["cited_text", "title"]: + if value := annotation.get(field): + out[field] = value + + for key, value in annotation.get("extras", {}).items(): + out[key] = value + + return out + + if "start_block_index" in annotation.get("extras", {}): + # content_block_location + out = {} + if cited_text := annotation.get("cited_text"): + out["cited_text"] = cited_text + if "document_index" in annotation.get("extras", {}): + out["document_index"] = annotation["extras"]["document_index"] + if "title" in annotation: + out["document_title"] = annotation["title"] + + for key, value in annotation.get("extras", {}).items(): + if key not in out: + out[key] = value + + out["type"] = "content_block_location" + return out + + if "start_page_number" in annotation.get("extras", {}): + # page_location + out = {"type": "page_location"} + for field in ["cited_text"]: + if value := annotation.get(field): + out[field] = value + if title := annotation.get("title"): + out["document_title"] = title + + for key, value in annotation.get("extras", {}).items(): + out[key] = value + + return out + + return cast(dict[str, Any], annotation) + + return cast(dict[str, Any], annotation) + + +def _convert_from_v1_to_anthropic( + content: list[types.ContentBlock], + tool_calls: list[types.ToolCall], + model_provider: Optional[str], +) -> list[dict[str, Any]]: + new_content: list = [] + for block in content: + if block["type"] == "text": + if model_provider == "anthropic" and "annotations" in block: + new_block: dict[str, Any] = {"type": "text"} + new_block["citations"] = [ + _convert_annotation_from_v1(a) for a in block["annotations"] + ] + if "text" in block: + new_block["text"] = block["text"] + else: + new_block = {"text": block.get("text", ""), "type": "text"} + new_content.append(new_block) + + elif block["type"] == "tool_call": + new_content.append( + { + "type": "tool_use", + "name": block.get("name", ""), + "input": block.get("args", {}), + "id": block.get("id", ""), + } + ) + + elif block["type"] == "tool_call_chunk": + if isinstance(block["args"], str): + try: + input_ = json.loads(block["args"] or "{}") + except json.JSONDecodeError: + input_ = {} + else: + input_ = block.get("args") or {} + new_content.append( + { + "type": "tool_use", + "name": block.get("name", ""), + "input": input_, + "id": block.get("id", ""), + } + ) + + elif block["type"] == "reasoning" and model_provider == "anthropic": + new_block = {} + if "reasoning" in block: + new_block["thinking"] = block["reasoning"] + new_block["type"] = "thinking" + if signature := block.get("extras", {}).get("signature"): + new_block["signature"] = signature + + new_content.append(new_block) + + elif block["type"] == "web_search_call" and model_provider == "anthropic": + new_block = {} + if "id" in block: + new_block["id"] = block["id"] + + if (query := block.get("query")) and "input" not in block: + new_block["input"] = {"query": query} + elif input_ := block.get("extras", {}).get("input"): + new_block["input"] = input_ + elif partial_json := block.get("extras", {}).get("partial_json"): + new_block["input"] = {} + new_block["partial_json"] = partial_json + else: + pass + new_block["name"] = "web_search" + new_block["type"] = "server_tool_use" + new_content.append(new_block) + + elif block["type"] == "web_search_result" and model_provider == "anthropic": + new_block = {} + if "content" in block.get("extras", {}): + new_block["content"] = block["extras"]["content"] + if "id" in block: + new_block["tool_use_id"] = block["id"] + new_block["type"] = "web_search_tool_result" + new_content.append(new_block) + + elif block["type"] == "code_interpreter_call" and model_provider == "anthropic": + new_block = {} + if "id" in block: + new_block["id"] = block["id"] + if (code := block.get("code")) and "input" not in block: + new_block["input"] = {"code": code} + elif input_ := block.get("extras", {}).get("input"): + new_block["input"] = input_ + elif partial_json := block.get("extras", {}).get("partial_json"): + new_block["input"] = {} + new_block["partial_json"] = partial_json + else: + pass + new_block["name"] = "code_execution" + new_block["type"] = "server_tool_use" + new_content.append(new_block) + + elif ( + block["type"] == "code_interpreter_result" and model_provider == "anthropic" + ): + new_block = {} + if (output := block.get("output", [])) and len(output) == 1: + code_interpreter_output = output[0] + code_execution_content = {} + if "content" in block.get("extras", {}): + code_execution_content["content"] = block["extras"]["content"] + elif (file_ids := block.get("file_ids")) and isinstance(file_ids, list): + code_execution_content["content"] = [ + {"file_id": file_id, "type": "code_execution_output"} + for file_id in file_ids + ] + else: + code_execution_content["content"] = [] + if "return_code" in code_interpreter_output: + code_execution_content["return_code"] = code_interpreter_output[ + "return_code" + ] + code_execution_content["stderr"] = code_interpreter_output.get( + "stderr", "" + ) + if "stdout" in code_interpreter_output: + code_execution_content["stdout"] = code_interpreter_output["stdout"] + code_execution_content["type"] = "code_execution_result" + new_block["content"] = code_execution_content + elif "error_code" in block.get("extras", {}): + code_execution_content = { + "error_code": block["extras"]["error_code"], + "type": "code_execution_tool_result_error", + } + new_block["content"] = code_execution_content + else: + pass + if "id" in block: + new_block["tool_use_id"] = block["id"] + new_block["type"] = "code_execution_tool_result" + new_content.append(new_block) + + elif ( + block["type"] == "non_standard" + and "value" in block + and model_provider == "anthropic" + ): + new_content.append(block["value"]) + else: + new_content.append(block) + + return new_content diff --git a/libs/partners/anthropic/langchain_anthropic/chat_models.py b/libs/partners/anthropic/langchain_anthropic/chat_models.py index 641b630ddb35c..18651fb5f2c03 100644 --- a/libs/partners/anthropic/langchain_anthropic/chat_models.py +++ b/libs/partners/anthropic/langchain_anthropic/chat_models.py @@ -33,6 +33,7 @@ ToolMessage, is_data_content_block, ) +from langchain_core.messages import content as types from langchain_core.messages.ai import InputTokenDetails, UsageMetadata from langchain_core.messages.tool import tool_call_chunk as create_tool_call_chunk from langchain_core.output_parsers import JsonOutputKeyToolsParser, PydanticToolsParser @@ -51,6 +52,7 @@ _get_default_async_httpx_client, _get_default_httpx_client, ) +from langchain_anthropic._compat import _convert_from_v1_to_anthropic from langchain_anthropic.output_parsers import extract_tool_calls _message_type_lookups = { @@ -212,7 +214,7 @@ def _merge_messages( def _format_data_content_block(block: dict) -> dict: """Format standard data content block to format expected by Anthropic.""" if block["type"] == "image": - if block["source_type"] == "url": + if "url" in block: if block["url"].startswith("data:"): # Data URI formatted_block = { @@ -224,16 +226,24 @@ def _format_data_content_block(block: dict) -> dict: "type": "image", "source": {"type": "url", "url": block["url"]}, } - elif block["source_type"] == "base64": + elif "base64" in block or block.get("source_type") == "base64": formatted_block = { "type": "image", "source": { "type": "base64", "media_type": block["mime_type"], - "data": block["data"], + "data": block.get("base64") or block.get("data", ""), }, } - elif block["source_type"] == "id": + elif "file_id" in block: + formatted_block = { + "type": "image", + "source": { + "type": "file", + "file_id": block["file_id"], + }, + } + elif block.get("source_type") == "id": formatted_block = { "type": "image", "source": { @@ -243,7 +253,7 @@ def _format_data_content_block(block: dict) -> dict: } else: msg = ( - "Anthropic only supports 'url' and 'base64' source_type for image " + "Anthropic only supports 'url', 'base64', or 'id' keys for image " "content blocks." ) raise ValueError( @@ -251,7 +261,7 @@ def _format_data_content_block(block: dict) -> dict: ) elif block["type"] == "file": - if block["source_type"] == "url": + if "url" in block: formatted_block = { "type": "document", "source": { @@ -259,16 +269,16 @@ def _format_data_content_block(block: dict) -> dict: "url": block["url"], }, } - elif block["source_type"] == "base64": + elif "base64" in block or block.get("source_type") == "base64": formatted_block = { "type": "document", "source": { "type": "base64", "media_type": block.get("mime_type") or "application/pdf", - "data": block["data"], + "data": block.get("base64") or block.get("data", ""), }, } - elif block["source_type"] == "text": + elif block.get("source_type") == "text": formatted_block = { "type": "document", "source": { @@ -277,7 +287,15 @@ def _format_data_content_block(block: dict) -> dict: "data": block["text"], }, } - elif block["source_type"] == "id": + elif "file_id" in block: + formatted_block = { + "type": "document", + "source": { + "type": "file", + "file_id": block["file_id"], + }, + } + elif block.get("source_type") == "id": formatted_block = { "type": "document", "source": { @@ -285,6 +303,22 @@ def _format_data_content_block(block: dict) -> dict: "file_id": block["id"], }, } + else: + msg = ( + "Anthropic only supports 'url', 'base64', or 'id' keys for file " + "content blocks." + ) + raise ValueError(msg) + + elif block["type"] == "text-plain": + formatted_block = { + "type": "document", + "source": { + "type": "text", + "media_type": block.get("mime_type") or "text/plain", + "data": block["text"], + }, + } else: msg = f"Block of type {block['type']} is not supported." @@ -294,7 +328,10 @@ def _format_data_content_block(block: dict) -> dict: for key in ["cache_control", "citations", "title", "context"]: if key in block: formatted_block[key] = block[key] + elif (metadata := block.get("extras")) and key in metadata: + formatted_block[key] = metadata[key] elif (metadata := block.get("metadata")) and key in metadata: + # Backward compat formatted_block[key] = metadata[key] return formatted_block @@ -741,13 +778,11 @@ class Joke(BaseModel): }, { "type": "image", - "source_type": "base64", - "data": image_data, + "base64": image_data, "mime_type": "image/jpeg", }, { "type": "image", - "source_type": "url", "url": image_url, }, ], @@ -781,7 +816,6 @@ class Joke(BaseModel): }, { "type": "image", - "source_type": "id", "id": "file_abc123...", }, ], @@ -810,9 +844,8 @@ class Joke(BaseModel): "Summarize this document.", { "type": "file", - "source_type": "base64", "mime_type": "application/pdf", - "data": data, + "base64": data, }, ] ) @@ -846,7 +879,6 @@ class Joke(BaseModel): }, { "type": "file", - "source_type": "id", "id": "file_abc123...", }, ], @@ -1462,6 +1494,32 @@ def _get_request_payload( **kwargs: dict, ) -> dict: messages = self._convert_input(input_).to_messages() + + for idx, message in enumerate(messages): + # Translate v1 content + if ( + isinstance(message, AIMessage) + and message.response_metadata.get("output_version") == "v1" + ): + tcs: list[types.ToolCall] = [ + { + "type": "tool_call", + "name": tool_call["name"], + "args": tool_call["args"], + "id": tool_call.get("id"), + } + for tool_call in message.tool_calls + ] + messages[idx] = message.model_copy( + update={ + "content": _convert_from_v1_to_anthropic( + cast(list[types.ContentBlock], message.content), + tcs, + message.response_metadata.get("model_provider"), + ) + } + ) + system, formatted_messages = _format_messages(messages) # If cache_control is provided in kwargs, add it to last message @@ -1626,6 +1684,7 @@ def _format_output(self, data: Any, **kwargs: Any) -> ChatResult: llm_output = { k: v for k, v in data_dict.items() if k not in ("content", "role", "type") } + response_metadata = {"model_provider": "anthropic"} if "model" in llm_output and "model_name" not in llm_output: llm_output["model_name"] = llm_output["model"] if ( @@ -1633,15 +1692,18 @@ def _format_output(self, data: Any, **kwargs: Any) -> ChatResult: and content[0]["type"] == "text" and not content[0].get("citations") ): - msg = AIMessage(content=content[0]["text"]) + msg = AIMessage( + content=content[0]["text"], response_metadata=response_metadata + ) elif any(block["type"] == "tool_use" for block in content): tool_calls = extract_tool_calls(content) msg = AIMessage( content=content, tool_calls=tool_calls, + response_metadata=response_metadata, ) else: - msg = AIMessage(content=content) + msg = AIMessage(content=content, response_metadata=response_metadata) msg.usage_metadata = _create_usage_metadata(data.usage) return ChatResult( generations=[ChatGeneration(message=msg)], @@ -2363,7 +2425,7 @@ def _make_message_chunk_from_anthropic_event( elif event.type == "message_delta" and stream_usage: usage_metadata = _create_usage_metadata(event.usage) message_chunk = AIMessageChunk( - content="", + content="" if coerce_content_to_string else [], usage_metadata=usage_metadata, response_metadata={ "stop_reason": event.delta.stop_reason, @@ -2375,6 +2437,8 @@ def _make_message_chunk_from_anthropic_event( else: pass + if message_chunk: + message_chunk.response_metadata["model_provider"] = "anthropic" return message_chunk, block_start_event diff --git a/libs/partners/anthropic/tests/cassettes/test_agent_loop.yaml.gz b/libs/partners/anthropic/tests/cassettes/test_agent_loop.yaml.gz new file mode 100644 index 0000000000000..d53dffb02da7f Binary files /dev/null and b/libs/partners/anthropic/tests/cassettes/test_agent_loop.yaml.gz differ diff --git a/libs/partners/anthropic/tests/cassettes/test_agent_loop_streaming.yaml.gz b/libs/partners/anthropic/tests/cassettes/test_agent_loop_streaming.yaml.gz new file mode 100644 index 0000000000000..8e76e86628fef Binary files /dev/null and b/libs/partners/anthropic/tests/cassettes/test_agent_loop_streaming.yaml.gz differ diff --git a/libs/partners/anthropic/tests/cassettes/test_citations.yaml.gz b/libs/partners/anthropic/tests/cassettes/test_citations.yaml.gz new file mode 100644 index 0000000000000..c704ad451e382 Binary files /dev/null and b/libs/partners/anthropic/tests/cassettes/test_citations.yaml.gz differ diff --git a/libs/partners/anthropic/tests/integration_tests/test_chat_models.py b/libs/partners/anthropic/tests/integration_tests/test_chat_models.py index 3e5dd85762881..6181f008abad1 100644 --- a/libs/partners/anthropic/tests/integration_tests/test_chat_models.py +++ b/libs/partners/anthropic/tests/integration_tests/test_chat_models.py @@ -6,7 +6,7 @@ import json import os from base64 import b64encode -from typing import Optional, cast +from typing import Literal, Optional, cast import httpx import pytest @@ -29,10 +29,11 @@ from pydantic import BaseModel, Field from langchain_anthropic import ChatAnthropic, ChatAnthropicMessages +from langchain_anthropic._compat import _convert_from_v1_to_anthropic from tests.unit_tests._utils import FakeCallbackHandler -MODEL_NAME = "claude-opus-4-1-20250805" -IMAGE_MODEL_NAME = "claude-opus-4-1-20250805" +MODEL_NAME = "claude-3-5-haiku-latest" +IMAGE_MODEL_NAME = "claude-3-5-haiku-latest" def test_stream() -> None: @@ -65,6 +66,9 @@ def test_stream() -> None: assert chunks_with_model_name == 1 # check token usage is populated assert isinstance(full, AIMessageChunk) + assert len(full.content_blocks) == 1 + assert full.content_blocks[0]["type"] == "text" + assert full.content_blocks[0]["text"] assert full.usage_metadata is not None assert full.usage_metadata["input_tokens"] > 0 assert full.usage_metadata["output_tokens"] > 0 @@ -105,6 +109,9 @@ async def test_astream() -> None: ) # check token usage is populated assert isinstance(full, AIMessageChunk) + assert len(full.content_blocks) == 1 + assert full.content_blocks[0]["type"] == "text" + assert full.content_blocks[0]["text"] assert full.usage_metadata is not None assert full.usage_metadata["input_tokens"] > 0 assert full.usage_metadata["output_tokens"] > 0 @@ -421,6 +428,14 @@ def test_tool_use() -> None: assert isinstance(tool_call["args"], dict) assert "location" in tool_call["args"] + content_blocks = response.content_blocks + assert len(content_blocks) == 2 + assert content_blocks[0]["type"] == "text" + assert content_blocks[0]["text"] + assert content_blocks[1]["type"] == "tool_call" + assert content_blocks[1]["name"] == "get_weather" + assert content_blocks[1]["args"] == tool_call["args"] + # Test streaming llm = ChatAnthropic( model="claude-3-7-sonnet-20250219", # type: ignore[call-arg] @@ -440,6 +455,8 @@ def test_tool_use() -> None: first = False else: gathered = gathered + chunk # type: ignore[assignment] + for block in chunk.content_blocks: + assert block["type"] in ("text", "tool_call_chunk") assert len(chunks) > 1 assert isinstance(gathered.content, list) assert len(gathered.content) == 2 @@ -461,6 +478,14 @@ def test_tool_use() -> None: assert "location" in tool_call["args"] assert tool_call["id"] is not None + content_blocks = gathered.content_blocks + assert len(content_blocks) == 2 + assert content_blocks[0]["type"] == "text" + assert content_blocks[0]["text"] + assert content_blocks[1]["type"] == "tool_call_chunk" + assert content_blocks[1]["name"] == "get_weather" + assert content_blocks[1]["args"] + # Testing token-efficient tools # https://docs.anthropic.com/en/docs/build-with-claude/tool-use/token-efficient-tool-use assert gathered.usage_metadata @@ -500,6 +525,13 @@ def test_builtin_tools() -> None: assert isinstance(response, AIMessage) assert response.tool_calls + content_blocks = response.content_blocks + assert len(content_blocks) == 2 + assert content_blocks[0]["type"] == "text" + assert content_blocks[0]["text"] + assert content_blocks[1]["type"] == "tool_call" + assert content_blocks[1]["name"] == "str_replace_editor" + class GenerateUsername(BaseModel): """Get a username based on someone's name and hair color.""" @@ -682,8 +714,74 @@ def test_pdf_document_input() -> None: assert len(result.content) > 0 -def test_citations() -> None: - llm = ChatAnthropic(model="claude-3-5-haiku-latest") # type: ignore[call-arg] +@pytest.mark.default_cassette("test_agent_loop.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_agent_loop(output_version: Literal["v0", "v1"]) -> None: + @tool + def get_weather(location: str) -> str: + """Get the weather for a location.""" + return "It's sunny." + + llm = ChatAnthropic(model="claude-3-5-haiku-latest", output_version=output_version) # type: ignore[call-arg] + llm_with_tools = llm.bind_tools([get_weather]) + input_message = HumanMessage("What is the weather in San Francisco, CA?") + tool_call_message = llm_with_tools.invoke([input_message]) + assert isinstance(tool_call_message, AIMessage) + tool_calls = tool_call_message.tool_calls + assert len(tool_calls) == 1 + tool_call = tool_calls[0] + tool_message = get_weather.invoke(tool_call) + assert isinstance(tool_message, ToolMessage) + response = llm_with_tools.invoke( + [ + input_message, + tool_call_message, + tool_message, + ] + ) + assert isinstance(response, AIMessage) + + +@pytest.mark.default_cassette("test_agent_loop_streaming.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_agent_loop_streaming(output_version: Literal["v0", "v1"]) -> None: + @tool + def get_weather(location: str) -> str: + """Get the weather for a location.""" + return "It's sunny." + + llm = ChatAnthropic( + model="claude-3-5-haiku-latest", + streaming=True, + output_version=output_version, # type: ignore[call-arg] + ) + llm_with_tools = llm.bind_tools([get_weather]) + input_message = HumanMessage("What is the weather in San Francisco, CA?") + tool_call_message = llm_with_tools.invoke([input_message]) + assert isinstance(tool_call_message, AIMessage) + + tool_calls = tool_call_message.tool_calls + assert len(tool_calls) == 1 + tool_call = tool_calls[0] + tool_message = get_weather.invoke(tool_call) + assert isinstance(tool_message, ToolMessage) + response = llm_with_tools.invoke( + [ + input_message, + tool_call_message, + tool_message, + ] + ) + assert isinstance(response, AIMessage) + + +@pytest.mark.default_cassette("test_citations.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_citations(output_version: Literal["v0", "v1"]) -> None: + llm = ChatAnthropic(model="claude-3-5-haiku-latest", output_version=output_version) # type: ignore[call-arg] messages = [ { "role": "user", @@ -706,7 +804,10 @@ def test_citations() -> None: response = llm.invoke(messages) assert isinstance(response, AIMessage) assert isinstance(response.content, list) - assert any("citations" in block for block in response.content) + if output_version == "v1": + assert any("annotations" in block for block in response.content) + else: + assert any("citations" in block for block in response.content) # Test streaming full: Optional[BaseMessageChunk] = None @@ -714,8 +815,11 @@ def test_citations() -> None: full = cast(BaseMessageChunk, chunk) if full is None else full + chunk assert isinstance(full, AIMessageChunk) assert isinstance(full.content, list) - assert any("citations" in block for block in full.content) assert not any("citation" in block for block in full.content) + if output_version == "v1": + assert any("annotations" in block for block in full.content) + else: + assert any("citations" in block for block in full.content) # Test pass back in next_message = { @@ -762,25 +866,77 @@ def test_thinking() -> None: _ = llm.invoke([input_message, full, next_message]) +@pytest.mark.default_cassette("test_thinking.yaml.gz") @pytest.mark.vcr -def test_redacted_thinking() -> None: +def test_thinking_v1() -> None: llm = ChatAnthropic( model="claude-3-7-sonnet-latest", # type: ignore[call-arg] max_tokens=5_000, # type: ignore[call-arg] thinking={"type": "enabled", "budget_tokens": 2_000}, + output_version="v1", + ) + + input_message = {"role": "user", "content": "Hello"} + response = llm.invoke([input_message]) + assert any("reasoning" in block for block in response.content) + for block in response.content: + assert isinstance(block, dict) + if block["type"] == "reasoning": + assert set(block.keys()) == {"type", "reasoning", "extras"} + assert block["reasoning"] and isinstance(block["reasoning"], str) + signature = block["extras"]["signature"] + assert signature and isinstance(signature, str) + + # Test streaming + full: Optional[BaseMessageChunk] = None + for chunk in llm.stream([input_message]): + full = cast(BaseMessageChunk, chunk) if full is None else full + chunk + assert isinstance(full, AIMessageChunk) + assert isinstance(full.content, list) + assert any("reasoning" in block for block in full.content) + for block in full.content: + assert isinstance(block, dict) + if block["type"] == "reasoning": + assert set(block.keys()) == {"type", "reasoning", "extras", "index"} + assert block["reasoning"] and isinstance(block["reasoning"], str) + signature = block["extras"]["signature"] + assert signature and isinstance(signature, str) + + # Test pass back in + next_message = {"role": "user", "content": "How are you?"} + _ = llm.invoke([input_message, full, next_message]) + + +@pytest.mark.default_cassette("test_redacted_thinking.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_redacted_thinking(output_version: Literal["v0", "v1"]) -> None: + llm = ChatAnthropic( + model="claude-3-7-sonnet-latest", # type: ignore[call-arg] + max_tokens=5_000, # type: ignore[call-arg] + thinking={"type": "enabled", "budget_tokens": 2_000}, + output_version=output_version, ) query = "ANTHROPIC_MAGIC_STRING_TRIGGER_REDACTED_THINKING_46C9A13E193C177646C7398A98432ECCCE4C1253D5E2D82641AC0E52CC2876CB" # noqa: E501 input_message = {"role": "user", "content": query} response = llm.invoke([input_message]) - has_reasoning = False + value = None for block in response.content: assert isinstance(block, dict) if block["type"] == "redacted_thinking": - has_reasoning = True - assert set(block.keys()) == {"type", "data"} - assert block["data"] and isinstance(block["data"], str) - assert has_reasoning + value = block + elif ( + block["type"] == "non_standard" + and block["value"]["type"] == "redacted_thinking" + ): + value = block["value"] + else: + pass + if value: + assert set(value.keys()) == {"type", "data"} + assert value["data"] and isinstance(value["data"], str) + assert value is not None # Test streaming full: Optional[BaseMessageChunk] = None @@ -788,14 +944,25 @@ def test_redacted_thinking() -> None: full = cast(BaseMessageChunk, chunk) if full is None else full + chunk assert isinstance(full, AIMessageChunk) assert isinstance(full.content, list) - stream_has_reasoning = False + value = None for block in full.content: assert isinstance(block, dict) if block["type"] == "redacted_thinking": - stream_has_reasoning = True - assert set(block.keys()) == {"type", "data", "index"} - assert block["data"] and isinstance(block["data"], str) - assert stream_has_reasoning + value = block + assert set(value.keys()) == {"type", "data", "index"} + assert "index" in block + elif ( + block["type"] == "non_standard" + and block["value"]["type"] == "redacted_thinking" + ): + value = block["value"] + assert set(value.keys()) == {"type", "data"} + assert "index" in block + else: + pass + if value: + assert value["data"] and isinstance(value["data"], str) + assert value is not None # Test pass back in next_message = {"role": "user", "content": "What?"} @@ -899,11 +1066,14 @@ class color_picker(BaseModel): llm.bind_tools([color_picker]).invoke(messages) +@pytest.mark.default_cassette("test_web_search.yaml.gz") @pytest.mark.vcr -def test_web_search() -> None: +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_web_search(output_version: Literal["v0", "v1"]) -> None: llm = ChatAnthropic( model="claude-3-5-sonnet-latest", # type: ignore[call-arg] max_tokens=1024, + output_version=output_version, ) tool = {"type": "web_search_20250305", "name": "web_search", "max_uses": 1} @@ -921,7 +1091,10 @@ def test_web_search() -> None: response = llm_with_tools.invoke([input_message]) assert all(isinstance(block, dict) for block in response.content) block_types = {block["type"] for block in response.content} # type: ignore[index] - assert block_types == {"text", "server_tool_use", "web_search_tool_result"} + if output_version == "v0": + assert block_types == {"text", "server_tool_use", "web_search_tool_result"} + else: + assert block_types == {"text", "web_search_call", "web_search_result"} # Test streaming full: Optional[BaseMessageChunk] = None @@ -931,7 +1104,10 @@ def test_web_search() -> None: assert isinstance(full, AIMessageChunk) assert isinstance(full.content, list) block_types = {block["type"] for block in full.content} # type: ignore[index] - assert block_types == {"text", "server_tool_use", "web_search_tool_result"} + if output_version == "v0": + assert block_types == {"text", "server_tool_use", "web_search_tool_result"} + else: + assert block_types == {"text", "web_search_call", "web_search_result"} # Test we can pass back in next_message = { @@ -943,12 +1119,15 @@ def test_web_search() -> None: ) +@pytest.mark.default_cassette("test_code_execution.yaml.gz") @pytest.mark.vcr -def test_code_execution() -> None: +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_code_execution(output_version: Literal["v0", "v1"]) -> None: llm = ChatAnthropic( model="claude-sonnet-4-20250514", # type: ignore[call-arg] betas=["code-execution-2025-05-22"], max_tokens=10_000, # type: ignore[call-arg] + output_version=output_version, ) tool = {"type": "code_execution_20250522", "name": "code_execution"} @@ -969,7 +1148,14 @@ def test_code_execution() -> None: response = llm_with_tools.invoke([input_message]) assert all(isinstance(block, dict) for block in response.content) block_types = {block["type"] for block in response.content} # type: ignore[index] - assert block_types == {"text", "server_tool_use", "code_execution_tool_result"} + if output_version == "v0": + assert block_types == {"text", "server_tool_use", "code_execution_tool_result"} + else: + assert block_types == { + "text", + "code_interpreter_call", + "code_interpreter_result", + } # Test streaming full: Optional[BaseMessageChunk] = None @@ -979,7 +1165,14 @@ def test_code_execution() -> None: assert isinstance(full, AIMessageChunk) assert isinstance(full.content, list) block_types = {block["type"] for block in full.content} # type: ignore[index] - assert block_types == {"text", "server_tool_use", "code_execution_tool_result"} + if output_version == "v0": + assert block_types == {"text", "server_tool_use", "code_execution_tool_result"} + else: + assert block_types == { + "text", + "code_interpreter_call", + "code_interpreter_result", + } # Test we can pass back in next_message = { @@ -991,8 +1184,10 @@ def test_code_execution() -> None: ) +@pytest.mark.default_cassette("test_remote_mcp.yaml.gz") @pytest.mark.vcr -def test_remote_mcp() -> None: +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_remote_mcp(output_version: Literal["v0", "v1"]) -> None: mcp_servers = [ { "type": "url", @@ -1008,6 +1203,7 @@ def test_remote_mcp() -> None: betas=["mcp-client-2025-04-04"], mcp_servers=mcp_servers, max_tokens=10_000, # type: ignore[call-arg] + output_version=output_version, ) input_message = { @@ -1025,7 +1221,10 @@ def test_remote_mcp() -> None: response = llm.invoke([input_message]) assert all(isinstance(block, dict) for block in response.content) block_types = {block["type"] for block in response.content} # type: ignore[index] - assert block_types == {"text", "mcp_tool_use", "mcp_tool_result"} + if output_version == "v0": + assert block_types == {"text", "mcp_tool_use", "mcp_tool_result"} + else: + assert block_types == {"text", "non_standard"} # Test streaming full: Optional[BaseMessageChunk] = None @@ -1036,7 +1235,10 @@ def test_remote_mcp() -> None: assert isinstance(full.content, list) assert all(isinstance(block, dict) for block in full.content) block_types = {block["type"] for block in full.content} # type: ignore[index] - assert block_types == {"text", "mcp_tool_use", "mcp_tool_result"} + if output_version == "v0": + assert block_types == {"text", "mcp_tool_use", "mcp_tool_result"} + else: + assert block_types == {"text", "non_standard"} # Test we can pass back in next_message = { @@ -1069,8 +1271,7 @@ def test_files_api_image(block_format: str) -> None: # standard block format block = { "type": "image", - "source_type": "id", - "id": image_file_id, + "file_id": image_file_id, } input_message = { "role": "user", @@ -1097,8 +1298,7 @@ def test_files_api_pdf(block_format: str) -> None: # standard block format block = { "type": "file", - "source_type": "id", - "id": pdf_file_id, + "file_id": pdf_file_id, } input_message = { "role": "user", @@ -1163,6 +1363,11 @@ def retrieval_tool(query: str) -> list[dict]: assert isinstance(result.content, list) assert any("citations" in block for block in result.content) + assert ( + _convert_from_v1_to_anthropic(result.content_blocks, [], "anthropic") + == result.content + ) + def test_search_result_top_level() -> None: llm = ChatAnthropic( @@ -1209,6 +1414,11 @@ def test_search_result_top_level() -> None: assert isinstance(result.content, list) assert any("citations" in block for block in result.content) + assert ( + _convert_from_v1_to_anthropic(result.content_blocks, [], "anthropic") + == result.content + ) + def test_async_shared_client() -> None: llm = ChatAnthropic(model="claude-3-5-haiku-latest") # type: ignore[call-arg] diff --git a/libs/partners/anthropic/tests/unit_tests/__snapshots__/test_standard.ambr b/libs/partners/anthropic/tests/unit_tests/__snapshots__/test_standard.ambr index b831aef469b44..5c9164caae365 100644 --- a/libs/partners/anthropic/tests/unit_tests/__snapshots__/test_standard.ambr +++ b/libs/partners/anthropic/tests/unit_tests/__snapshots__/test_standard.ambr @@ -20,6 +20,7 @@ 'max_retries': 2, 'max_tokens': 100, 'model': 'claude-3-haiku-20240307', + 'output_version': 'v0', 'stop_sequences': list([ ]), 'stream_usage': True, diff --git a/libs/partners/anthropic/tests/unit_tests/test_chat_models.py b/libs/partners/anthropic/tests/unit_tests/test_chat_models.py index 382d2f774c5dc..3cf2b0e44ee27 100644 --- a/libs/partners/anthropic/tests/unit_tests/test_chat_models.py +++ b/libs/partners/anthropic/tests/unit_tests/test_chat_models.py @@ -211,6 +211,7 @@ def test__format_output() -> None: "total_tokens": 3, "input_token_details": {}, }, + response_metadata={"model_provider": "anthropic"}, ) llm = ChatAnthropic(model="test", anthropic_api_key="test") # type: ignore[call-arg, call-arg] actual = llm._format_output(anthropic_msg) @@ -241,6 +242,7 @@ def test__format_output_cached() -> None: "total_tokens": 10, "input_token_details": {"cache_creation": 3, "cache_read": 4}, }, + response_metadata={"model_provider": "anthropic"}, ) llm = ChatAnthropic(model="test", anthropic_api_key="test") # type: ignore[call-arg, call-arg] @@ -849,7 +851,7 @@ def test__format_messages_with_cache_control() -> None: assert expected_system == actual_system assert expected_messages == actual_messages - # Test standard multi-modal format + # Test standard multi-modal format (v0) messages = [ HumanMessage( [ @@ -891,6 +893,183 @@ def test__format_messages_with_cache_control() -> None: ] assert actual_messages == expected_messages + # Test standard multi-modal format (v1) + messages = [ + HumanMessage( + [ + { + "type": "text", + "text": "Summarize this document:", + }, + { + "type": "file", + "mime_type": "application/pdf", + "base64": "", + "extras": {"cache_control": {"type": "ephemeral"}}, + }, + ], + ), + ] + actual_system, actual_messages = _format_messages(messages) + assert actual_system is None + expected_messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Summarize this document:", + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "", + }, + "cache_control": {"type": "ephemeral"}, + }, + ], + }, + ] + assert actual_messages == expected_messages + + # Test standard multi-modal format (v1, unpacked extras) + messages = [ + HumanMessage( + [ + { + "type": "text", + "text": "Summarize this document:", + }, + { + "type": "file", + "mime_type": "application/pdf", + "base64": "", + "cache_control": {"type": "ephemeral"}, + }, + ], + ), + ] + actual_system, actual_messages = _format_messages(messages) + assert actual_system is None + expected_messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Summarize this document:", + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "", + }, + "cache_control": {"type": "ephemeral"}, + }, + ], + }, + ] + assert actual_messages == expected_messages + + # Also test file inputs + ## Images + for block in [ + # v1 + { + "type": "image", + "file_id": "abc123", + }, + # v0 + { + "type": "image", + "source_type": "id", + "id": "abc123", + }, + ]: + messages = [ + HumanMessage( + [ + { + "type": "text", + "text": "Summarize this image:", + }, + block, + ], + ), + ] + actual_system, actual_messages = _format_messages(messages) + assert actual_system is None + expected_messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Summarize this image:", + }, + { + "type": "image", + "source": { + "type": "file", + "file_id": "abc123", + }, + }, + ], + }, + ] + assert actual_messages == expected_messages + + ## Documents + for block in [ + # v1 + { + "type": "file", + "file_id": "abc123", + }, + # v0 + { + "type": "file", + "source_type": "id", + "id": "abc123", + }, + ]: + messages = [ + HumanMessage( + [ + { + "type": "text", + "text": "Summarize this document:", + }, + block, + ], + ), + ] + actual_system, actual_messages = _format_messages(messages) + assert actual_system is None + expected_messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Summarize this document:", + }, + { + "type": "document", + "source": { + "type": "file", + "file_id": "abc123", + }, + }, + ], + }, + ] + assert actual_messages == expected_messages + def test__format_messages_with_citations() -> None: input_messages = [ diff --git a/libs/partners/openai/langchain_openai/chat_models/_compat.py b/libs/partners/openai/langchain_openai/chat_models/_compat.py index 25ff3eb607cc3..eb9ce3f40d464 100644 --- a/libs/partners/openai/langchain_openai/chat_models/_compat.py +++ b/libs/partners/openai/langchain_openai/chat_models/_compat.py @@ -1,7 +1,10 @@ """ -This module converts between AIMessage output formats for the Responses API. +This module converts between AIMessage output formats, which are governed by the +``output_version`` attribute on ChatOpenAI. Supported values are ``"v0"`` and +``"responses/v1"``. -ChatOpenAI v0.3 stores reasoning and tool outputs in AIMessage.additional_kwargs: +``"v0"`` corresponds to the format as of ChatOpenAI v0.3. For the Responses API, it +stores reasoning and tool outputs in AIMessage.additional_kwargs: .. code-block:: python @@ -28,8 +31,9 @@ id="msg_123", ) -To retain information about response item sequencing (and to accommodate multiple -reasoning items), ChatOpenAI now stores these items in the content sequence: +``"responses/v1"`` is only applicable to the Responses API. It retains information +about response item sequencing and accommodates multiple reasoning items by +representing these items in the content sequence: .. code-block:: python @@ -57,18 +61,20 @@ content blocks, rather than on the AIMessage.id, which now stores the response ID. For backwards compatibility, this module provides functions to convert between the -old and new formats. The functions are used internally by ChatOpenAI. - +formats. The functions are used internally by ChatOpenAI. """ # noqa: E501 import json -from typing import Union +from collections.abc import Iterable, Iterator +from typing import Any, Literal, Union, cast -from langchain_core.messages import AIMessage +from langchain_core.messages import AIMessage, is_data_content_block +from langchain_core.messages import content as types _FUNCTION_CALL_IDS_MAP_KEY = "__openai_function_call_ids__" +# v0.3 / Responses def _convert_to_v03_ai_message( message: AIMessage, has_reasoning: bool = False ) -> AIMessage: @@ -253,3 +259,241 @@ def _convert_from_v03_ai_message(message: AIMessage) -> AIMessage: }, deep=False, ) + + +# v1 / Chat Completions +def _convert_from_v1_to_chat_completions(message: AIMessage) -> AIMessage: + """Convert a v1 message to the Chat Completions format.""" + if isinstance(message.content, list): + new_content: list = [] + for block in message.content: + if isinstance(block, dict): + block_type = block.get("type") + if block_type == "text": + # Strip annotations + new_content.append({"type": "text", "text": block["text"]}) + elif block_type in ("reasoning", "tool_call"): + pass + else: + new_content.append(block) + else: + new_content.append(block) + return message.model_copy(update={"content": new_content}) + + return message + + +# v1 / Responses +def _convert_annotation_from_v1(annotation: types.Annotation) -> dict[str, Any]: + if annotation["type"] == "citation": + new_ann: dict[str, Any] = {} + for field in ("end_index", "start_index"): + if field in annotation: + new_ann[field] = annotation[field] + + if "url" in annotation: + # URL citation + if "title" in annotation: + new_ann["title"] = annotation["title"] + new_ann["type"] = "url_citation" + new_ann["url"] = annotation["url"] + else: + # Document citation + new_ann["type"] = "file_citation" + if "title" in annotation: + new_ann["filename"] = annotation["title"] + + if extra_fields := annotation.get("extras"): + for field, value in extra_fields.items(): + new_ann[field] = value + + return new_ann + + elif annotation["type"] == "non_standard_annotation": + return annotation["value"] + + else: + return dict(annotation) + + +def _implode_reasoning_blocks(blocks: list[dict[str, Any]]) -> Iterable[dict[str, Any]]: + i = 0 + n = len(blocks) + + while i < n: + block = blocks[i] + + # Skip non-reasoning blocks or blocks already in Responses format + if block.get("type") != "reasoning" or "summary" in block: + yield dict(block) + i += 1 + continue + elif "reasoning" not in block and "summary" not in block: + # {"type": "reasoning", "id": "rs_..."} + oai_format = {**block, "summary": []} + if "extras" in oai_format: + oai_format.update(oai_format.pop("extras")) + oai_format["type"] = oai_format.pop("type", "reasoning") + if "encrypted_content" in oai_format: + oai_format["encrypted_content"] = oai_format.pop("encrypted_content") + yield oai_format + i += 1 + continue + else: + pass + + summary: list[dict[str, str]] = [ + {"type": "summary_text", "text": block.get("reasoning", "")} + ] + # 'common' is every field except the exploded 'reasoning' + common = {k: v for k, v in block.items() if k != "reasoning"} + if "extras" in common: + common.update(common.pop("extras")) + + i += 1 + while i < n: + next_ = blocks[i] + if next_.get("type") == "reasoning" and "reasoning" in next_: + summary.append( + {"type": "summary_text", "text": next_.get("reasoning", "")} + ) + i += 1 + else: + break + + merged = dict(common) + merged["summary"] = summary + merged["type"] = merged.pop("type", "reasoning") + yield merged + + +def _consolidate_calls( + items: Iterable[dict[str, Any]], + call_name: Literal["web_search_call", "code_interpreter_call"], + result_name: Literal["web_search_result", "code_interpreter_result"], +) -> Iterator[dict[str, Any]]: + """ + Generator that walks through *items* and, whenever it meets the pair + + {"type": "web_search_call", "id": X, ...} + {"type": "web_search_result", "id": X} + + merges them into + + {"id": X, + "action": …, + "status": …, + "type": "web_search_call"} + + keeping every other element untouched. + """ + items = iter(items) # make sure we have a true iterator + for current in items: + # Only a call can start a pair worth collapsing + if current.get("type") != call_name: + yield current + continue + + try: + nxt = next(items) # look-ahead one element + except StopIteration: # no “result” – just yield the call back + yield current + break + + # If this really is the matching “result” – collapse + if nxt.get("type") == result_name and nxt.get("id") == current.get("id"): + if call_name == "web_search_call": + collapsed = {"id": current["id"]} + if "action" in current: + collapsed["action"] = current["action"] + collapsed["status"] = current["status"] + collapsed["type"] = "web_search_call" + + if call_name == "code_interpreter_call": + collapsed = {"id": current["id"]} + for key in ("code", "container_id"): + if key in current: + collapsed[key] = current[key] + elif key in current.get("extras", {}): + collapsed[key] = current["extras"][key] + else: + pass + + for key in ("outputs", "status"): + if key in nxt: + collapsed[key] = nxt[key] + elif key in nxt.get("extras", {}): + collapsed[key] = nxt["extras"][key] + else: + pass + collapsed["type"] = "code_interpreter_call" + + yield collapsed + + else: + # Not a matching pair – emit both, in original order + yield current + yield nxt + + +def _convert_from_v1_to_responses( + content: list[types.ContentBlock], tool_calls: list[types.ToolCall] +) -> list[dict[str, Any]]: + new_content: list = [] + for block in content: + if block["type"] == "text" and "annotations" in block: + # Need a copy because we’re changing the annotations list + new_block = dict(block) + new_block["annotations"] = [ + _convert_annotation_from_v1(a) for a in block["annotations"] + ] + new_content.append(new_block) + elif block["type"] == "tool_call": + new_block = {"type": "function_call", "call_id": block["id"]} + if "extras" in block and "item_id" in block["extras"]: + new_block["id"] = block["extras"]["item_id"] + if "name" in block: + new_block["name"] = block["name"] + if "extras" in block and "arguments" in block["extras"]: + new_block["arguments"] = block["extras"]["arguments"] + if any(key not in block for key in ("name", "arguments")): + matching_tool_calls = [ + call for call in tool_calls if call["id"] == block["id"] + ] + if matching_tool_calls: + tool_call = matching_tool_calls[0] + if "name" not in block: + new_block["name"] = tool_call["name"] + if "arguments" not in block: + new_block["arguments"] = json.dumps(tool_call["args"]) + new_content.append(new_block) + elif ( + is_data_content_block(cast(dict, block)) + and block["type"] == "image" + and "base64" in block + and isinstance(block.get("id"), str) + and block["id"].startswith("ig_") + ): + new_block = {"type": "image_generation_call", "result": block["base64"]} + for extra_key in ("id", "status"): + if extra_key in block: + new_block[extra_key] = block[extra_key] # type: ignore[typeddict-item] + elif extra_key in block.get("extras", {}): + new_block[extra_key] = block["extras"][extra_key] + new_content.append(new_block) + elif block["type"] == "non_standard" and "value" in block: + new_content.append(block["value"]) + else: + new_content.append(block) + + new_content = list(_implode_reasoning_blocks(new_content)) + new_content = list( + _consolidate_calls(new_content, "web_search_call", "web_search_result") + ) + new_content = list( + _consolidate_calls( + new_content, "code_interpreter_call", "code_interpreter_result" + ) + ) + + return new_content diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py index 95688bd87ce84..e409713b54d98 100644 --- a/libs/partners/openai/langchain_openai/chat_models/base.py +++ b/libs/partners/openai/langchain_openai/chat_models/base.py @@ -64,6 +64,7 @@ convert_to_openai_data_block, is_data_content_block, ) +from langchain_core.messages import content as types from langchain_core.messages.ai import ( InputTokenDetails, OutputTokenDetails, @@ -108,6 +109,8 @@ ) from langchain_openai.chat_models._compat import ( _convert_from_v03_ai_message, + _convert_from_v1_to_chat_completions, + _convert_from_v1_to_responses, _convert_to_v03_ai_message, ) @@ -202,7 +205,7 @@ def _convert_dict_to_message(_dict: Mapping[str, Any]) -> BaseMessage: return ChatMessage(content=_dict.get("content", ""), role=role, id=id_) # type: ignore[arg-type] -def _format_message_content(content: Any) -> Any: +def _format_message_content(content: Any, responses_ai_msg: bool = False) -> Any: """Format message content.""" if content and isinstance(content, list): formatted_content = [] @@ -214,7 +217,13 @@ def _format_message_content(content: Any) -> Any: and block["type"] in ("tool_use", "thinking", "reasoning_content") ): continue - elif isinstance(block, dict) and is_data_content_block(block): + elif ( + isinstance(block, dict) + and is_data_content_block(block) + # Responses API messages handled separately in _compat (parsed into + # image generation calls) + and not responses_ai_msg + ): formatted_content.append(convert_to_openai_data_block(block)) # Anthropic image blocks elif ( @@ -247,7 +256,9 @@ def _format_message_content(content: Any) -> Any: return formatted_content -def _convert_message_to_dict(message: BaseMessage) -> dict: +def _convert_message_to_dict( + message: BaseMessage, responses_ai_msg: bool = False +) -> dict: """Convert a LangChain message to a dictionary. Args: @@ -256,7 +267,11 @@ def _convert_message_to_dict(message: BaseMessage) -> dict: Returns: The dictionary. """ - message_dict: dict[str, Any] = {"content": _format_message_content(message.content)} + message_dict: dict[str, Any] = { + "content": _format_message_content( + message.content, responses_ai_msg=responses_ai_msg + ) + } if (name := message.name or message.additional_kwargs.get("name")) is not None: message_dict["name"] = name @@ -291,15 +306,25 @@ def _convert_message_to_dict(message: BaseMessage) -> dict: if "function_call" in message_dict or "tool_calls" in message_dict: message_dict["content"] = message_dict["content"] or None - if "audio" in message.additional_kwargs: - # openai doesn't support passing the data back - only the id - # https://platform.openai.com/docs/guides/audio/multi-turn-conversations + audio: Optional[dict[str, Any]] = None + for block in message.content: + if ( + isinstance(block, dict) + and block.get("type") == "audio" + and (id_ := block.get("id")) + and not responses_ai_msg + ): + # openai doesn't support passing the data back - only the id + # https://platform.openai.com/docs/guides/audio/multi-turn-conversations + audio = {"id": id_} + if not audio and "audio" in message.additional_kwargs: raw_audio = message.additional_kwargs["audio"] audio = ( {"id": message.additional_kwargs["audio"]["id"]} if "id" in raw_audio else raw_audio ) + if audio: message_dict["audio"] = audio elif isinstance(message, SystemMessage): message_dict["role"] = message.additional_kwargs.get( @@ -681,7 +706,7 @@ class BaseChatOpenAI(BaseChatModel): .. versionadded:: 0.3.9 """ - output_version: Literal["v0", "responses/v1"] = "v0" + output_version: str = "v0" """Version of AIMessage output format to use. This field is used to roll-out new output formats for chat model AIMessages @@ -691,10 +716,8 @@ class BaseChatOpenAI(BaseChatModel): - ``'v0'``: AIMessage format as of langchain-openai 0.3.x. - ``'responses/v1'``: Formats Responses API output - items into AIMessage content blocks. - - Currently only impacts the Responses API. ``output_version='responses/v1'`` is - recommended. + items into AIMessage content blocks (Responses API only) + - ``"v1"``: v1 of LangChain cross-provider standard. .. versionadded:: 0.3.25 @@ -899,6 +922,10 @@ def _convert_chunk_to_generation_chunk( message=default_chunk_class(content="", usage_metadata=usage_metadata), generation_info=base_generation_info, ) + if self.output_version == "v1": + generation_chunk.message.content = [] + generation_chunk.message.response_metadata["output_version"] = "v1" + return generation_chunk choice = choices[0] @@ -911,6 +938,7 @@ def _convert_chunk_to_generation_chunk( generation_info = {**base_generation_info} if base_generation_info else {} if finish_reason := choice.get("finish_reason"): + generation_info["model_provider"] = "openai" generation_info["finish_reason"] = finish_reason if model_name := chunk.get("model"): generation_info["model_name"] = model_name @@ -1219,7 +1247,12 @@ def _get_request_payload( else: payload = _construct_responses_api_payload(messages, payload) else: - payload["messages"] = [_convert_message_to_dict(m) for m in messages] + payload["messages"] = [ + _convert_message_to_dict(_convert_from_v1_to_chat_completions(m)) + if isinstance(m, AIMessage) + else _convert_message_to_dict(m) + for m in messages + ] return payload def _create_chat_result( @@ -1268,6 +1301,7 @@ def _create_chat_result( generations.append(gen) llm_output = { "token_usage": token_usage, + "model_provider": "openai", "model_name": response_dict.get("model", self.model_name), "system_fingerprint": response_dict.get("system_fingerprint", ""), } @@ -1499,7 +1533,7 @@ def get_token_ids(self, text: str) -> list[int]: def get_num_tokens_from_messages( self, - messages: list[BaseMessage], + messages: Sequence[BaseMessage], tools: Optional[ Sequence[Union[dict[str, Any], type, Callable, BaseTool]] ] = None, @@ -3387,6 +3421,20 @@ def _oai_structured_outputs_parser( return parsed elif ai_msg.additional_kwargs.get("refusal"): raise OpenAIRefusalError(ai_msg.additional_kwargs["refusal"]) + elif any( + isinstance(block, dict) + and block.get("type") == "non_standard" + and "refusal" in block["value"] + for block in ai_msg.content + ): + refusal = next( + block["value"]["refusal"] + for block in ai_msg.content + if isinstance(block, dict) + and block["type"] == "non_standard" + and "refusal" in block["value"] + ) + raise OpenAIRefusalError(refusal) elif ai_msg.tool_calls: return None else: @@ -3503,7 +3551,7 @@ def _get_last_messages( msg = messages[i] if isinstance(msg, AIMessage): response_id = msg.response_metadata.get("id") - if response_id: + if response_id and response_id.startswith("resp_"): return messages[i + 1 :], response_id else: return messages, None @@ -3612,23 +3660,45 @@ def _construct_responses_api_payload( return payload -def _make_computer_call_output_from_message(message: ToolMessage) -> dict: - computer_call_output: dict = { - "call_id": message.tool_call_id, - "type": "computer_call_output", - } +def _make_computer_call_output_from_message( + message: ToolMessage, +) -> Optional[dict[str, Any]]: + computer_call_output: Optional[dict[str, Any]] = None if isinstance(message.content, list): - # Use first input_image block - output = next( - block - for block in message.content - if cast(dict, block)["type"] == "input_image" - ) + for block in message.content: + if ( + message.additional_kwargs.get("type") == "computer_call_output" + and isinstance(block, dict) + and block.get("type") == "input_image" + ): + # Use first input_image block + computer_call_output = { + "call_id": message.tool_call_id, + "type": "computer_call_output", + "output": block, + } + break + elif ( + isinstance(block, dict) + and block.get("type") == "non_standard" + and block.get("value", {}).get("type") == "computer_call_output" + ): + computer_call_output = block["value"] + break + else: + pass else: - # string, assume image_url - output = {"type": "input_image", "image_url": message.content} - computer_call_output["output"] = output - if "acknowledged_safety_checks" in message.additional_kwargs: + if message.additional_kwargs.get("type") == "computer_call_output": + # string, assume image_url + computer_call_output = { + "call_id": message.tool_call_id, + "type": "computer_call_output", + "output": {"type": "input_image", "image_url": message.content}, + } + if ( + computer_call_output is not None + and "acknowledged_safety_checks" in message.additional_kwargs + ): computer_call_output["acknowledged_safety_checks"] = message.additional_kwargs[ "acknowledged_safety_checks" ] @@ -3645,6 +3715,15 @@ def _make_custom_tool_output_from_message(message: ToolMessage) -> Optional[dict "output": block.get("output") or "", } break + elif ( + isinstance(block, dict) + and block.get("type") == "non_standard" + and block.get("value", {}).get("type") == "custom_tool_call_output" + ): + custom_tool_output = block["value"] + break + else: + pass return custom_tool_output @@ -3669,20 +3748,40 @@ def _construct_responses_api_input(messages: Sequence[BaseMessage]) -> list: for lc_msg in messages: if isinstance(lc_msg, AIMessage): lc_msg = _convert_from_v03_ai_message(lc_msg) - msg = _convert_message_to_dict(lc_msg) + msg = _convert_message_to_dict(lc_msg, responses_ai_msg=True) + if isinstance(msg.get("content"), list) and all( + isinstance(block, dict) for block in msg["content"] + ): + tcs: list[types.ToolCall] = [ + { + "type": "tool_call", + "name": tool_call["name"], + "args": tool_call["args"], + "id": tool_call.get("id"), + } + for tool_call in lc_msg.tool_calls + ] + msg["content"] = _convert_from_v1_to_responses(msg["content"], tcs) + else: + msg = _convert_message_to_dict(lc_msg) + # Get content from non-standard content blocks + if isinstance(msg["content"], list): + for i, block in enumerate(msg["content"]): + if isinstance(block, dict) and block.get("type") == "non_standard": + msg["content"][i] = block["value"] # "name" parameter unsupported if "name" in msg: msg.pop("name") if msg["role"] == "tool": tool_output = msg["content"] + computer_call_output = _make_computer_call_output_from_message( + cast(ToolMessage, lc_msg) + ) custom_tool_output = _make_custom_tool_output_from_message(lc_msg) # type: ignore[arg-type] - if custom_tool_output: - input_.append(custom_tool_output) - elif lc_msg.additional_kwargs.get("type") == "computer_call_output": - computer_call_output = _make_computer_call_output_from_message( - cast(ToolMessage, lc_msg) - ) + if computer_call_output: input_.append(computer_call_output) + elif custom_tool_output: + input_.append(custom_tool_output) else: if not isinstance(tool_output, str): tool_output = _stringify(tool_output) @@ -3837,7 +3936,7 @@ def _construct_lc_result_from_responses_api( response: Response, schema: Optional[type[_BM]] = None, metadata: Optional[dict] = None, - output_version: Literal["v0", "responses/v1"] = "v0", + output_version: str = "v0", ) -> ChatResult: """Construct ChatResponse from OpenAI Response API response.""" if response.error: @@ -3864,6 +3963,7 @@ def _construct_lc_result_from_responses_api( if metadata: response_metadata.update(metadata) # for compatibility with chat completion calls. + response_metadata["model_provider"] = "openai" response_metadata["model_name"] = response_metadata.get("model") if response.usage: usage_metadata = _create_usage_metadata_responses(response.usage.model_dump()) @@ -3977,6 +4077,7 @@ def _construct_lc_result_from_responses_api( additional_kwargs["parsed"] = parsed except json.JSONDecodeError: pass + message = AIMessage( content=content_blocks, id=response.id, @@ -3988,8 +4089,7 @@ def _construct_lc_result_from_responses_api( ) if output_version == "v0": message = _convert_to_v03_ai_message(message) - else: - pass + return ChatResult(generations=[ChatGeneration(message=message)]) @@ -4001,7 +4101,7 @@ def _convert_responses_chunk_to_generation_chunk( schema: Optional[type[_BM]] = None, metadata: Optional[dict] = None, has_reasoning: bool = False, - output_version: Literal["v0", "responses/v1"] = "v0", + output_version: str = "v0", ) -> tuple[int, int, int, Optional[ChatGenerationChunk]]: def _advance(output_idx: int, sub_idx: Optional[int] = None) -> None: """Advance indexes tracked during streaming. @@ -4055,6 +4155,7 @@ def _advance(output_idx: int, sub_idx: Optional[int] = None) -> None: response_metadata = metadata else: response_metadata = {} + response_metadata["model_provider"] = "openai" usage_metadata = None id = None if chunk.type == "response.output_text.delta": @@ -4067,9 +4168,12 @@ def _advance(output_idx: int, sub_idx: Optional[int] = None) -> None: annotation = chunk.annotation else: annotation = chunk.annotation.model_dump(exclude_none=True, mode="json") - content.append({"annotations": [annotation], "index": current_index}) + + content.append( + {"type": "text", "annotations": [annotation], "index": current_index} + ) elif chunk.type == "response.output_text.done": - content.append({"id": chunk.item_id, "index": current_index}) + content.append({"type": "text", "id": chunk.item_id, "index": current_index}) elif chunk.type == "response.created": id = chunk.response.id response_metadata["id"] = chunk.response.id # Backwards compatibility @@ -4162,6 +4266,7 @@ def _advance(output_idx: int, sub_idx: Optional[int] = None) -> None: content.append({"type": "refusal", "refusal": chunk.refusal}) elif chunk.type == "response.output_item.added" and chunk.item.type == "reasoning": _advance(chunk.output_index) + current_sub_index = 0 reasoning = chunk.item.model_dump(exclude_none=True, mode="json") reasoning["index"] = current_index content.append(reasoning) @@ -4175,6 +4280,7 @@ def _advance(output_idx: int, sub_idx: Optional[int] = None) -> None: ], "index": current_index, "type": "reasoning", + "id": chunk.item_id, } ) elif chunk.type == "response.image_generation_call.partial_image": @@ -4211,8 +4317,7 @@ def _advance(output_idx: int, sub_idx: Optional[int] = None) -> None: AIMessageChunk, _convert_to_v03_ai_message(message, has_reasoning=has_reasoning), ) - else: - pass + return ( current_index, current_output_index, diff --git a/libs/partners/openai/tests/cassettes/test_function_calling.yaml.gz b/libs/partners/openai/tests/cassettes/test_function_calling.yaml.gz new file mode 100644 index 0000000000000..197a8402cf6eb Binary files /dev/null and b/libs/partners/openai/tests/cassettes/test_function_calling.yaml.gz differ diff --git a/libs/partners/openai/tests/cassettes/test_parsed_pydantic_schema.yaml.gz b/libs/partners/openai/tests/cassettes/test_parsed_pydantic_schema.yaml.gz new file mode 100644 index 0000000000000..13c0b8896decc Binary files /dev/null and b/libs/partners/openai/tests/cassettes/test_parsed_pydantic_schema.yaml.gz differ diff --git a/libs/partners/openai/tests/cassettes/test_web_search.yaml.gz b/libs/partners/openai/tests/cassettes/test_web_search.yaml.gz index e99f1c2e13a14..a202dfe9c6141 100644 Binary files a/libs/partners/openai/tests/cassettes/test_web_search.yaml.gz and b/libs/partners/openai/tests/cassettes/test_web_search.yaml.gz differ diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_responses_api.py b/libs/partners/openai/tests/integration_tests/chat_models/test_responses_api.py index 1430d7008b55d..bd9b83752a289 100644 --- a/libs/partners/openai/tests/integration_tests/chat_models/test_responses_api.py +++ b/libs/partners/openai/tests/integration_tests/chat_models/test_responses_api.py @@ -28,8 +28,9 @@ def _check_response(response: Optional[BaseMessage]) -> None: for block in response.content: assert isinstance(block, dict) if block["type"] == "text": - assert isinstance(block["text"], str) - for annotation in block["annotations"]: + assert isinstance(block.get("text"), str) + annotations = block.get("annotations", []) + for annotation in annotations: if annotation["type"] == "file_citation": assert all( key in annotation @@ -40,8 +41,12 @@ def _check_response(response: Optional[BaseMessage]) -> None: key in annotation for key in ["end_index", "start_index", "title", "type", "url"] ) - - text_content = response.text() + elif annotation["type"] == "citation": + assert all(key in annotation for key in ["title", "type"]) + if "url" in annotation: + assert "start_index" in annotation + assert "end_index" in annotation + text_content = response.text() # type: ignore[operator,misc] assert isinstance(text_content, str) assert text_content assert response.usage_metadata @@ -49,12 +54,14 @@ def _check_response(response: Optional[BaseMessage]) -> None: assert response.usage_metadata["output_tokens"] > 0 assert response.usage_metadata["total_tokens"] > 0 assert response.response_metadata["model_name"] - assert response.response_metadata["service_tier"] + assert response.response_metadata["service_tier"] # type: ignore[typeddict-item] +@pytest.mark.default_cassette("test_web_search.yaml.gz") @pytest.mark.vcr -def test_web_search() -> None: - llm = ChatOpenAI(model=MODEL_NAME, output_version="responses/v1") +@pytest.mark.parametrize("output_version", ["responses/v1", "v1"]) +def test_web_search(output_version: Literal["responses/v1", "v1"]) -> None: + llm = ChatOpenAI(model=MODEL_NAME, output_version=output_version) first_response = llm.invoke( "What was a positive news story from today?", tools=[{"type": "web_search_preview"}], @@ -82,20 +89,9 @@ def test_web_search() -> None: # Manually pass in chat history response = llm.invoke( [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What was a positive news story from today?", - } - ], - }, + {"role": "user", "content": "What was a positive news story from today?"}, first_response, - { - "role": "user", - "content": [{"type": "text", "text": "what about a negative one"}], - }, + {"role": "user", "content": "what about a negative one"}, ], tools=[{"type": "web_search_preview"}], ) @@ -108,9 +104,12 @@ def test_web_search() -> None: _check_response(response) for msg in [first_response, full, response]: - assert isinstance(msg, AIMessage) + assert msg is not None block_types = [block["type"] for block in msg.content] # type: ignore[index] - assert block_types == ["web_search_call", "text"] + if output_version == "responses/v1": + assert block_types == ["web_search_call", "text"] + else: + assert block_types == ["web_search_call", "web_search_result", "text"] @pytest.mark.flaky(retries=3, delay=1) @@ -141,13 +140,15 @@ async def test_web_search_async() -> None: assert tool_output["type"] == "web_search_call" -@pytest.mark.flaky(retries=3, delay=1) -def test_function_calling() -> None: +@pytest.mark.default_cassette("test_function_calling.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["v0", "responses/v1", "v1"]) +def test_function_calling(output_version: Literal["v0", "responses/v1", "v1"]) -> None: def multiply(x: int, y: int) -> int: """return x * y""" return x * y - llm = ChatOpenAI(model=MODEL_NAME) + llm = ChatOpenAI(model=MODEL_NAME, output_version=output_version) bound_llm = llm.bind_tools([multiply, {"type": "web_search_preview"}]) ai_msg = cast(AIMessage, bound_llm.invoke("whats 5 * 4")) assert len(ai_msg.tool_calls) == 1 @@ -174,8 +175,15 @@ class FooDict(TypedDict): response: str -def test_parsed_pydantic_schema() -> None: - llm = ChatOpenAI(model=MODEL_NAME, use_responses_api=True) +@pytest.mark.default_cassette("test_parsed_pydantic_schema.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["v0", "responses/v1", "v1"]) +def test_parsed_pydantic_schema( + output_version: Literal["v0", "responses/v1", "v1"], +) -> None: + llm = ChatOpenAI( + model=MODEL_NAME, use_responses_api=True, output_version=output_version + ) response = llm.invoke("how are ya", response_format=Foo) parsed = Foo(**json.loads(response.text())) assert parsed == response.additional_kwargs["parsed"] @@ -297,8 +305,8 @@ def multiply(x: int, y: int) -> int: @pytest.mark.default_cassette("test_reasoning.yaml.gz") @pytest.mark.vcr -@pytest.mark.parametrize("output_version", ["v0", "responses/v1"]) -def test_reasoning(output_version: Literal["v0", "responses/v1"]) -> None: +@pytest.mark.parametrize("output_version", ["v0", "responses/v1", "v1"]) +def test_reasoning(output_version: Literal["v0", "responses/v1", "v1"]) -> None: llm = ChatOpenAI( model="o4-mini", use_responses_api=True, output_version=output_version ) @@ -358,27 +366,32 @@ def test_computer_calls() -> None: def test_file_search() -> None: pytest.skip() # TODO: set up infra - llm = ChatOpenAI(model=MODEL_NAME) + llm = ChatOpenAI(model=MODEL_NAME, use_responses_api=True) tool = { "type": "file_search", "vector_store_ids": [os.environ["OPENAI_VECTOR_STORE_ID"]], } - response = llm.invoke("What is deep research by OpenAI?", tools=[tool]) + + input_message = {"role": "user", "content": "What is deep research by OpenAI?"} + response = llm.invoke([input_message], tools=[tool]) _check_response(response) full: Optional[BaseMessageChunk] = None - for chunk in llm.stream("What is deep research by OpenAI?", tools=[tool]): + for chunk in llm.stream([input_message], tools=[tool]): assert isinstance(chunk, AIMessageChunk) full = chunk if full is None else full + chunk assert isinstance(full, AIMessageChunk) _check_response(full) + next_message = {"role": "user", "content": "Thank you."} + _ = llm.invoke([input_message, full, next_message]) + @pytest.mark.default_cassette("test_stream_reasoning_summary.yaml.gz") @pytest.mark.vcr -@pytest.mark.parametrize("output_version", ["v0", "responses/v1"]) +@pytest.mark.parametrize("output_version", ["v0", "responses/v1", "v1"]) def test_stream_reasoning_summary( - output_version: Literal["v0", "responses/v1"], + output_version: Literal["v0", "responses/v1", "v1"], ) -> None: llm = ChatOpenAI( model="o4-mini", @@ -398,7 +411,14 @@ def test_stream_reasoning_summary( if output_version == "v0": reasoning = response_1.additional_kwargs["reasoning"] assert set(reasoning.keys()) == {"id", "type", "summary"} - else: + summary = reasoning["summary"] + assert isinstance(summary, list) + for block in summary: + assert isinstance(block, dict) + assert isinstance(block["type"], str) + assert isinstance(block["text"], str) + assert block["text"] + elif output_version == "responses/v1": reasoning = next( block for block in response_1.content @@ -407,13 +427,27 @@ def test_stream_reasoning_summary( if isinstance(reasoning, str): reasoning = json.loads(reasoning) assert set(reasoning.keys()) == {"id", "type", "summary", "index"} - summary = reasoning["summary"] - assert isinstance(summary, list) - for block in summary: - assert isinstance(block, dict) - assert isinstance(block["type"], str) - assert isinstance(block["text"], str) - assert block["text"] + summary = reasoning["summary"] + assert isinstance(summary, list) + for block in summary: + assert isinstance(block, dict) + assert isinstance(block["type"], str) + assert isinstance(block["text"], str) + assert block["text"] + else: + # v1 + total_reasoning_blocks = 0 + for block in response_1.content_blocks: + if block["type"] == "reasoning": + total_reasoning_blocks += 1 + assert isinstance(block.get("id"), str) and block.get( + "id", "" + ).startswith("rs_") + assert isinstance(block.get("reasoning"), str) + assert isinstance(block.get("index"), str) + assert ( + total_reasoning_blocks > 1 + ) # This query typically generates multiple reasoning blocks # Check we can pass back summaries message_2 = {"role": "user", "content": "Thank you."} @@ -421,9 +455,13 @@ def test_stream_reasoning_summary( assert isinstance(response_2, AIMessage) +@pytest.mark.default_cassette("test_code_interpreter.yaml.gz") @pytest.mark.vcr -def test_code_interpreter() -> None: - llm = ChatOpenAI(model="o4-mini", use_responses_api=True) +@pytest.mark.parametrize("output_version", ["v0", "responses/v1", "v1"]) +def test_code_interpreter(output_version: Literal["v0", "responses/v1", "v1"]) -> None: + llm = ChatOpenAI( + model="o4-mini", use_responses_api=True, output_version=output_version + ) llm_with_tools = llm.bind_tools( [{"type": "code_interpreter", "container": {"type": "auto"}}] ) @@ -432,16 +470,43 @@ def test_code_interpreter() -> None: "content": "Write and run code to answer the question: what is 3^3?", } response = llm_with_tools.invoke([input_message]) + assert isinstance(response, AIMessage) _check_response(response) - tool_outputs = response.additional_kwargs["tool_outputs"] - assert tool_outputs - assert any(output["type"] == "code_interpreter_call" for output in tool_outputs) + if output_version == "v0": + tool_outputs = [ + item + for item in response.additional_kwargs["tool_outputs"] + if item["type"] == "code_interpreter_call" + ] + assert len(tool_outputs) == 1 + elif output_version == "responses/v1": + tool_outputs = [ + item + for item in response.content + if isinstance(item, dict) and item["type"] == "code_interpreter_call" + ] + assert len(tool_outputs) == 1 + else: + # v1 + tool_outputs = [ + item + for item in response.content_blocks + if item["type"] == "code_interpreter_call" + ] + code_interpreter_result = next( + item + for item in response.content_blocks + if item["type"] == "code_interpreter_result" + ) + assert tool_outputs + assert code_interpreter_result + assert len(tool_outputs) == 1 # Test streaming # Use same container - tool_outputs = response.additional_kwargs["tool_outputs"] - assert len(tool_outputs) == 1 - container_id = tool_outputs[0]["container_id"] + container_id = tool_outputs[0].get("container_id") or tool_outputs[0].get( + "extras", {} + ).get("container_id") llm_with_tools = llm.bind_tools( [{"type": "code_interpreter", "container": container_id}] ) @@ -451,9 +516,34 @@ def test_code_interpreter() -> None: assert isinstance(chunk, AIMessageChunk) full = chunk if full is None else full + chunk assert isinstance(full, AIMessageChunk) - tool_outputs = full.additional_kwargs["tool_outputs"] - assert tool_outputs - assert any(output["type"] == "code_interpreter_call" for output in tool_outputs) + if output_version == "v0": + tool_outputs = [ + item + for item in response.additional_kwargs["tool_outputs"] + if item["type"] == "code_interpreter_call" + ] + assert tool_outputs + elif output_version == "responses/v1": + tool_outputs = [ + item + for item in response.content + if isinstance(item, dict) and item["type"] == "code_interpreter_call" + ] + assert tool_outputs + else: + # v1 + code_interpreter_call = next( + item + for item in full.content_blocks + if item["type"] == "code_interpreter_call" + ) + code_interpreter_result = next( + item + for item in full.content_blocks + if item["type"] == "code_interpreter_result" + ) + assert code_interpreter_call + assert code_interpreter_result # Test we can pass back in next_message = {"role": "user", "content": "Please add more comments to the code."} @@ -548,10 +638,69 @@ def test_mcp_builtin_zdr() -> None: _ = llm_with_tools.invoke([input_message, full, approval_message]) -@pytest.mark.vcr() -def test_image_generation_streaming() -> None: +@pytest.mark.default_cassette("test_mcp_builtin_zdr.yaml.gz") +@pytest.mark.vcr +def test_mcp_builtin_zdr_v1() -> None: + llm = ChatOpenAI( + model="o4-mini", + output_version="v1", + store=False, + include=["reasoning.encrypted_content"], + ) + + llm_with_tools = llm.bind_tools( + [ + { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": {"always": {"tool_names": ["read_wiki_structure"]}}, + } + ] + ) + input_message = { + "role": "user", + "content": ( + "What transport protocols does the 2025-03-26 version of the MCP spec " + "support?" + ), + } + full: Optional[BaseMessageChunk] = None + for chunk in llm_with_tools.stream([input_message]): + assert isinstance(chunk, AIMessageChunk) + full = chunk if full is None else full + chunk + + assert isinstance(full, AIMessageChunk) + assert all(isinstance(block, dict) for block in full.content) + + approval_message = HumanMessage( + [ + { + "type": "non_standard", + "value": { + "type": "mcp_approval_response", + "approve": True, + "approval_request_id": block["value"]["id"], # type: ignore[index] + }, + } + for block in full.content_blocks + if block["type"] == "non_standard" + and block["value"]["type"] == "mcp_approval_request" # type: ignore[index] + ] + ) + _ = llm_with_tools.invoke([input_message, full, approval_message]) + + +@pytest.mark.default_cassette("test_image_generation_streaming.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["v0", "responses/v1"]) +def test_image_generation_streaming( + output_version: Literal["v0", "responses/v1"], +) -> None: """Test image generation streaming.""" - llm = ChatOpenAI(model="gpt-4.1", use_responses_api=True) + llm = ChatOpenAI( + model="gpt-4.1", use_responses_api=True, output_version=output_version + ) tool = { "type": "image_generation", # For testing purposes let's keep the quality low, so the test runs faster. @@ -598,15 +747,69 @@ def test_image_generation_streaming() -> None: # At the moment, the streaming API does not pick up annotations fully. # So the following check is commented out. # _check_response(complete_ai_message) - tool_output = complete_ai_message.additional_kwargs["tool_outputs"][0] - assert set(tool_output.keys()).issubset(expected_keys) + if output_version == "v0": + assert complete_ai_message.additional_kwargs["tool_outputs"] + tool_output = complete_ai_message.additional_kwargs["tool_outputs"][0] + assert set(tool_output.keys()).issubset(expected_keys) + else: + # "responses/v1" + tool_output = next( + block + for block in complete_ai_message.content + if isinstance(block, dict) and block["type"] == "image_generation_call" + ) + assert set(tool_output.keys()).issubset(expected_keys) -@pytest.mark.vcr() -def test_image_generation_multi_turn() -> None: +@pytest.mark.default_cassette("test_image_generation_streaming.yaml.gz") +@pytest.mark.vcr +def test_image_generation_streaming_v1() -> None: + """Test image generation streaming.""" + llm = ChatOpenAI(model="gpt-4.1", use_responses_api=True, output_version="v1") + tool = { + "type": "image_generation", + "quality": "low", + "output_format": "jpeg", + "output_compression": 100, + "size": "1024x1024", + } + + standard_keys = {"type", "base64", "mime_type", "id", "index"} + extra_keys = { + "background", + "output_format", + "quality", + "revised_prompt", + "size", + "status", + } + + full: Optional[BaseMessageChunk] = None + for chunk in llm.stream("Draw a random short word in green font.", tools=[tool]): + assert isinstance(chunk, AIMessageChunk) + full = chunk if full is None else full + chunk + complete_ai_message = cast(AIMessageChunk, full) + + tool_output = next( + block + for block in complete_ai_message.content + if isinstance(block, dict) and block["type"] == "image" + ) + assert set(standard_keys).issubset(tool_output.keys()) + assert set(extra_keys).issubset(tool_output["extras"].keys()) + + +@pytest.mark.default_cassette("test_image_generation_multi_turn.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["v0", "responses/v1"]) +def test_image_generation_multi_turn( + output_version: Literal["v0", "responses/v1"], +) -> None: """Test multi-turn editing of image generation by passing in history.""" # Test multi-turn - llm = ChatOpenAI(model="gpt-4.1", use_responses_api=True) + llm = ChatOpenAI( + model="gpt-4.1", use_responses_api=True, output_version=output_version + ) # Test invocation tool = { "type": "image_generation", @@ -622,10 +825,41 @@ def test_image_generation_multi_turn() -> None: {"role": "user", "content": "Draw a random short word in green font."} ] ai_message = llm_with_tools.invoke(chat_history) + assert isinstance(ai_message, AIMessage) _check_response(ai_message) - tool_output = ai_message.additional_kwargs["tool_outputs"][0] - # Example tool output for an image + expected_keys = { + "id", + "background", + "output_format", + "quality", + "result", + "revised_prompt", + "size", + "status", + "type", + } + + if output_version == "v0": + tool_output = ai_message.additional_kwargs["tool_outputs"][0] + assert set(tool_output.keys()).issubset(expected_keys) + elif output_version == "responses/v1": + tool_output = next( + block + for block in ai_message.content + if isinstance(block, dict) and block["type"] == "image_generation_call" + ) + assert set(tool_output.keys()).issubset(expected_keys) + else: + standard_keys = {"type", "base64", "id", "status"} + tool_output = next( + block + for block in ai_message.content + if isinstance(block, dict) and block["type"] == "image" + ) + assert set(standard_keys).issubset(tool_output.keys()) + + # Example tool output for an image (v0) # { # "background": "opaque", # "id": "ig_683716a8ddf0819888572b20621c7ae4029ec8c11f8dacf8", @@ -641,19 +875,78 @@ def test_image_generation_multi_turn() -> None: # "result": # base64 encode image data # } - expected_keys = { - "id", + chat_history.extend( + [ + # AI message with tool output + ai_message, + # New request + { + "role": "user", + "content": ( + "Now, change the font to blue. Keep the word and everything else " + "the same." + ), + }, + ] + ) + + ai_message2 = llm_with_tools.invoke(chat_history) + assert isinstance(ai_message2, AIMessage) + _check_response(ai_message2) + + if output_version == "v0": + tool_output = ai_message2.additional_kwargs["tool_outputs"][0] + assert set(tool_output.keys()).issubset(expected_keys) + else: + # "responses/v1" + tool_output = next( + block + for block in ai_message2.content + if isinstance(block, dict) and block["type"] == "image_generation_call" + ) + assert set(tool_output.keys()).issubset(expected_keys) + + +@pytest.mark.default_cassette("test_image_generation_multi_turn.yaml.gz") +@pytest.mark.vcr +def test_image_generation_multi_turn_v1() -> None: + """Test multi-turn editing of image generation by passing in history.""" + # Test multi-turn + llm = ChatOpenAI(model="gpt-4.1", use_responses_api=True, output_version="v1") + # Test invocation + tool = { + "type": "image_generation", + "quality": "low", + "output_format": "jpeg", + "output_compression": 100, + "size": "1024x1024", + } + llm_with_tools = llm.bind_tools([tool]) + + chat_history: list[MessageLikeRepresentation] = [ + {"role": "user", "content": "Draw a random short word in green font."} + ] + ai_message = llm_with_tools.invoke(chat_history) + assert isinstance(ai_message, AIMessage) + _check_response(ai_message) + + standard_keys = {"type", "base64", "mime_type", "id"} + extra_keys = { "background", "output_format", "quality", - "result", "revised_prompt", "size", "status", - "type", } - assert set(tool_output.keys()).issubset(expected_keys) + tool_output = next( + block + for block in ai_message.content + if isinstance(block, dict) and block["type"] == "image" + ) + assert set(standard_keys).issubset(tool_output.keys()) + assert set(extra_keys).issubset(tool_output["extras"].keys()) chat_history.extend( [ @@ -671,9 +964,16 @@ def test_image_generation_multi_turn() -> None: ) ai_message2 = llm_with_tools.invoke(chat_history) + assert isinstance(ai_message2, AIMessage) _check_response(ai_message2) - tool_output2 = ai_message2.additional_kwargs["tool_outputs"][0] - assert set(tool_output2.keys()).issubset(expected_keys) + + tool_output = next( + block + for block in ai_message2.content + if isinstance(block, dict) and block["type"] == "image" + ) + assert set(standard_keys).issubset(tool_output.keys()) + assert set(extra_keys).issubset(tool_output["extras"].keys()) def test_verbosity_parameter() -> None: @@ -689,14 +989,16 @@ def test_verbosity_parameter() -> None: assert response.content -@pytest.mark.vcr() -def test_custom_tool() -> None: +@pytest.mark.default_cassette("test_custom_tool.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["responses/v1", "v1"]) +def test_custom_tool(output_version: Literal["responses/v1", "v1"]) -> None: @custom_tool def execute_code(code: str) -> str: """Execute python code.""" return "27" - llm = ChatOpenAI(model="gpt-5", output_version="responses/v1").bind_tools( + llm = ChatOpenAI(model="gpt-5", output_version=output_version).bind_tools( [execute_code] ) diff --git a/libs/partners/openai/tests/unit_tests/chat_models/test_base.py b/libs/partners/openai/tests/unit_tests/chat_models/test_base.py index 68caee5d63f36..a63f89647869a 100644 --- a/libs/partners/openai/tests/unit_tests/chat_models/test_base.py +++ b/libs/partners/openai/tests/unit_tests/chat_models/test_base.py @@ -20,13 +20,14 @@ ToolCall, ToolMessage, ) +from langchain_core.messages import content as types from langchain_core.messages.ai import UsageMetadata from langchain_core.outputs import ChatGeneration, ChatResult from langchain_core.runnables import RunnableLambda from langchain_core.tracers.base import BaseTracer from langchain_core.tracers.schemas import Run from openai.types.responses import ResponseOutputMessage, ResponseReasoningItem -from openai.types.responses.response import IncompleteDetails, Response, ResponseUsage +from openai.types.responses.response import IncompleteDetails, Response from openai.types.responses.response_error import ResponseError from openai.types.responses.response_file_search_tool_call import ( ResponseFileSearchToolCall, @@ -43,6 +44,7 @@ from openai.types.responses.response_usage import ( InputTokensDetails, OutputTokensDetails, + ResponseUsage, ) from pydantic import BaseModel, Field, SecretStr from typing_extensions import TypedDict @@ -51,6 +53,8 @@ from langchain_openai.chat_models._compat import ( _FUNCTION_CALL_IDS_MAP_KEY, _convert_from_v03_ai_message, + _convert_from_v1_to_chat_completions, + _convert_from_v1_to_responses, _convert_to_v03_ai_message, ) from langchain_openai.chat_models.base import ( @@ -1231,7 +1235,7 @@ def test_structured_outputs_parser() -> None: serialized = dumps(llm_output) deserialized = loads(serialized) assert isinstance(deserialized, ChatGeneration) - result = output_parser.invoke(deserialized.message) + result = output_parser.invoke(cast(AIMessage, deserialized.message)) assert result == parsed_response @@ -2379,7 +2383,7 @@ def mock_create(*args: Any, **kwargs: Any) -> Response: assert payload["tools"][0]["headers"]["Authorization"] == "Bearer PLACEHOLDER" -def test_compat() -> None: +def test_compat_responses_v03() -> None: # Check compatibility with v0.3 message format message_v03 = AIMessage( content=[ @@ -2440,6 +2444,159 @@ def test_compat() -> None: assert message_v03_output is not message_v03 +@pytest.mark.parametrize( + "message_v1, expected", + [ + ( + AIMessage( + [ + {"type": "reasoning", "reasoning": "Reasoning text"}, + { + "type": "tool_call", + "id": "call_123", + "name": "get_weather", + "args": {"location": "San Francisco"}, + }, + { + "type": "text", + "text": "Hello, world!", + "annotations": [ + {"type": "citation", "url": "https://example.com"} + ], + }, + ], + id="chatcmpl-123", + response_metadata={"model_provider": "openai", "model_name": "gpt-4.1"}, + ), + AIMessage( + [{"type": "text", "text": "Hello, world!"}], + id="chatcmpl-123", + response_metadata={"model_provider": "openai", "model_name": "gpt-4.1"}, + ), + ) + ], +) +def test_convert_from_v1_to_chat_completions( + message_v1: AIMessage, expected: AIMessage +) -> None: + result = _convert_from_v1_to_chat_completions(message_v1) + assert result == expected + assert result.tool_calls == message_v1.tool_calls # tool calls remain cached + + # Check no mutation + assert message_v1 != result + + +@pytest.mark.parametrize( + "message_v1, expected", + [ + ( + AIMessage( + content_blocks=[ + {"type": "reasoning", "id": "abc123"}, + {"type": "reasoning", "id": "abc234", "reasoning": "foo "}, + {"type": "reasoning", "id": "abc234", "reasoning": "bar"}, + { + "type": "tool_call", + "id": "call_123", + "name": "get_weather", + "args": {"location": "San Francisco"}, + }, + { + "type": "tool_call", + "id": "call_234", + "name": "get_weather_2", + "args": {"location": "New York"}, + "extras": {"item_id": "fc_123"}, + }, + {"type": "text", "text": "Hello "}, + { + "type": "text", + "text": "world", + "annotations": [ + {"type": "citation", "url": "https://example.com"}, + { + "type": "citation", + "title": "my doc", + "extras": {"file_id": "file_123", "index": 1}, + }, + { + "type": "non_standard_annotation", + "value": {"bar": "baz"}, + }, + ], + }, + {"type": "image", "base64": "...", "id": "ig_123"}, + { + "type": "non_standard", + "value": {"type": "something_else", "foo": "bar"}, + }, + ], + id="resp123", + ), + [ + {"type": "reasoning", "id": "abc123", "summary": []}, + { + "type": "reasoning", + "id": "abc234", + "summary": [ + {"type": "summary_text", "text": "foo "}, + {"type": "summary_text", "text": "bar"}, + ], + }, + { + "type": "function_call", + "call_id": "call_123", + "name": "get_weather", + "arguments": '{"location": "San Francisco"}', + }, + { + "type": "function_call", + "call_id": "call_234", + "name": "get_weather_2", + "arguments": '{"location": "New York"}', + "id": "fc_123", + }, + {"type": "text", "text": "Hello "}, + { + "type": "text", + "text": "world", + "annotations": [ + {"type": "url_citation", "url": "https://example.com"}, + { + "type": "file_citation", + "filename": "my doc", + "index": 1, + "file_id": "file_123", + }, + {"bar": "baz"}, + ], + }, + {"type": "image_generation_call", "id": "ig_123", "result": "..."}, + {"type": "something_else", "foo": "bar"}, + ], + ) + ], +) +def test_convert_from_v1_to_responses( + message_v1: AIMessage, expected: list[dict[str, Any]] +) -> None: + tcs: list[types.ToolCall] = [ + { + "type": "tool_call", + "name": tool_call["name"], + "args": tool_call["args"], + "id": tool_call.get("id"), + } + for tool_call in message_v1.tool_calls + ] + result = _convert_from_v1_to_responses(message_v1.content_blocks, tcs) + assert result == expected + + # Check no mutation + assert message_v1 != result + + def test_get_last_messages() -> None: messages: list[BaseMessage] = [HumanMessage("Hello")] last_messages, previous_response_id = _get_last_messages(messages) diff --git a/libs/partners/openai/tests/unit_tests/chat_models/test_responses_stream.py b/libs/partners/openai/tests/unit_tests/chat_models/test_responses_stream.py index eca5ee1c2559a..fd4e716e882ab 100644 --- a/libs/partners/openai/tests/unit_tests/chat_models/test_responses_stream.py +++ b/libs/partners/openai/tests/unit_tests/chat_models/test_responses_stream.py @@ -1,6 +1,7 @@ from typing import Any, Optional from unittest.mock import MagicMock, patch +import pytest from langchain_core.messages import AIMessageChunk, BaseMessageChunk from openai.types.responses import ( ResponseCompletedEvent, @@ -20,7 +21,7 @@ ResponseTextDeltaEvent, ResponseTextDoneEvent, ) -from openai.types.responses.response import Response, ResponseUsage +from openai.types.responses.response import Response from openai.types.responses.response_output_text import ResponseOutputText from openai.types.responses.response_reasoning_item import Summary from openai.types.responses.response_reasoning_summary_part_added_event import ( @@ -32,6 +33,7 @@ from openai.types.responses.response_usage import ( InputTokensDetails, OutputTokensDetails, + ResponseUsage, ) from openai.types.shared.reasoning import Reasoning from openai.types.shared.response_format_text import ResponseFormatText @@ -337,7 +339,7 @@ id="rs_234", summary=[], type="reasoning", - encrypted_content=None, + encrypted_content="encrypted-content", status=None, ), output_index=2, @@ -416,7 +418,7 @@ Summary(text="still more reasoning", type="summary_text"), ], type="reasoning", - encrypted_content=None, + encrypted_content="encrypted-content", status=None, ), output_index=2, @@ -562,7 +564,7 @@ Summary(text="still more reasoning", type="summary_text"), ], type="reasoning", - encrypted_content=None, + encrypted_content="encrypted-content", status=None, ), ResponseOutputMessage( @@ -620,8 +622,104 @@ def _strip_none(obj: Any) -> Any: return obj -def test_responses_stream() -> None: - llm = ChatOpenAI(model="o4-mini", output_version="responses/v1") +@pytest.mark.parametrize( + "output_version, expected_content", + [ + ( + "responses/v1", + [ + { + "id": "rs_123", + "summary": [ + { + "index": 0, + "type": "summary_text", + "text": "reasoning block one", + }, + { + "index": 1, + "type": "summary_text", + "text": "another reasoning block", + }, + ], + "type": "reasoning", + "index": 0, + }, + {"type": "text", "text": "text block one", "index": 1, "id": "msg_123"}, + { + "type": "text", + "text": "another text block", + "index": 2, + "id": "msg_123", + }, + { + "id": "rs_234", + "summary": [ + {"index": 0, "type": "summary_text", "text": "more reasoning"}, + { + "index": 1, + "type": "summary_text", + "text": "still more reasoning", + }, + ], + "encrypted_content": "encrypted-content", + "type": "reasoning", + "index": 3, + }, + {"type": "text", "text": "more", "index": 4, "id": "msg_234"}, + {"type": "text", "text": "text", "index": 5, "id": "msg_234"}, + ], + ), + ( + "v1", + [ + { + "type": "reasoning", + "reasoning": "reasoning block one", + "id": "rs_123", + "index": "lc_rs_305f30", + }, + { + "type": "reasoning", + "reasoning": "another reasoning block", + "id": "rs_123", + "index": "lc_rs_305f31", + }, + { + "type": "text", + "text": "text block one", + "index": "lc_txt_1", + "id": "msg_123", + }, + { + "type": "text", + "text": "another text block", + "index": "lc_txt_2", + "id": "msg_123", + }, + { + "type": "reasoning", + "reasoning": "more reasoning", + "id": "rs_234", + "extras": {"encrypted_content": "encrypted-content"}, + "index": "lc_rs_335f30", + }, + { + "type": "reasoning", + "reasoning": "still more reasoning", + "id": "rs_234", + "index": "lc_rs_335f31", + }, + {"type": "text", "text": "more", "index": "lc_txt_4", "id": "msg_234"}, + {"type": "text", "text": "text", "index": "lc_txt_5", "id": "msg_234"}, + ], + ), + ], +) +def test_responses_stream(output_version: str, expected_content: list[dict]) -> None: + llm = ChatOpenAI( + model="o4-mini", use_responses_api=True, output_version=output_version + ) mock_client = MagicMock() def mock_create(*args: Any, **kwargs: Any) -> MockSyncContextManager: @@ -630,36 +728,14 @@ def mock_create(*args: Any, **kwargs: Any) -> MockSyncContextManager: mock_client.responses.create = mock_create full: Optional[BaseMessageChunk] = None + chunks = [] with patch.object(llm, "root_client", mock_client): for chunk in llm.stream("test"): assert isinstance(chunk, AIMessageChunk) full = chunk if full is None else full + chunk + chunks.append(chunk) assert isinstance(full, AIMessageChunk) - expected_content = [ - { - "id": "rs_123", - "summary": [ - {"index": 0, "type": "summary_text", "text": "reasoning block one"}, - {"index": 1, "type": "summary_text", "text": "another reasoning block"}, - ], - "type": "reasoning", - "index": 0, - }, - {"type": "text", "text": "text block one", "index": 1, "id": "msg_123"}, - {"type": "text", "text": "another text block", "index": 2, "id": "msg_123"}, - { - "id": "rs_234", - "summary": [ - {"index": 0, "type": "summary_text", "text": "more reasoning"}, - {"index": 1, "type": "summary_text", "text": "still more reasoning"}, - ], - "type": "reasoning", - "index": 3, - }, - {"type": "text", "text": "more", "index": 4, "id": "msg_234"}, - {"type": "text", "text": "text", "index": 5, "id": "msg_234"}, - ] assert full.content == expected_content assert full.additional_kwargs == {} assert full.id == "resp_123"