From dfd16a84d8c3b53596d9b5cfbc0f7f0bc12ceba5 Mon Sep 17 00:00:00 2001 From: justin-tahara Date: Mon, 22 Sep 2025 17:29:56 -0700 Subject: [PATCH] fix(processing): Mime types for Image Summarization --- .../file_processing/image_summarization.py | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/backend/onyx/file_processing/image_summarization.py b/backend/onyx/file_processing/image_summarization.py index c8c6add996a..12f65060862 100644 --- a/backend/onyx/file_processing/image_summarization.py +++ b/backend/onyx/file_processing/image_summarization.py @@ -10,11 +10,16 @@ from onyx.configs.app_configs import IMAGE_SUMMARIZATION_USER_PROMPT from onyx.llm.interfaces import LLM from onyx.llm.utils import message_to_string +from onyx.utils.b64 import get_image_type_from_bytes from onyx.utils.logger import setup_logger logger = setup_logger() +class UnsupportedImageFormatError(ValueError): + """Raised when an image uses a MIME type unsupported by the summarization flow.""" + + def prepare_image_bytes(image_data: bytes) -> str: """Prepare image bytes for summarization. Resizes image if it's larger than 20MB. Encodes image as a base64 string.""" @@ -74,7 +79,14 @@ def summarize_image_with_error_handling( user_prompt = ( f"The image has the file name '{context_name}'.\n{user_prompt_template}" ) - return summarize_image_pipeline(llm, image_data, user_prompt, system_prompt) + try: + return summarize_image_pipeline(llm, image_data, user_prompt, system_prompt) + except UnsupportedImageFormatError: + logger.info( + "Skipping image summarization due to unsupported MIME type for %s", + context_name, + ) + return None def _summarize_image( @@ -109,10 +121,17 @@ def _summarize_image( def _encode_image_for_llm_prompt(image_data: bytes) -> str: - """Getting the base64 string.""" + """Prepare a data URL with the correct MIME type for the LLM message.""" + try: + mime_type = get_image_type_from_bytes(image_data) + except ValueError as exc: + raise UnsupportedImageFormatError( + "Unsupported image format for summarization" + ) from exc + base64_encoded_data = base64.b64encode(image_data).decode("utf-8") - return f"data:image/jpeg;base64,{base64_encoded_data}" + return f"data:{mime_type};base64,{base64_encoded_data}" def _resize_image_if_needed(image_data: bytes, max_size_mb: int = 20) -> bytes: