Merge pull request #1000 from andreadimaio/main

Enable watsonx.ai to process ImageContent in UserMessage

Author: geoand

model-providers/watsonx/deployment/src/test/java/io/quarkiverse/langchain4j/watsonx/deployment/AiChatServiceTest.java

@@ -20,7 +20,11 @@
 import com.github.tomakehurst.wiremock.stubbing.Scenario;
 
 import dev.langchain4j.agent.tool.Tool;
+import dev.langchain4j.data.image.Image;
 import dev.langchain4j.data.message.AiMessage;
+import dev.langchain4j.data.message.ImageContent;
+import dev.langchain4j.data.message.ImageContent.DetailLevel;
+import dev.langchain4j.data.message.TextContent;
 import dev.langchain4j.data.message.ToolExecutionResultMessage;
 import dev.langchain4j.service.MemoryId;
 import dev.langchain4j.service.SystemMessage;
@@ -72,6 +76,12 @@ interface AIService {
         Multi<String> streaming(String text);
     }
 
+    @Singleton
+    @RegisterAiService(chatMemoryProviderSupplier = RegisterAiService.NoChatMemoryProviderSupplier.class)
+    interface ImageDescriptor {
+        String chat(Image image, @UserMessage String text);
+    }
+
     @Singleton
     @RegisterAiService(tools = Calculator.class)
     @SystemMessage("This is a systemMessage")
@@ -84,6 +94,9 @@ interface AIServiceWithTool {
     @Inject
     AIService aiService;
 
+    @Inject
+    ImageDescriptor imageDescriptor;
+
     @Inject
     AIServiceWithTool aiServiceWithTool;
 
@@ -92,7 +105,6 @@ interface AIServiceWithTool {
 
     @Singleton
     static class Calculator {
-
         @Tool("Execute the sum of two numbers")
         public int sum(int first, int second) {
             return first + second;
@@ -125,6 +137,53 @@ void chat() throws Exception {
         assertEquals("AI Response", aiService.chat("Hello"));
     }
 
+    @Test
+    void chat_with_image() throws Exception {
+
+        var messages = List.<TextChatMessage> of(
+                TextChatMessageUser.of(
+                        dev.langchain4j.data.message.UserMessage.from(
+                                TextContent.from("Tell me more about this image"),
+                                ImageContent.from("test", "jpeg", DetailLevel.LOW))));
+
+        var RESPONSE = """
+                {
+                    "id": "chat-0102753c2c33412fa639a6b0eb5401da",
+                    "model_id": "meta-llama/llama-3-2-90b-vision-instruct",
+                    "choices": [
+                        {
+                            "index": 0,
+                            "message": {
+                                "role": "assistant",
+                                "content": "The image depicts a white cat with yellow eyes."
+                            },
+                            "finish_reason": "stop"
+                        }
+                    ],
+                    "created": 1729517211,
+                    "model_version": "3.2.0",
+                    "created_at": "2024-10-21T13:26:57.471Z",
+                    "usage": {
+                        "completion_tokens": 123,
+                        "prompt_tokens": 6422,
+                        "total_tokens": 6545
+                    }
+                }""";
+
+        mockServers.mockWatsonxBuilder(WireMockUtil.URL_WATSONX_CHAT_API, 200)
+                .body(mapper.writeValueAsString(generateChatRequest(messages, null)))
+                .response(RESPONSE)
+                .build();
+
+        Image image = Image.builder()
+                .base64Data("test")
+                .mimeType("jpeg")
+                .build();
+
+        assertEquals("The image depicts a white cat with yellow eyes.",
+                imageDescriptor.chat(image, "Tell me more about this image"));
+    }
+
     @Test
     void chat_with_tool() throws Exception {
 

model-providers/watsonx/runtime/src/main/java/io/quarkiverse/langchain4j/watsonx/WatsonxUtils.java

@@ -1,10 +1,14 @@
 package io.quarkiverse.langchain4j.watsonx;
 
+import java.util.Base64;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.Callable;
 
 import jakarta.ws.rs.WebApplicationException;
 
+import dev.langchain4j.data.image.Image;
+import dev.langchain4j.internal.Utils;
 import io.quarkiverse.langchain4j.watsonx.bean.WatsonxError;
 import io.quarkiverse.langchain4j.watsonx.exception.WatsonxException;
 
@@ -44,4 +48,20 @@ public static <T> T retryOn(Callable<T> action) {
         }
         throw new RuntimeException("Failed after " + maxAttempts + " attempts");
     }
+
+    public static String base64Image(Image image) {
+
+        if (Objects.nonNull(image.base64Data()))
+            return image.base64Data();
+
+        try {
+            byte[] bytes = switch (image.url().getScheme()) {
+                case "http", "https", "file" -> Utils.readBytes(image.url().toString());
+                default -> throw new RuntimeException("The only supported image schemes are: [http, https, file]");
+            };
+            return Base64.getEncoder().encodeToString(bytes);
+        } catch (Exception e) {
+            throw new RuntimeException("Error converting the image to base64, see the log for more details", e);
+        }
+    }
 }

model-providers/watsonx/runtime/src/main/java/io/quarkiverse/langchain4j/watsonx/bean/TextChatMessage.java

@@ -1,5 +1,7 @@
 package io.quarkiverse.langchain4j.watsonx.bean;
 
+import static io.quarkiverse.langchain4j.watsonx.WatsonxUtils.base64Image;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -9,6 +11,7 @@
 import dev.langchain4j.data.message.AiMessage;
 import dev.langchain4j.data.message.ChatMessage;
 import dev.langchain4j.data.message.Content;
+import dev.langchain4j.data.message.ImageContent;
 import dev.langchain4j.data.message.SystemMessage;
 import dev.langchain4j.data.message.TextContent;
 import dev.langchain4j.data.message.ToolExecutionResultMessage;
@@ -125,7 +128,18 @@ public static TextChatMessageUser of(UserMessage userMessage) {
                                 "type", "text",
                                 "text", textContent.text()));
                     }
-                    case AUDIO, IMAGE, PDF, TEXT_FILE, VIDEO ->
+                    case IMAGE -> {
+                        var imageContent = ImageContent.class.cast(content);
+                        var base64 = "data:image/%s;base64,%s".formatted(
+                                imageContent.image().mimeType(),
+                                base64Image(imageContent.image()));
+                        values.add(Map.of(
+                                "type", "image_url",
+                                "image_url", Map.of(
+                                        "url", base64,
+                                        "detail", imageContent.detailLevel().name().toLowerCase())));
+                    }
+                    case AUDIO, PDF, TEXT_FILE, VIDEO ->
                         throw new UnsupportedOperationException("Unimplemented case: " + content.type());
                 }
             }