[Feature Request: Brief Description]

[Anatolf]

Hello. The com.aallam.openai library treats content as a string, not as an array of JSON objects.
As a result, the attached image is not parsed.

fun analyzeImageWithGPT(apiKey: String, imageUrl: String): String = runBlocking {
    val openAI = OpenAI(apiKey)
    
    val request = ChatCompletionRequest(
        model = ModelId("gpt-4o-mini"),
        messages = listOf(
            ChatMessage(
                role = ChatRole.User,
                content = """
                [
                    {
                        "type": "text",
                        "text": "What is in this image?"
                    },
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "$imageUrl"
                        }
                    }
                ]
            """.trimIndent()
            )
        ),
        maxTokens = 300
    )
    
    val response = openAI.chatCompletion(request)
    
    response.choices.firstOrNull()?.message?.content ?: "No response"
}

Maybe I'm not making the request correctly, please tell me?

[ThanosFisherman]

Did you figure it out?

[Anatolf]

Try this:

private fun goTest() {
    val result = analyzeUrlImageWithGPT(GPT_API_KEY, "https://www.hmy.com/cdn-cgi/image/width=1200,format=auto/https://www.hmy.com/storage/article/images/19842/Nimbus-1.png")
    println("Gpt answer: $result")
}

private fun analyzeUrlImageWithGPT(apiKey: String, imageUrl: String): String = runBlocking {
    val client = HttpClient()

    try {
        val requestBody = """
    {
        "model": "gpt-4o-mini",
        "messages": [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "What is in this photo?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "$imageUrl"
                        }
                    }
                ]
            }
        ],
        "max_tokens": 300
    }
    """.trimIndent()

        // send Post
        val response: HttpResponse = client.post("https://api.openai.com/v1/chat/completions") {
            header(HttpHeaders.Authorization, "Bearer $apiKey")
            header(HttpHeaders.ContentType, ContentType.Application.Json)
            setBody(requestBody)
        }

        val responseBody = response.bodyAsText()

        val json = Json { ignoreUnknownKeys = true }
        val gptResponse = json.decodeFromString(GPTResponse.serializer(), responseBody)

        gptResponse.choices?.firstOrNull()?.message?.content ?: "No content found"

    } catch (e: Exception) {
        e.printStackTrace()
        "Error: ${e.message}"
    } finally {
        client.close()
    }
}

[ThanosFisherman]

Thank you. Your approach seems good at first glance, but I decided to stick with the library and ended up doing it as mentioned in this link #309 (comment)