[Vertex AI] Add URI-based file data support (#12886)

Author: andrewheard

FirebaseVertexAI/Sample/FunctionCallingSample/ViewModels/FunctionCallingViewModel.swift

@@ -156,7 +156,7 @@ class FunctionCallingViewModel: ObservableObject {
       case let .functionCall(functionCall):
         messages.insert(functionCall.chatMessage(), at: messages.count - 1)
         functionCalls.append(functionCall)
-      case .data, .functionResponse:
+      case .data, .fileData, .functionResponse:
         fatalError("Unsupported response content.")
       }
     }

FirebaseVertexAI/Sources/Chat.swift

@@ -153,7 +153,7 @@ public class Chat {
         case let .text(str):
           combinedText += str
 
-        case .data, .functionCall, .functionResponse:
+        case .data, .fileData, .functionCall, .functionResponse:
           // Don't combine it, just add to the content. If there's any text pending, add that as
           // a part.
           if !combinedText.isEmpty {

FirebaseVertexAI/Sources/ModelContent.swift

@@ -28,6 +28,13 @@ public struct ModelContent: Equatable {
     /// Data with a specified media type. Not all media types may be supported by the AI model.
     case data(mimetype: String, Data)
 
+    /// URI-based data with a specified media type.
+    ///
+    /// > Note: Supported media types depends on the model; see [supported file formats
+    /// > ](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/send-multimodal-prompts#media_requirements)
+    /// > for details.
+    case fileData(mimetype: String, uri: String)
+
     /// A predicted function call returned from the model.
     case functionCall(FunctionCall)
 
@@ -109,6 +116,7 @@ extension ModelContent.Part: Codable {
   enum CodingKeys: String, CodingKey {
     case text
     case inlineData
+    case fileData
     case functionCall
     case functionResponse
   }
@@ -118,6 +126,11 @@ extension ModelContent.Part: Codable {
     case bytes = "data"
   }
 
+  enum FileDataKeys: String, CodingKey {
+    case mimeType = "mime_type"
+    case uri = "file_uri"
+  }
+
   public func encode(to encoder: Encoder) throws {
     var container = encoder.container(keyedBy: CodingKeys.self)
     switch self {
@@ -130,6 +143,13 @@ extension ModelContent.Part: Codable {
       )
       try inlineDataContainer.encode(mimetype, forKey: .mimeType)
       try inlineDataContainer.encode(bytes, forKey: .bytes)
+    case let .fileData(mimetype: mimetype, url):
+      var fileDataContainer = container.nestedContainer(
+        keyedBy: FileDataKeys.self,
+        forKey: .fileData
+      )
+      try fileDataContainer.encode(mimetype, forKey: .mimeType)
+      try fileDataContainer.encode(url, forKey: .uri)
     case let .functionCall(functionCall):
       try container.encode(functionCall, forKey: .functionCall)
     case let .functionResponse(functionResponse):

FirebaseVertexAI/Tests/Unit/ModelContentTests.swift

@@ -0,0 +1,49 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import XCTest
+
+@testable import FirebaseVertexAI
+
+@available(iOS 15.0, macOS 11.0, macCatalyst 15.0, *)
+final class ModelContentTests: XCTestCase {
+  let encoder = JSONEncoder()
+
+  override func setUp() {
+    encoder.outputFormatting = .init(
+      arrayLiteral: .prettyPrinted, .sortedKeys, .withoutEscapingSlashes
+    )
+  }
+
+  // MARK: ModelContent.Part Encoding
+
+  func testEncodeFileDataPart() throws {
+    let mimeType = "image/jpeg"
+    let fileURI = "gs://test-bucket/image.jpg"
+    let fileDataPart = ModelContent.Part.fileData(mimetype: mimeType, uri: fileURI)
+
+    let jsonData = try encoder.encode(fileDataPart)
+
+    let json = try XCTUnwrap(String(data: jsonData, encoding: .utf8))
+    XCTAssertEqual(json, """
+    {
+      "fileData" : {
+        "file_uri" : "\(fileURI)",
+        "mime_type" : "\(mimeType)"
+      }
+    }
+    """)
+  }
+}

FirebaseVertexAI/Tests/Unit/VertexAIAPITests.swift

@@ -90,6 +90,10 @@ final class VertexAIAPITests: XCTestCase {
     let _ = try await genAI.generateContent(str)
     let _ = try await genAI.generateContent([str])
     let _ = try await genAI.generateContent(str, "abc", "def")
+    let _ = try await genAI.generateContent(
+      str,
+      ModelContent.Part.fileData(mimetype: "image/jpeg", uri: "gs://test-bucket/image.jpg")
+    )
     #if canImport(UIKit)
       _ = try await genAI.generateContent(UIImage())
       _ = try await genAI.generateContent([UIImage()])