diff --git a/docs/readme.md b/docs/readme.md index 47a83a0c..02f82f42 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -115,6 +115,8 @@ Release Notes - Analytics: Removed deprecated `FirebaseAnalytics.ParameterGroupId` and `Parameter.Dispose` methods. - Auth: Removed deprecated `FirebaseUser.UpdateEmailAsync`. + - Firebase AI: Add support for image generation via Imagen. For more info, see + https://firebase.google.com/docs/ai-logic/generate-images-imagen - Firebase AI: Deprecated `CountTokensResponse.TotalBillableCharacters`, use `CountTokensResponse.TotalTokens` instead. - Messaging: Removed deprecated `FirebaseMessage.Dispose`, diff --git a/firebaseai/src/FirebaseAI.cs b/firebaseai/src/FirebaseAI.cs index eb5c6986..09ea4258 100644 --- a/firebaseai/src/FirebaseAI.cs +++ b/firebaseai/src/FirebaseAI.cs @@ -191,6 +191,28 @@ public LiveGenerativeModel GetLiveModel( liveGenerationConfig, tools, systemInstruction, requestOptions); } + + /// + /// Initializes an `ImagenModel` with the given parameters. + /// + /// - Important: Only Imagen 3 models (named `imagen-3.0-*`) are supported. + /// + /// The name of the Imagen 3 model to use, for example `"imagen-3.0-generate-002"`; + /// see [model versions](https://firebase.google.com/docs/vertex-ai/models) for a list of + /// supported Imagen 3 models. + /// Configuration options for generating images with Imagen. + /// Settings describing what types of potentially harmful content your model + /// should allow. + /// Configuration parameters for sending requests to the backend. + /// The initialized `ImagenModel` instance. + public ImagenModel GetImagenModel( + string modelName, + ImagenGenerationConfig? generationConfig = null, + ImagenSafetySettings? safetySettings = null, + RequestOptions? requestOptions = null) { + return new ImagenModel(_firebaseApp, _backend, modelName, + generationConfig, safetySettings, requestOptions); + } } } diff --git a/firebaseai/src/GenerativeModel.cs b/firebaseai/src/GenerativeModel.cs index d3cf9814..e45802e1 100644 --- a/firebaseai/src/GenerativeModel.cs +++ b/firebaseai/src/GenerativeModel.cs @@ -200,10 +200,11 @@ public Chat StartChat(IEnumerable history) { private async Task GenerateContentAsyncInternal( IEnumerable content, CancellationToken cancellationToken) { - HttpRequestMessage request = new(HttpMethod.Post, GetURL() + ":generateContent"); + HttpRequestMessage request = new(HttpMethod.Post, + HttpHelpers.GetURL(_firebaseApp, _backend, _modelName) + ":generateContent"); // Set the request headers - await SetRequestHeaders(request); + await HttpHelpers.SetRequestHeaders(request, _firebaseApp); // Set the content string bodyJson = MakeGenerateContentRequest(content); @@ -214,7 +215,7 @@ private async Task GenerateContentAsyncInternal( #endif var response = await _httpClient.SendAsync(request, cancellationToken); - await ValidateHttpResponse(response); + await HttpHelpers.ValidateHttpResponse(response); string result = await response.Content.ReadAsStringAsync(); @@ -225,40 +226,14 @@ private async Task GenerateContentAsyncInternal( return GenerateContentResponse.FromJson(result, _backend.Provider); } - // Helper function to throw an exception if the Http Response indicates failure. - // Useful as EnsureSuccessStatusCode can leave out relevant information. - private async Task ValidateHttpResponse(HttpResponseMessage response) { - if (response.IsSuccessStatusCode) { - return; - } - - // Status code indicates failure, try to read the content for more details - string errorContent = "No error content available."; - if (response.Content != null) { - try { - errorContent = await response.Content.ReadAsStringAsync(); - } catch (Exception readEx) { - // Handle being unable to read the content - errorContent = $"Failed to read error content: {readEx.Message}"; - } - } - - // Construct the exception with as much information as possible. - var ex = new HttpRequestException( - $"HTTP request failed with status code: {(int)response.StatusCode} ({response.ReasonPhrase}).\n" + - $"Error Content: {errorContent}" - ); - - throw ex; - } - private async IAsyncEnumerable GenerateContentStreamAsyncInternal( IEnumerable content, [EnumeratorCancellation] CancellationToken cancellationToken) { - HttpRequestMessage request = new(HttpMethod.Post, GetURL() + ":streamGenerateContent?alt=sse"); + HttpRequestMessage request = new(HttpMethod.Post, + HttpHelpers.GetURL(_firebaseApp, _backend, _modelName) + ":streamGenerateContent?alt=sse"); // Set the request headers - await SetRequestHeaders(request); + await HttpHelpers.SetRequestHeaders(request, _firebaseApp); // Set the content string bodyJson = MakeGenerateContentRequest(content); @@ -269,7 +244,7 @@ private async IAsyncEnumerable GenerateContentStreamAsy #endif var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); - await ValidateHttpResponse(response); + await HttpHelpers.ValidateHttpResponse(response); // We are expecting a Stream as the response, so handle that. using var stream = await response.Content.ReadAsStreamAsync(); @@ -291,10 +266,11 @@ private async IAsyncEnumerable GenerateContentStreamAsy private async Task CountTokensAsyncInternal( IEnumerable content, CancellationToken cancellationToken) { - HttpRequestMessage request = new(HttpMethod.Post, GetURL() + ":countTokens"); + HttpRequestMessage request = new(HttpMethod.Post, + HttpHelpers.GetURL(_firebaseApp, _backend, _modelName) + ":countTokens"); // Set the request headers - await SetRequestHeaders(request); + await HttpHelpers.SetRequestHeaders(request, _firebaseApp); // Set the content string bodyJson = MakeCountTokensRequest(content); @@ -305,7 +281,7 @@ private async Task CountTokensAsyncInternal( #endif var response = await _httpClient.SendAsync(request, cancellationToken); - await ValidateHttpResponse(response); + await HttpHelpers.ValidateHttpResponse(response); string result = await response.Content.ReadAsStringAsync(); @@ -316,33 +292,6 @@ private async Task CountTokensAsyncInternal( return CountTokensResponse.FromJson(result); } - private string GetURL() { - if (_backend.Provider == FirebaseAI.Backend.InternalProvider.VertexAI) { - return "https://firebasevertexai.googleapis.com/v1beta" + - "/projects/" + _firebaseApp.Options.ProjectId + - "/locations/" + _backend.Location + - "/publishers/google/models/" + _modelName; - } else if (_backend.Provider == FirebaseAI.Backend.InternalProvider.GoogleAI) { - return "https://firebasevertexai.googleapis.com/v1beta" + - "/projects/" + _firebaseApp.Options.ProjectId + - "/models/" + _modelName; - } else { - throw new NotSupportedException($"Missing support for backend: {_backend.Provider}"); - } - } - - private async Task SetRequestHeaders(HttpRequestMessage request) { - request.Headers.Add("x-goog-api-key", _firebaseApp.Options.ApiKey); - string version = FirebaseInterops.GetVersionInfoSdkVersion(); - request.Headers.Add("x-goog-api-client", $"gl-csharp/8.0 fire/{version}"); - if (FirebaseInterops.GetIsDataCollectionDefaultEnabled(_firebaseApp)) { - request.Headers.Add("X-Firebase-AppId", _firebaseApp.Options.AppId); - request.Headers.Add("X-Firebase-AppVersion", UnityEngine.Application.version); - } - // Add additional Firebase tokens to the header. - await FirebaseInterops.AddFirebaseTokensAsync(request, _firebaseApp); - } - private string MakeGenerateContentRequest(IEnumerable contents) { Dictionary jsonDict = MakeGenerateContentRequestAsDictionary(contents); return Json.Serialize(jsonDict); diff --git a/firebaseai/src/Imagen.meta b/firebaseai/src/Imagen.meta new file mode 100644 index 00000000..19bfc117 --- /dev/null +++ b/firebaseai/src/Imagen.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 7b3c5e22bd9534b68984f3443531862d +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/firebaseai/src/Imagen/ImagenConfig.cs b/firebaseai/src/Imagen/ImagenConfig.cs new file mode 100644 index 00000000..d9315e08 --- /dev/null +++ b/firebaseai/src/Imagen/ImagenConfig.cs @@ -0,0 +1,206 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; + +namespace Firebase.AI { + /// + /// An aspect ratio for images generated by Imagen. + /// + /// To specify an aspect ratio for generated images, set `AspectRatio` in + /// your `ImagenGenerationConfig`. See the [Cloud + /// documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/image/generate-images#aspect-ratio) + /// for more details and examples of the supported aspect ratios. + /// + public enum ImagenAspectRatio { + /// + /// Square (1:1) aspect ratio. + /// + /// Common uses for this aspect ratio include social media posts. + /// + Square1x1, + /// + /// Portrait widescreen (9:16) aspect ratio. + /// + /// This is the `Landscape16x9` aspect ratio rotated 90 degrees. This a relatively new aspect + /// ratio that has been popularized by short form video apps (for example, YouTube shorts). Use + /// this for tall objects with strong vertical orientations such as buildings, trees, waterfalls, + /// or other similar objects. + /// + Portrait9x16, + /// + /// Widescreen (16:9) aspect ratio. + /// + /// This ratio has replaced `Landscape4x3` as the most common aspect ratio for TVs, monitors, + /// and mobile phone screens (landscape). Use this aspect ratio when you want to capture more of + /// the background (for example, scenic landscapes). + /// + Landscape16x9, + /// + /// Portrait full screen (3:4) aspect ratio. + /// + /// This is the `Landscape4x3` aspect ratio rotated 90 degrees. This lets to capture more of + /// the scene vertically compared to the `Square1x1` aspect ratio. + /// + Portrait3x4, + /// + /// Fullscreen (4:3) aspect ratio. + /// + /// This aspect ratio is commonly used in media or film. It is also the dimensions of most old + /// (non-widescreen) TVs and medium format cameras. It captures more of the scene horizontally + /// (compared to `Square1x1`), making it a preferred aspect ratio for photography. + /// + Landscape4x3 + } + + /// + /// An image format for images generated by Imagen. + /// + /// To specify an image format for generated images, set `ImageFormat` in + /// your `ImagenGenerationConfig`. See the [Cloud + /// documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#output-options) + /// for more details. + /// + public readonly struct ImagenImageFormat { +#if !DOXYGEN + public string MimeType { get; } + public int? CompressionQuality { get; } +#endif + + private ImagenImageFormat(string mimeType, int? compressionQuality = null) { + MimeType = mimeType; + CompressionQuality = compressionQuality; + } + + /// + /// PNG image format. + /// + /// Portable Network Graphic (PNG) is a lossless image format, meaning no image data is lost + /// during compression. Images in PNG format are *typically* larger than JPEG images, though this + /// depends on the image content and JPEG compression quality. + /// + public static ImagenImageFormat Png() { + return new ImagenImageFormat("image/png"); + } + + /// + /// JPEG image format. + /// + /// Joint Photographic Experts Group (JPEG) is a lossy compression format, meaning some image data + /// is discarded during compression. Images in JPEG format are *typically* larger than PNG images, + /// though this depends on the image content and JPEG compression quality. + /// + /// The JPEG quality setting from 0 to 100, where `0` is highest level of + /// compression (lowest image quality, smallest file size) and `100` is the lowest level of + /// compression (highest image quality, largest file size); defaults to `75`. + public static ImagenImageFormat Jpeg(int? compressionQuality = null) { + return new ImagenImageFormat("image/jpeg", compressionQuality); + } + + /// + /// Intended for internal use only. + /// This method is used for serializing the object to JSON for the API request. + /// + internal Dictionary ToJson() { + Dictionary jsonDict = new() { + ["mimeType"] = MimeType + }; + if (CompressionQuality != null) { + jsonDict["compressionQuality"] = CompressionQuality.Value; + } + return jsonDict; + } + } + + /// + /// Configuration options for generating images with Imagen. + /// + /// See [Parameters for Imagen + /// models](https://firebase.google.com/docs/vertex-ai/model-parameters?platform=unity#imagen) to + /// learn about parameters available for use with Imagen models, including how to configure them. + /// + public readonly struct ImagenGenerationConfig { +#if !DOXYGEN + public string NegativePrompt { get; } + public int? NumberOfImages { get; } + public ImagenAspectRatio? AspectRatio { get; } + public ImagenImageFormat? ImageFormat { get; } + public bool? AddWatermark { get; } +#endif + + /// + /// Initializes configuration options for generating images with Imagen. + /// + /// Specifies elements to exclude from the generated image; + /// disabled if not specified. + /// The number of image samples to generate; + /// defaults to 1 if not specified. + /// The aspect ratio of generated images; + /// defaults to to square, 1:1. + /// The image format of generated images; + /// defaults to PNG. + /// Whether to add an invisible watermark to generated images; + /// the default value depends on the model. + public ImagenGenerationConfig( + string negativePrompt = null, + int? numberOfImages = null, + ImagenAspectRatio? aspectRatio = null, + ImagenImageFormat? imageFormat = null, + bool? addWatermark = null) { + NegativePrompt = negativePrompt; + NumberOfImages = numberOfImages; + AspectRatio = aspectRatio; + ImageFormat = imageFormat; + AddWatermark = addWatermark; + } + + private static string ConvertAspectRatio(ImagenAspectRatio aspectRatio) { + return aspectRatio switch { + ImagenAspectRatio.Square1x1 => "1:1", + ImagenAspectRatio.Portrait9x16 => "9:16", + ImagenAspectRatio.Landscape16x9 => "16:9", + ImagenAspectRatio.Portrait3x4 => "3:4", + ImagenAspectRatio.Landscape4x3 => "4:3", + _ => aspectRatio.ToString(), // Fallback + }; + } + + /// + /// Intended for internal use only. + /// This method is used for serializing the object to JSON for the API request. + /// + internal Dictionary ToJson() { + Dictionary jsonDict = new() { + ["sampleCount"] = NumberOfImages ?? 1 + }; + if (!string.IsNullOrEmpty(NegativePrompt)) { + jsonDict["negativePrompt"] = NegativePrompt; + } + if (AspectRatio != null) { + jsonDict["aspectRatio"] = ConvertAspectRatio(AspectRatio.Value); + } + if (ImageFormat != null) { + jsonDict["outputOptions"] = ImageFormat?.ToJson(); + } + if (AddWatermark != null) { + jsonDict["addWatermark"] = AddWatermark; + } + + return jsonDict; + } + } + +} diff --git a/firebaseai/src/Imagen/ImagenConfig.cs.meta b/firebaseai/src/Imagen/ImagenConfig.cs.meta new file mode 100644 index 00000000..ffab0125 --- /dev/null +++ b/firebaseai/src/Imagen/ImagenConfig.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 98ceaff0ff5394024bc30e00fb37fd0e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/firebaseai/src/Imagen/ImagenModel.cs b/firebaseai/src/Imagen/ImagenModel.cs new file mode 100644 index 00000000..b7a73469 --- /dev/null +++ b/firebaseai/src/Imagen/ImagenModel.cs @@ -0,0 +1,148 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Firebase.AI.Internal; +using Google.MiniJSON; + +namespace Firebase.AI { + + /// + /// Represents a remote Imagen model with the ability to generate images using text prompts. + /// + /// See the [generate images + /// documentation](https://firebase.google.com/docs/vertex-ai/generate-images-imagen?platform=unity) + /// for more details about the image generation capabilities offered by the Imagen model in the + /// Firebase AI SDK SDK. + /// + /// > Warning: For Firebase AI SDK, image generation using Imagen 3 models is in Public + /// Preview, which means that the feature is not subject to any SLA or deprecation policy and + /// could change in backwards-incompatible ways. + /// + public class ImagenModel { + private readonly FirebaseApp _firebaseApp; + private readonly FirebaseAI.Backend _backend; + private readonly string _modelName; + private readonly ImagenGenerationConfig? _generationConfig; + private readonly ImagenSafetySettings? _safetySettings; + private readonly RequestOptions? _requestOptions; + + private readonly HttpClient _httpClient; + + internal ImagenModel(FirebaseApp firebaseApp, + FirebaseAI.Backend backend, + string modelName, + ImagenGenerationConfig? generationConfig = null, + ImagenSafetySettings? safetySettings = null, + RequestOptions? requestOptions = null) { + _firebaseApp = firebaseApp; + _backend = backend; + _modelName = modelName; + _generationConfig = generationConfig; + _safetySettings = safetySettings; + _requestOptions = requestOptions; + + // Create a HttpClient using the timeout requested, or the default one. + _httpClient = new HttpClient() { + Timeout = requestOptions?.Timeout ?? RequestOptions.DefaultTimeout + }; + } + + /// + /// Generates images using the Imagen model and returns them as inline data. + /// + /// > Warning: For Firebase AI SDK, image generation using Imagen 3 models is in Public + /// Preview, which means that the feature is not subject to any SLA or deprecation policy and + /// could change in backwards-incompatible ways. + /// + /// A text prompt describing the image(s) to generate. + /// An optional token to cancel the operation. + /// The generated content response from the model. + /// Thrown when an error occurs during content generation. + public Task> GenerateImagesAsync( + string prompt, CancellationToken cancellationToken = default) { + return GenerateImagesAsyncInternal(prompt, cancellationToken); + } + + private async Task> GenerateImagesAsyncInternal( + string prompt, CancellationToken cancellationToken) { + HttpRequestMessage request = new(HttpMethod.Post, + HttpHelpers.GetURL(_firebaseApp, _backend, _modelName) + ":predict"); + + // Set the request headers + await HttpHelpers.SetRequestHeaders(request, _firebaseApp); + + // Set the content + string bodyJson = MakeGenerateImagenRequest(prompt); + request.Content = new StringContent(bodyJson, Encoding.UTF8, "application/json"); + +#if FIREBASE_LOG_REST_CALLS + UnityEngine.Debug.Log("Request:\n" + bodyJson); +#endif + + var response = await _httpClient.SendAsync(request, cancellationToken); + await HttpHelpers.ValidateHttpResponse(response); + + string result = await response.Content.ReadAsStringAsync(); + +#if FIREBASE_LOG_REST_CALLS + UnityEngine.Debug.Log("Response:\n" + result); +#endif + + return ImagenGenerationResponse.FromJson(result); + } + + private string MakeGenerateImagenRequest(string prompt) { + Dictionary jsonDict = MakeGenerateImagenRequestAsDictionary(prompt); + return Json.Serialize(jsonDict); + } + + private Dictionary MakeGenerateImagenRequestAsDictionary( + string prompt) { + Dictionary parameters = new(); + // Merge the settings into a single parameter dictionary + if (_generationConfig != null) { + _generationConfig?.ToJson().ToList() + .ForEach(x => parameters.Add(x.Key, x.Value)); + } else { + // We want the change the default behavior for sampleCount to return 1. + parameters["sampleCount"] = 1; + } + if (_safetySettings != null) { + _safetySettings?.ToJson().ToList() + .ForEach(x => parameters.Add(x.Key, x.Value)); + } + + Dictionary jsonDict = new() { + ["instances"] = new Dictionary() { + ["prompt"] = prompt, + } + }; + if (parameters.Count > 0) { + jsonDict["parameters"] = parameters; + } + + return jsonDict; + } + } + +} diff --git a/firebaseai/src/Imagen/ImagenModel.cs.meta b/firebaseai/src/Imagen/ImagenModel.cs.meta new file mode 100644 index 00000000..6ec200c6 --- /dev/null +++ b/firebaseai/src/Imagen/ImagenModel.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6367767758e66481e9108be4f0cc4256 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/firebaseai/src/Imagen/ImagenResponse.cs b/firebaseai/src/Imagen/ImagenResponse.cs new file mode 100644 index 00000000..4f7729ba --- /dev/null +++ b/firebaseai/src/Imagen/ImagenResponse.cs @@ -0,0 +1,144 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using Firebase.AI.Internal; +using Google.MiniJSON; +using UnityEngine; + +namespace Firebase.AI { + /// + /// An image generated by Imagen. + /// + public interface IImagenImage { + /// + /// The IANA standard MIME type of the image file; either `"image/png"` or `"image/jpeg"`. + /// + /// > Note: To request a different format, set `ImageFormat` in + /// your `ImagenGenerationConfig`. + /// + public string MimeType { get; } + } + + /// + /// An image generated by Imagen, represented as inline data. + /// + public readonly struct ImagenInlineImage : IImagenImage { + /// + /// The IANA standard MIME type of the image file; either `"image/png"` or `"image/jpeg"`. + /// + /// > Note: To request a different format, set `ImageFormat` in + /// your `ImagenGenerationConfig`. + /// + public string MimeType { get; } + /// + /// The image data in PNG or JPEG format. + /// + public byte[] Data { get; } + + /// + /// Convert the image data into a `UnityEngine.Texture2D`. + /// + /// + public UnityEngine.Texture2D AsTexture2D() { + var texture = new Texture2D(1, 1); + texture.LoadImage(Data); + return texture; + } + + private ImagenInlineImage(string mimeType, byte[] data) { + MimeType = mimeType; + Data = data; + } + + /// + /// Intended for internal use only. + /// This method is used for deserializing JSON responses and should not be called directly. + /// + internal static IImagenImage FromJson(Dictionary jsonDict) { + return new ImagenInlineImage( + jsonDict.ParseValue("mimeType", JsonParseOptions.ThrowEverything), + Convert.FromBase64String(jsonDict.ParseValue("bytesBase64Encoded", JsonParseOptions.ThrowEverything))); + } + } + + /// + /// A response from a request to generate images with Imagen. + /// + /// This type is returned from: + /// - `ImagenModel.GenerateImagesAsync(prompt)` where `T` is `ImagenInlineImage` + /// + public readonly struct ImagenGenerationResponse where T : IImagenImage { + /// + /// The images generated by Imagen; see `ImagenInlineImage`. + /// + /// > Important: The number of images generated may be fewer than the number requested if one or + /// more were filtered out; see `FilteredReason`. + /// + public IReadOnlyList Images { get; } + /// The reason, if any, that generated images were filtered out. + /// + /// This property will only be populated if fewer images were generated than were requested, + /// otherwise it will be `null`. Images may be filtered out due to the `SafetyFilterLevel`, + /// the `PersonFilterLevel`, or filtering included in the model. The filter levels may be + /// adjusted in your `ImagenSafetySettings`. See the [Responsible AI and usage guidelines for + /// Imagen](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen) + /// for more details. + public string FilteredReason { get; } + + private ImagenGenerationResponse(List images, string filteredReason) { + Images = images ?? new List(); + FilteredReason = filteredReason; + } + + /// + /// Intended for internal use only. + /// This method is used for deserializing JSON responses and should not be called directly. + /// + internal static ImagenGenerationResponse FromJson(string jsonString) { + return FromJson(Json.Deserialize(jsonString) as Dictionary); + } + + /// + /// Intended for internal use only. + /// This method is used for deserializing JSON responses and should not be called directly. + /// + internal static ImagenGenerationResponse FromJson(Dictionary jsonDict) { + if (!jsonDict.ContainsKey("predictions") || jsonDict["predictions"] is not List) { + return new ImagenGenerationResponse(null, + "Model response missing predictions"); + } + + var predictions = (jsonDict["predictions"] as List) + .OfType>(); + + List images = new(); + string filteredReason = null; + + foreach (var pred in predictions) { + if (pred.ContainsKey("bytesBase64Encoded") && typeof(T) == typeof(ImagenInlineImage)) { + images.Add((T)ImagenInlineImage.FromJson(pred)); + } else if (pred.ContainsKey("raiFilteredReason")) { + filteredReason = pred.ParseValue("raiFilteredReason", JsonParseOptions.ThrowEverything); + } + } + + return new ImagenGenerationResponse(images, filteredReason); + } + } +} diff --git a/firebaseai/src/Imagen/ImagenResponse.cs.meta b/firebaseai/src/Imagen/ImagenResponse.cs.meta new file mode 100644 index 00000000..d0f70974 --- /dev/null +++ b/firebaseai/src/Imagen/ImagenResponse.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: ee017dc5719c946f19556e92e2fd5219 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/firebaseai/src/Imagen/ImagenSafety.cs b/firebaseai/src/Imagen/ImagenSafety.cs new file mode 100644 index 00000000..22e9c1c3 --- /dev/null +++ b/firebaseai/src/Imagen/ImagenSafety.cs @@ -0,0 +1,152 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; + +namespace Firebase.AI { + + /// Settings for controlling the aggressiveness of filtering out sensitive content. + /// + /// See the [Responsible AI and usage + /// guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#config-safety-filters) + /// for more details. + public readonly struct ImagenSafetySettings { + + /// + /// A filter level controlling how aggressively to filter sensitive content. + /// + /// Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI + /// are assessed against a list of safety filters, which include 'harmful categories' (for example, + /// `violence`, `sexual`, `derogatory`, and `toxic`). This filter level controls how aggressively to + /// filter out potentially harmful content from responses. See the + /// [`safetySetting`](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#parameter_list) + /// documentation and the [Responsible AI and usage + /// guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) + /// for more details. + /// + public enum SafetyFilterLevel { + /// + /// The most aggressive filtering level; most strict blocking. + /// + BlockLowAndAbove, + /// + /// Blocks some problematic prompts and responses. + /// + BlockMediumAndAbove, + /// + /// Reduces the number of requests blocked due to safety filters. + /// + /// > Important: This may increase objectionable content generated by Imagen. + /// + BlockOnlyHigh, + /// + /// The least aggressive filtering level; blocks very few problematic prompts and responses. + /// + /// > Important: Access to this feature is restricted and may require your use case to be reviewed + /// and approved by Cloud support. + /// + BlockNone + } + + /// + /// A filter level controlling whether generation of images containing people or faces is allowed. + /// + /// See the + /// [`personGeneration`](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#parameter_list) + /// documentation for more details. + /// + public enum PersonFilterLevel { + /// + /// Disallow generation of images containing people or faces; images of people are filtered out. + /// + BlockAll, + /// + /// Allow generation of images containing adults only; images of children are filtered out. + /// + /// > Important: Generation of images containing people or faces may require your use case to be + /// reviewed and approved by Cloud support; see the [Responsible AI and usage + /// guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#person-face-gen) + /// for more details. + /// + AllowAdult, + /// + /// Allow generation of images containing people of all ages. + /// + /// > Important: Generation of images containing people or faces may require your use case to be + /// reviewed and approved; see the [Responsible AI and usage + /// guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#person-face-gen) + /// for more details. + /// + AllowAll + } + +#if !DOXYGEN + public SafetyFilterLevel? SafetyFilter { get; } + public PersonFilterLevel? PersonFilter { get; } +#endif + + /// + /// Initializes safety settings for the Imagen model. + /// + /// A filter level controlling how aggressively to + /// filter out sensitive content from generated images. + /// A filter level controlling whether generation + /// of images containing people or faces is allowed. + public ImagenSafetySettings( + SafetyFilterLevel? safetyFilterLevel = null, + PersonFilterLevel? personFilterLevel = null) { + SafetyFilter = safetyFilterLevel; + PersonFilter = personFilterLevel; + } + + private static string ConvertSafetyFilter(SafetyFilterLevel safetyFilter) { + return safetyFilter switch { + SafetyFilterLevel.BlockLowAndAbove => "block_low_and_above", + SafetyFilterLevel.BlockMediumAndAbove => "block_medium_and_above", + SafetyFilterLevel.BlockOnlyHigh => "block_only_high", + SafetyFilterLevel.BlockNone => "block_none", + _ => safetyFilter.ToString(), // Fallback + }; + } + + private static string ConvertPersonFilter(PersonFilterLevel safetyFilter) { + return safetyFilter switch { + PersonFilterLevel.BlockAll => "dont_allow", + PersonFilterLevel.AllowAdult => "allow_adult", + PersonFilterLevel.AllowAll => "allow_all", + _ => safetyFilter.ToString(), // Fallback + }; + } + + /// + /// Intended for internal use only. + /// This method is used for serializing the object to JSON for the API request. + /// + internal Dictionary ToJson() { + Dictionary jsonDict = new(); + + if (PersonFilter != null) { + jsonDict["personGeneration"] = ConvertPersonFilter(PersonFilter.Value); + } + if (SafetyFilter != null) { + jsonDict["safetySetting"] = ConvertSafetyFilter(SafetyFilter.Value); + } + + return jsonDict; + } + } + +} diff --git a/firebaseai/src/Imagen/ImagenSafety.cs.meta b/firebaseai/src/Imagen/ImagenSafety.cs.meta new file mode 100644 index 00000000..4e5534c4 --- /dev/null +++ b/firebaseai/src/Imagen/ImagenSafety.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 3f224beb821e84e54bb3214a44cd2f9a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/firebaseai/src/Internal/HttpHelpers.cs b/firebaseai/src/Internal/HttpHelpers.cs new file mode 100644 index 00000000..370ca165 --- /dev/null +++ b/firebaseai/src/Internal/HttpHelpers.cs @@ -0,0 +1,80 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Net.Http; +using System.Threading.Tasks; + +namespace Firebase.AI.Internal { + // Helper functions to help handling the Http calls. + internal static class HttpHelpers { + // Get the URL to use for the rest calls based on the backend. + internal static string GetURL(FirebaseApp firebaseApp, + FirebaseAI.Backend backend, string modelName) { + if (backend.Provider == FirebaseAI.Backend.InternalProvider.VertexAI) { + return "https://firebasevertexai.googleapis.com/v1beta" + + "/projects/" + firebaseApp.Options.ProjectId + + "/locations/" + backend.Location + + "/publishers/google/models/" + modelName; + } else if (backend.Provider == FirebaseAI.Backend.InternalProvider.GoogleAI) { + return "https://firebasevertexai.googleapis.com/v1beta" + + "/projects/" + firebaseApp.Options.ProjectId + + "/models/" + modelName; + } else { + throw new NotSupportedException($"Missing support for backend: {backend.Provider}"); + } + } + + internal static async Task SetRequestHeaders(HttpRequestMessage request, FirebaseApp firebaseApp) { + request.Headers.Add("x-goog-api-key", firebaseApp.Options.ApiKey); + string version = FirebaseInterops.GetVersionInfoSdkVersion(); + request.Headers.Add("x-goog-api-client", $"gl-csharp/8.0 fire/{version}"); + if (FirebaseInterops.GetIsDataCollectionDefaultEnabled(firebaseApp)) { + request.Headers.Add("X-Firebase-AppId", firebaseApp.Options.AppId); + request.Headers.Add("X-Firebase-AppVersion", UnityEngine.Application.version); + } + // Add additional Firebase tokens to the header. + await FirebaseInterops.AddFirebaseTokensAsync(request, firebaseApp); + } + + // Helper function to throw an exception if the Http Response indicates failure. + // Useful as EnsureSuccessStatusCode can leave out relevant information. + internal static async Task ValidateHttpResponse(HttpResponseMessage response) { + if (response.IsSuccessStatusCode) { + return; + } + + // Status code indicates failure, try to read the content for more details + string errorContent = "No error content available."; + if (response.Content != null) { + try { + errorContent = await response.Content.ReadAsStringAsync(); + } catch (Exception readEx) { + // Handle being unable to read the content + errorContent = $"Failed to read error content: {readEx.Message}"; + } + } + + // Construct the exception with as much information as possible. + var ex = new HttpRequestException( + $"HTTP request failed with status code: {(int)response.StatusCode} ({response.ReasonPhrase}).\n" + + $"Error Content: {errorContent}" + ); + + throw ex; + } + } +} diff --git a/firebaseai/src/Internal/HttpHelpers.cs.meta b/firebaseai/src/Internal/HttpHelpers.cs.meta new file mode 100644 index 00000000..06286826 --- /dev/null +++ b/firebaseai/src/Internal/HttpHelpers.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 41bec814fadbb4099bc7489060279f26 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs b/firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs index 9a1a7006..5f95bcda 100644 --- a/firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs +++ b/firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs @@ -70,6 +70,8 @@ protected override void Start() { TestCountTokens, TestYoutubeLink, TestGenerateImage, + TestImagenGenerateImage, + TestImagenGenerateImageOptions }; // Set of tests that only run the single time. Func[] singleTests = { @@ -86,6 +88,9 @@ protected override void Start() { InternalTestBasicResponseLongUsageMetadata, InternalTestGoogleAIBasicReplyShort, InternalTestGoogleAICitations, + InternalTestGenerateImagesBase64, + InternalTestGenerateImagesAllFiltered, + InternalTestGenerateImagesBase64SomeFiltered, }; // Create the set of tests, combining the above lists. @@ -678,6 +683,58 @@ async Task TestGenerateImage(Backend backend) { Assert($"Missing expected modalities. Text: {foundText}, Image: {foundImage}", foundText && foundImage); } + async Task TestImagenGenerateImage(Backend backend) { + var model = GetFirebaseAI(backend).GetImagenModel("imagen-3.0-generate-002"); + + var response = await model.GenerateImagesAsync( + "Generate an image of a cartoon dog."); + + // We can't easily test if the image is correct, but can check other random data. + AssertEq("FilteredReason", response.FilteredReason, null); + AssertEq("Image Count", response.Images.Count, 1); + + AssertEq($"Image MimeType", response.Images[0].MimeType, "image/png"); + + var texture = response.Images[0].AsTexture2D(); + Assert($"Image as Texture2D", texture != null); + // By default the image should be Square 1x1, so check for that. + Assert($"Image Height > 0", texture.height > 0); + AssertEq($"Image Height = Width", texture.height, texture.width); + } + + async Task TestImagenGenerateImageOptions(Backend backend) { + var model = GetFirebaseAI(backend).GetImagenModel( + modelName: "imagen-3.0-generate-002", + generationConfig: new ImagenGenerationConfig( + // negativePrompt and addWatermark are not supported on this version of the model. + numberOfImages: 2, + aspectRatio: ImagenAspectRatio.Landscape4x3, + imageFormat: ImagenImageFormat.Jpeg(50) + ), + safetySettings: new ImagenSafetySettings( + safetyFilterLevel: ImagenSafetySettings.SafetyFilterLevel.BlockLowAndAbove, + personFilterLevel: ImagenSafetySettings.PersonFilterLevel.BlockAll), + requestOptions: new RequestOptions(timeout: TimeSpan.FromMinutes(1))); + + var response = await model.GenerateImagesAsync( + "Generate an image of a cartoon dog."); + + // We can't easily test if the image is correct, but can check other random data. + AssertEq("FilteredReason", response.FilteredReason, null); + AssertEq("Image Count", response.Images.Count, 2); + + for (int i = 0; i < 2; i++) { + AssertEq($"Image {i} MimeType", response.Images[i].MimeType, "image/jpeg"); + + var texture = response.Images[i].AsTexture2D(); + Assert($"Image {i} as Texture2D", texture != null); + // By default the image should be Landscape 4x3, so check for that. + Assert($"Image {i} Height > 0", texture.height > 0); + Assert($"Image {i} Height < Width {texture.height} < {texture.width}", + texture.height < texture.width); + } + } + // Test providing a file from a GCS bucket (Firebase Storage) to the model. async Task TestReadFile() { // GCS is currently only supported with VertexAI. @@ -1173,5 +1230,54 @@ async Task InternalTestGoogleAICitations() { AssertEq("CandidatesTokensDetails[0].Modality", candidatesDetails[0].Modality, ContentModality.Text); AssertEq("CandidatesTokensDetails[0].TokenCount", candidatesDetails[0].TokenCount, 1667); } + + async Task InternalTestGenerateImagesBase64() { + Dictionary json = await GetVertexJsonTestData("unary-success-generate-images-base64.json"); + var response = ImagenGenerationResponse.FromJson(json); + + AssertEq("FilteredReason", response.FilteredReason, null); + AssertEq("Image Count", response.Images.Count, 4); + + for (int i = 0; i < response.Images.Count; i++) { + var image = response.Images[i]; + AssertEq($"Image {i} MimeType", image.MimeType, "image/png"); + Assert($"Image {i} Length: {image.Data.Length}", image.Data.Length > 0); + + var texture = image.AsTexture2D(); + Assert($"Failed to convert Image {i}", texture != null); + } + } + + async Task InternalTestGenerateImagesAllFiltered() { + Dictionary json = await GetVertexJsonTestData("unary-failure-generate-images-all-filtered.json"); + var response = ImagenGenerationResponse.FromJson(json); + + AssertEq("FilteredReason", response.FilteredReason, + "Unable to show generated images. All images were filtered out because " + + "they violated Vertex AI's usage guidelines. You will not be charged for " + + "blocked images. Try rephrasing the prompt. If you think this was an error, " + + "send feedback. Support codes: 39322892, 29310472"); + AssertEq("Image Count", response.Images.Count, 0); + } + + async Task InternalTestGenerateImagesBase64SomeFiltered() { + Dictionary json = await GetVertexJsonTestData("unary-failure-generate-images-base64-some-filtered.json"); + var response = ImagenGenerationResponse.FromJson(json); + + AssertEq("FilteredReason", response.FilteredReason, + "Your current safety filter threshold filtered out 2 generated images. " + + "You will not be charged for blocked images. Try rephrasing the prompt. " + + "If you think this was an error, send feedback."); + AssertEq("Image Count", response.Images.Count, 2); + + for (int i = 0; i < response.Images.Count; i++) { + var image = response.Images[i]; + AssertEq($"Image {i} MimeType", image.MimeType, "image/png"); + Assert($"Image {i} Length: {image.Data.Length}", image.Data.Length > 0); + + var texture = image.AsTexture2D(); + Assert($"Failed to convert Image {i}", texture != null); + } + } } }