diff --git a/.changeset/five-kids-grow.md b/.changeset/five-kids-grow.md new file mode 100644 index 00000000000..4303518b6ff --- /dev/null +++ b/.changeset/five-kids-grow.md @@ -0,0 +1,6 @@ +--- +'firebase': minor +'@firebase/ai': minor +--- + +Add support for Grounding with Google Search. diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 23c13688fb1..7fec37c12a8 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -485,6 +485,15 @@ export interface GoogleAIGenerateContentResponse { usageMetadata?: UsageMetadata; } +// @public +export interface GoogleSearch { +} + +// @public +export interface GoogleSearchTool { + googleSearch: GoogleSearch; +} + // @public @deprecated (undocumented) export interface GroundingAttribution { // (undocumented) @@ -497,16 +506,29 @@ export interface GroundingAttribution { web?: WebAttribution; } +// @public +export interface GroundingChunk { + web?: WebGroundingChunk; +} + // @public export interface GroundingMetadata { // @deprecated (undocumented) groundingAttributions: GroundingAttribution[]; - // (undocumented) + groundingChunks?: GroundingChunk[]; + groundingSupports?: GroundingSupport[]; + // @deprecated (undocumented) retrievalQueries?: string[]; - // (undocumented) + searchEntryPoint?: SearchEntrypoint; webSearchQueries?: string[]; } +// @public +export interface GroundingSupport { + groundingChunkIndices?: number[]; + segment?: Segment; +} + // @public export enum HarmBlockMethod { PROBABILITY = "PROBABILITY", @@ -857,14 +879,17 @@ export enum SchemaType { STRING = "string" } -// @public (undocumented) +// @public +export interface SearchEntrypoint { + renderedContent?: string; +} + +// @public export interface Segment { - // (undocumented) endIndex: number; - // (undocumented) partIndex: number; - // (undocumented) startIndex: number; + text: string; } // @public @@ -901,7 +926,7 @@ export interface TextPart { } // @public -export type Tool = FunctionDeclarationsTool; +export type Tool = FunctionDeclarationsTool | GoogleSearchTool; // @public export interface ToolConfig { @@ -961,5 +986,12 @@ export interface WebAttribution { uri: string; } +// @public +export interface WebGroundingChunk { + domain?: string; + title?: string; + uri?: string; +} + ``` diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index b77a6b5910e..bf1b8978b57 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -76,10 +76,18 @@ toc: path: /docs/reference/js/ai.generativemodel.md - title: GoogleAIBackend path: /docs/reference/js/ai.googleaibackend.md + - title: GoogleSearch + path: /docs/reference/js/ai.googlesearch.md + - title: GoogleSearchTool + path: /docs/reference/js/ai.googlesearchtool.md - title: GroundingAttribution path: /docs/reference/js/ai.groundingattribution.md + - title: GroundingChunk + path: /docs/reference/js/ai.groundingchunk.md - title: GroundingMetadata path: /docs/reference/js/ai.groundingmetadata.md + - title: GroundingSupport + path: /docs/reference/js/ai.groundingsupport.md - title: ImagenGCSImage path: /docs/reference/js/ai.imagengcsimage.md - title: ImagenGenerationConfig @@ -130,6 +138,8 @@ toc: path: /docs/reference/js/ai.schemarequest.md - title: SchemaShared path: /docs/reference/js/ai.schemashared.md + - title: SearchEntrypoint + path: /docs/reference/js/ai.searchentrypoint.md - title: Segment path: /docs/reference/js/ai.segment.md - title: StartChatParams @@ -150,6 +160,8 @@ toc: path: /docs/reference/js/ai.videometadata.md - title: WebAttribution path: /docs/reference/js/ai.webattribution.md + - title: WebGroundingChunk + path: /docs/reference/js/ai.webgroundingchunk.md - title: analytics path: /docs/reference/js/analytics.md section: diff --git a/docs-devsite/ai.googlesearch.md b/docs-devsite/ai.googlesearch.md new file mode 100644 index 00000000000..f2c206f9bfe --- /dev/null +++ b/docs-devsite/ai.googlesearch.md @@ -0,0 +1,21 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# GoogleSearch interface +Configuration for the [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface). + +Currently, this interface is empty and serves as a placeholder for future configuration options. + +Signature: + +```typescript +export declare interface GoogleSearch +``` diff --git a/docs-devsite/ai.googlesearchtool.md b/docs-devsite/ai.googlesearchtool.md new file mode 100644 index 00000000000..a24ead3c680 --- /dev/null +++ b/docs-devsite/ai.googlesearchtool.md @@ -0,0 +1,37 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# GoogleSearchTool interface +A tool that allows the generative model to connect to Google Search to access and incorporate up-to-date information from the web into its responses. + +When using Grounding with Google Search, you may receive a response with data generated from Google's search engine. This response is a "Grounded Result" subject to the Grounding with Google Search terms in the [Service Specific Terms](https://cloud.google.com/terms/service-terms). + +Signature: + +```typescript +export declare interface GoogleSearchTool +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [googleSearch](./ai.googlesearchtool.md#googlesearchtoolgooglesearch) | [GoogleSearch](./ai.googlesearch.md#googlesearch_interface) | Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. | + +## GoogleSearchTool.googleSearch + +Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. + +Signature: + +```typescript +googleSearch: GoogleSearch; +``` diff --git a/docs-devsite/ai.groundingchunk.md b/docs-devsite/ai.groundingchunk.md new file mode 100644 index 00000000000..c2f4da04876 --- /dev/null +++ b/docs-devsite/ai.groundingchunk.md @@ -0,0 +1,37 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# GroundingChunk interface +Represents a chunk of retrieved data that supports a claim in the model's response. This is part of the grounding information provided when grounding is enabled. + +Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". + +Signature: + +```typescript +export interface GroundingChunk +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [web](./ai.groundingchunk.md#groundingchunkweb) | [WebGroundingChunk](./ai.webgroundingchunk.md#webgroundingchunk_interface) | Contains details if the grounding chunk is from a web source. | + +## GroundingChunk.web + +Contains details if the grounding chunk is from a web source. + +Signature: + +```typescript +web?: WebGroundingChunk; +``` diff --git a/docs-devsite/ai.groundingmetadata.md b/docs-devsite/ai.groundingmetadata.md index 90994d9c01c..3b080fa6f84 100644 --- a/docs-devsite/ai.groundingmetadata.md +++ b/docs-devsite/ai.groundingmetadata.md @@ -10,7 +10,11 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # GroundingMetadata interface -Metadata returned to client when grounding is enabled. +Metadata returned when grounding is enabled. + +Currently, the only way to use grounding is to include a [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface) in your [GenerationConfig](./ai.generationconfig.md#generationconfig_interface). + +Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". Signature: @@ -23,13 +27,17 @@ export interface GroundingMetadata | Property | Type | Description | | --- | --- | --- | | [groundingAttributions](./ai.groundingmetadata.md#groundingmetadatagroundingattributions) | [GroundingAttribution](./ai.groundingattribution.md#groundingattribution_interface)\[\] | | +| [groundingChunks](./ai.groundingmetadata.md#groundingmetadatagroundingchunks) | [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface)\[\] | A list of [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) objects. Each chunk represents a piece of retrieved content (e.g. from a web page). that the model used to ground its response.Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". | +| [groundingSupports](./ai.groundingmetadata.md#groundingmetadatagroundingsupports) | [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface)\[\] | A list of [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) objects. Each object details how specific segments of the model's response are supported by the groundingChunks.Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". | | [retrievalQueries](./ai.groundingmetadata.md#groundingmetadataretrievalqueries) | string\[\] | | -| [webSearchQueries](./ai.groundingmetadata.md#groundingmetadatawebsearchqueries) | string\[\] | | +| [searchEntryPoint](./ai.groundingmetadata.md#groundingmetadatasearchentrypoint) | [SearchEntrypoint](./ai.searchentrypoint.md#searchentrypoint_interface) | Google search entry point for web searches. This contains An HTML/CSS snippet that \*must\* be embedded in an app to display a Google Search Entry point for follow-up web searches related to the model's "Grounded Response".Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". | +| [webSearchQueries](./ai.groundingmetadata.md#groundingmetadatawebsearchqueries) | string\[\] | A list of web search queries that the model performed to gather the grounding information. These can be used to allow users to explore the search results themselves.Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". | ## GroundingMetadata.groundingAttributions > Warning: This API is now obsolete. > +> Use [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) instead. > Signature: @@ -38,16 +46,61 @@ export interface GroundingMetadata groundingAttributions: GroundingAttribution[]; ``` +## GroundingMetadata.groundingChunks + +A list of [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) objects. Each chunk represents a piece of retrieved content (e.g. from a web page). that the model used to ground its response. + +Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". + +Signature: + +```typescript +groundingChunks?: GroundingChunk[]; +``` + +## GroundingMetadata.groundingSupports + +A list of [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) objects. Each object details how specific segments of the model's response are supported by the `groundingChunks`. + +Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". + +Signature: + +```typescript +groundingSupports?: GroundingSupport[]; +``` + ## GroundingMetadata.retrievalQueries +> Warning: This API is now obsolete. +> +> Use [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) instead. +> + Signature: ```typescript retrievalQueries?: string[]; ``` +## GroundingMetadata.searchEntryPoint + +Google search entry point for web searches. This contains An HTML/CSS snippet that \*must\* be embedded in an app to display a Google Search Entry point for follow-up web searches related to the model's "Grounded Response". + +Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". + +Signature: + +```typescript +searchEntryPoint?: SearchEntrypoint; +``` + ## GroundingMetadata.webSearchQueries +A list of web search queries that the model performed to gather the grounding information. These can be used to allow users to explore the search results themselves. + +Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". + Signature: ```typescript diff --git a/docs-devsite/ai.groundingsupport.md b/docs-devsite/ai.groundingsupport.md new file mode 100644 index 00000000000..b9a15111651 --- /dev/null +++ b/docs-devsite/ai.groundingsupport.md @@ -0,0 +1,48 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# GroundingSupport interface +Provides information about how a specific segment of the model's response is supported by the retrieved grounding chunks. + +Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". + +Signature: + +```typescript +export interface GroundingSupport +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [groundingChunkIndices](./ai.groundingsupport.md#groundingsupportgroundingchunkindices) | number\[\] | A list of indices that refer to specific [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) objects within the [GroundingMetadata.groundingChunks](./ai.groundingmetadata.md#groundingmetadatagroundingchunks) array. These referenced chunks are the sources that support the claim made in the associated segment of the response. For example, an array [1, 3, 4] means that groundingChunks[1], groundingChunks[3], and groundingChunks[4] are the retrieved content supporting this part of the response. | +| [segment](./ai.groundingsupport.md#groundingsupportsegment) | [Segment](./ai.segment.md#segment_interface) | Specifies the segment of the model's response content that this grounding support pertains to. | + +## GroundingSupport.groundingChunkIndices + +A list of indices that refer to specific [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) objects within the [GroundingMetadata.groundingChunks](./ai.groundingmetadata.md#groundingmetadatagroundingchunks) array. These referenced chunks are the sources that support the claim made in the associated `segment` of the response. For example, an array `[1, 3, 4]` means that `groundingChunks[1]`, `groundingChunks[3]`, and `groundingChunks[4]` are the retrieved content supporting this part of the response. + +Signature: + +```typescript +groundingChunkIndices?: number[]; +``` + +## GroundingSupport.segment + +Specifies the segment of the model's response content that this grounding support pertains to. + +Signature: + +```typescript +segment?: Segment; +``` diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index 286c8351fd7..c8fa00d7445 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -95,8 +95,12 @@ The Firebase AI Web SDK. | [GenerateContentStreamResult](./ai.generatecontentstreamresult.md#generatecontentstreamresult_interface) | Result object returned from [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) call. Iterate over stream to get chunks as they come in and/or use the response promise to get the aggregated response when the stream is done. | | [GenerationConfig](./ai.generationconfig.md#generationconfig_interface) | Config options for content-related requests | | [GenerativeContentBlob](./ai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | +| [GoogleSearch](./ai.googlesearch.md#googlesearch_interface) | Configuration for the [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface). | +| [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface) | A tool that allows the generative model to connect to Google Search to access and incorporate up-to-date information from the web into its responses.When using Grounding with Google Search, you may receive a response with data generated from Google's search engine. This response is a "Grounded Result" subject to the Grounding with Google Search terms in the [Service Specific Terms](https://cloud.google.com/terms/service-terms). | | [GroundingAttribution](./ai.groundingattribution.md#groundingattribution_interface) | | -| [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | +| [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) | Represents a chunk of retrieved data that supports a claim in the model's response. This is part of the grounding information provided when grounding is enabled.Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". | +| [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned when grounding is enabled.Currently, the only way to use grounding is to include a [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface) in your [GenerationConfig](./ai.generationconfig.md#generationconfig_interface).Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". | +| [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) | Provides information about how a specific segment of the model's response is supported by the retrieved grounding chunks.Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". | | [ImagenGCSImage](./ai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | | [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. | | [ImagenGenerationResponse](./ai.imagengenerationresponse.md#imagengenerationresponse_interface) | (Public Preview) The response from a request to generate images with Imagen. | @@ -116,7 +120,8 @@ The Firebase AI Web SDK. | [SchemaParams](./ai.schemaparams.md#schemaparams_interface) | Params passed to [Schema](./ai.schema.md#schema_class) static methods to create specific [Schema](./ai.schema.md#schema_class) classes. | | [SchemaRequest](./ai.schemarequest.md#schemarequest_interface) | Final format for [Schema](./ai.schema.md#schema_class) params passed to backend requests. | | [SchemaShared](./ai.schemashared.md#schemashared_interface) | Basic [Schema](./ai.schema.md#schema_class) properties shared across several Schema-related types. | -| [Segment](./ai.segment.md#segment_interface) | | +| [SearchEntrypoint](./ai.searchentrypoint.md#searchentrypoint_interface) | Google search entry point.Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". | +| [Segment](./ai.segment.md#segment_interface) | Represents a specific segment within a [Content](./ai.content.md#content_interface) object, often used to pinpoint the exact location of text or data that grounding information refers to. | | [StartChatParams](./ai.startchatparams.md#startchatparams_interface) | Params for [GenerativeModel.startChat()](./ai.generativemodel.md#generativemodelstartchat). | | [TextPart](./ai.textpart.md#textpart_interface) | Content part interface if the part represents a text string. | | [ToolConfig](./ai.toolconfig.md#toolconfig_interface) | Tool config. This config is shared for all tools provided in the request. | @@ -124,6 +129,7 @@ The Firebase AI Web SDK. | [VertexAIOptions](./ai.vertexaioptions.md#vertexaioptions_interface) | Options when initializing the Firebase AI SDK. | | [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Describes the input video content. | | [WebAttribution](./ai.webattribution.md#webattribution_interface) | | +| [WebGroundingChunk](./ai.webgroundingchunk.md#webgroundingchunk_interface) | A grounding chunk from the web.Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". | ## Variables @@ -400,7 +406,7 @@ Defines a tool that model can call to access external knowledge. Signature: ```typescript -export declare type Tool = FunctionDeclarationsTool; +export declare type Tool = FunctionDeclarationsTool | GoogleSearchTool; ``` ## TypedSchema diff --git a/docs-devsite/ai.searchentrypoint.md b/docs-devsite/ai.searchentrypoint.md new file mode 100644 index 00000000000..138778b0834 --- /dev/null +++ b/docs-devsite/ai.searchentrypoint.md @@ -0,0 +1,50 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# SearchEntrypoint interface +Google search entry point. + +Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". + +Signature: + +```typescript +export interface SearchEntrypoint +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [renderedContent](./ai.searchentrypoint.md#searchentrypointrenderedcontent) | string | HTML/CSS snippet that can be embedded in a web page. The snippet is designed to avoid undesired interaction with the rest of the page's CSS.To ensure proper rendering and prevent CSS conflicts, it is recommended to encapsulate this renderedContent within a shadow DOM when embedding it into a webpage. See [MDN: Using shadow DOM](https://developer.mozilla.org/en-US/docs/Web/API/Web_components/Using_shadow_DOM). | + +## SearchEntrypoint.renderedContent + +HTML/CSS snippet that can be embedded in a web page. The snippet is designed to avoid undesired interaction with the rest of the page's CSS. + +To ensure proper rendering and prevent CSS conflicts, it is recommended to encapsulate this `renderedContent` within a shadow DOM when embedding it into a webpage. See [MDN: Using shadow DOM](https://developer.mozilla.org/en-US/docs/Web/API/Web_components/Using_shadow_DOM). + +Signature: + +```typescript +renderedContent?: string; +``` + +### Example + + +```javascript +const container = document.createElement('div'); +document.body.appendChild(container); +container.attachShadow({ mode: 'open' }).innerHTML = renderedContent; + +``` + diff --git a/docs-devsite/ai.segment.md b/docs-devsite/ai.segment.md index 69f4aaf8407..35db1be5e83 100644 --- a/docs-devsite/ai.segment.md +++ b/docs-devsite/ai.segment.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # Segment interface +Represents a specific segment within a [Content](./ai.content.md#content_interface) object, often used to pinpoint the exact location of text or data that grounding information refers to. Signature: @@ -21,12 +22,15 @@ export interface Segment | Property | Type | Description | | --- | --- | --- | -| [endIndex](./ai.segment.md#segmentendindex) | number | | -| [partIndex](./ai.segment.md#segmentpartindex) | number | | -| [startIndex](./ai.segment.md#segmentstartindex) | number | | +| [endIndex](./ai.segment.md#segmentendindex) | number | The zero-based end index of the segment within the specified Part, measured in UTF-8 bytes. This offset is exclusive, meaning the character at this index is not included in the segment. | +| [partIndex](./ai.segment.md#segmentpartindex) | number | The zero-based index of the [Part](./ai.md#part) object within the parts array of its parent [Content](./ai.content.md#content_interface) object. This identifies which part of the content the segment belongs to. | +| [startIndex](./ai.segment.md#segmentstartindex) | number | The zero-based start index of the segment within the specified Part, measured in UTF-8 bytes. This offset is inclusive, starting from 0 at the beginning of the part's content (e.g., Part.text). | +| [text](./ai.segment.md#segmenttext) | string | The text corresponding to the segment from the response. | ## Segment.endIndex +The zero-based end index of the segment within the specified `Part`, measured in UTF-8 bytes. This offset is exclusive, meaning the character at this index is not included in the segment. + Signature: ```typescript @@ -35,6 +39,8 @@ endIndex: number; ## Segment.partIndex +The zero-based index of the [Part](./ai.md#part) object within the `parts` array of its parent [Content](./ai.content.md#content_interface) object. This identifies which part of the content the segment belongs to. + Signature: ```typescript @@ -43,8 +49,20 @@ partIndex: number; ## Segment.startIndex +The zero-based start index of the segment within the specified `Part`, measured in UTF-8 bytes. This offset is inclusive, starting from 0 at the beginning of the part's content (e.g., `Part.text`). + Signature: ```typescript startIndex: number; ``` + +## Segment.text + +The text corresponding to the segment from the response. + +Signature: + +```typescript +text: string; +``` diff --git a/docs-devsite/ai.webgroundingchunk.md b/docs-devsite/ai.webgroundingchunk.md new file mode 100644 index 00000000000..192940a2d4a --- /dev/null +++ b/docs-devsite/ai.webgroundingchunk.md @@ -0,0 +1,61 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# WebGroundingChunk interface +A grounding chunk from the web. + +Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". + +Signature: + +```typescript +export interface WebGroundingChunk +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [domain](./ai.webgroundingchunk.md#webgroundingchunkdomain) | string | The domain of the original URI from which the content was retrieved (e.g., example.com).This property is only supported in the Vertex AI Gemini API ([VertexAIBackend](./ai.vertexaibackend.md#vertexaibackend_class)). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)), this property will be undefined. | +| [title](./ai.webgroundingchunk.md#webgroundingchunktitle) | string | The title of the retrieved web page. | +| [uri](./ai.webgroundingchunk.md#webgroundingchunkuri) | string | The URI of the retrieved web page. | + +## WebGroundingChunk.domain + +The domain of the original URI from which the content was retrieved (e.g., `example.com`). + +This property is only supported in the Vertex AI Gemini API ([VertexAIBackend](./ai.vertexaibackend.md#vertexaibackend_class)). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)), this property will be undefined. + +Signature: + +```typescript +domain?: string; +``` + +## WebGroundingChunk.title + +The title of the retrieved web page. + +Signature: + +```typescript +title?: string; +``` + +## WebGroundingChunk.uri + +The URI of the retrieved web page. + +Signature: + +```typescript +uri?: string; +``` diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index af877396cc8..4dd65d6469e 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -113,6 +113,46 @@ describe('Generate Content', () => { ).to.be.closeTo(4, TOKEN_COUNT_DELTA); }); + it('generateContent: google search grounding', async () => { + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig: commonGenerationConfig, + safetySettings: commonSafetySettings, + tools: [{ googleSearch: {} }] + }); + + const result = await model.generateContent( + 'What is the speed of light in a vaccuum in meters per second?' + ); + const response = result.response; + const trimmedText = response.text().trim(); + const groundingMetadata = response.candidates?.[0].groundingMetadata; + expect(trimmedText).to.contain('299,792,458'); + expect(groundingMetadata).to.exist; + expect(groundingMetadata!.searchEntryPoint?.renderedContent).to.contain( + 'div' + ); + expect( + groundingMetadata!.groundingChunks + ).to.have.length.greaterThanOrEqual(1); + groundingMetadata!.groundingChunks!.forEach(groundingChunk => { + expect(groundingChunk.web).to.exist; + expect(groundingChunk.web!.uri).to.exist; + }); + expect( + groundingMetadata?.groundingSupports + ).to.have.length.greaterThanOrEqual(1); + groundingMetadata!.groundingSupports!.forEach(groundingSupport => { + expect( + groundingSupport.groundingChunkIndices + ).to.have.length.greaterThanOrEqual(1); + expect(groundingSupport.segment).to.exist; + expect(groundingSupport.segment?.endIndex).to.exist; + expect(groundingSupport.segment?.text).to.exist; + // Since partIndex and startIndex are commonly 0, they may be omitted from responses. + }); + }); + it('generateContentStream: text input, text output', async () => { const model = getGenerativeModel(testConfig.ai, { model: testConfig.model, diff --git a/packages/ai/src/methods/generate-content.test.ts b/packages/ai/src/methods/generate-content.test.ts index 13250fd83dd..a5e4c0d1b57 100644 --- a/packages/ai/src/methods/generate-content.test.ts +++ b/packages/ai/src/methods/generate-content.test.ts @@ -193,6 +193,53 @@ describe('generateContent()', () => { match.any ); }); + it('google search grounding', async () => { + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-google-search-grounding.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const result = await generateContent( + fakeApiSettings, + 'model', + fakeRequestParams + ); + expect(result.response.text()).to.include('The temperature is 67°F (19°C)'); + const groundingMetadata = result.response.candidates?.[0].groundingMetadata; + expect(groundingMetadata).to.not.be.undefined; + expect(groundingMetadata!.searchEntryPoint?.renderedContent).to.contain( + 'div' + ); + expect(groundingMetadata!.groundingChunks?.length).to.equal(2); + expect(groundingMetadata!.groundingChunks?.[0].web?.uri).to.contain( + 'https://vertexaisearch.cloud.google.com' + ); + expect(groundingMetadata!.groundingChunks?.[0].web?.title).to.equal( + 'accuweather.com' + ); + expect(groundingMetadata!.groundingSupports?.length).to.equal(3); + expect( + groundingMetadata!.groundingSupports?.[0].groundingChunkIndices + ).to.deep.equal([0]); + expect(groundingMetadata!.groundingSupports?.[0].segment).to.deep.equal({ + endIndex: 56, + text: 'The current weather in London, United Kingdom is cloudy.' + }); + expect(groundingMetadata!.groundingSupports?.[0].segment?.partIndex).to.be + .undefined; + expect(groundingMetadata!.groundingSupports?.[0].segment?.startIndex).to.be + .undefined; + + expect(makeRequestStub).to.be.calledWith( + 'model', + Task.GENERATE_CONTENT, + fakeApiSettings, + false, + match.any + ); + }); it('blocked prompt', async () => { const mockResponse = getMockResponse( 'vertexAI', diff --git a/packages/ai/src/requests/request.ts b/packages/ai/src/requests/request.ts index 31c5e9b8125..6af21b19572 100644 --- a/packages/ai/src/requests/request.ts +++ b/packages/ai/src/requests/request.ts @@ -185,7 +185,9 @@ export async function makeRequest( } if ( response.status === 403 && + errorDetails && errorDetails.some( + // FIXME: cannot read 'some' of undefined (detail: ErrorDetails) => detail.reason === 'SERVICE_DISABLED' ) && errorDetails.some((detail: ErrorDetails) => diff --git a/packages/ai/src/requests/stream-reader.ts b/packages/ai/src/requests/stream-reader.ts index 543d1d02266..c3a35b1da4a 100644 --- a/packages/ai/src/requests/stream-reader.ts +++ b/packages/ai/src/requests/stream-reader.ts @@ -190,6 +190,8 @@ export function aggregateResponses( candidate.finishMessage; aggregatedResponse.candidates[i].safetyRatings = candidate.safetyRatings; + aggregatedResponse.candidates[i].groundingMetadata = + candidate.groundingMetadata; /** * Candidates should always have content and parts, but this handles diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index 67f45095c2a..8c156164d72 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -165,7 +165,7 @@ export interface RequestOptions { * Defines a tool that model can call to access external knowledge. * @public */ -export declare type Tool = FunctionDeclarationsTool; +export declare type Tool = FunctionDeclarationsTool | GoogleSearchTool; /** * Structured representation of a function declaration as defined by the @@ -196,6 +196,32 @@ export declare interface FunctionDeclaration { parameters?: ObjectSchemaInterface; } +/** + * A tool that allows the generative model to connect to Google Search + * to access and incorporate up-to-date information from the web into its responses. + * + * When using Grounding with Google Search, you may receive a response with data generated from + * Google's search engine. This response is a "Grounded Result" subject to the Grounding with Google Search + * terms in the {@link https://cloud.google.com/terms/service-terms | Service Specific Terms}. + * + * @public + */ +export declare interface GoogleSearchTool { + /** + * Specifies the Google Search configuration. + * Currently, this is an empty object, but it's reserved for future configuration options. + */ + googleSearch: GoogleSearch; +} + +/** + * Configuration for the {@link GoogleSearchTool}. + * + * @remarks Currently, this interface is empty and serves as a placeholder for future configuration options. + * @public + */ +export declare interface GoogleSearch {} + /** * A `FunctionDeclarationsTool` is a piece of code that enables the system to * interact with external systems to perform an action, or set of actions, diff --git a/packages/ai/src/types/responses.ts b/packages/ai/src/types/responses.ts index e33b8a86bd3..1df8760d158 100644 --- a/packages/ai/src/types/responses.ts +++ b/packages/ai/src/types/responses.ts @@ -171,36 +171,192 @@ export interface Citation { } /** - * Metadata returned to client when grounding is enabled. + * Metadata returned when grounding is enabled. + * + * Currently, the only way to use grounding is to include a {@link GoogleSearchTool} in your {@link GenerationConfig}. + * + * Important: If using Grounding with Google Search, you are required to comply with the + * {@link https://cloud.google.com/terms/service-terms | Service Specific Terms} for "Grounding with Google Search". + * * @public */ export interface GroundingMetadata { + /** + * Google search entry point for web searches. + * This contains An HTML/CSS snippet that *must* be embedded in an app to display a Google Search Entry point + * for follow-up web searches related to the model's "Grounded Response". + * + * Important: If using Grounding with Google Search, you are required to comply with the + * {@link https://cloud.google.com/terms/service-terms | Service Specific Terms} for "Grounding with Google Search". + */ + searchEntryPoint?: SearchEntrypoint; + /** + * A list of {@link GroundingChunk} objects. Each chunk represents a piece of retrieved content + * (e.g. from a web page). that the model used to ground its response. + * + * Important: If using Grounding with Google Search, you are required to comply with the + * {@link https://cloud.google.com/terms/service-terms | Service Specific Terms} for "Grounding with Google Search". + */ + groundingChunks?: GroundingChunk[]; + /** + * A list of {@link GroundingSupport} objects. Each object details how specific + * segments of the model's response are supported by the `groundingChunks`. + * + * Important: If using Grounding with Google Search, you are required to comply with the + * {@link https://cloud.google.com/terms/service-terms | Service Specific Terms} for "Grounding with Google Search". + */ + groundingSupports?: GroundingSupport[]; + /** + * A list of web search queries that the model performed to gather the grounding information. + * These can be used to allow users to explore the search results themselves. + * + * Important: If using Grounding with Google Search, you are required to comply with the + * {@link https://cloud.google.com/terms/service-terms | Service Specific Terms} for "Grounding with Google Search". + */ webSearchQueries?: string[]; + /** + * @deprecated Use {@link GroundingSupport} instead. + */ retrievalQueries?: string[]; /** - * @deprecated + * @deprecated Use {@link GroundingChunk} instead. */ groundingAttributions: GroundingAttribution[]; } /** - * @deprecated + * Google search entry point. + * + * Important: If using Grounding with Google Search, you are required to comply with the + * {@link https://cloud.google.com/terms/service-terms | Service Specific Terms} for "Grounding with Google Search". + * * @public */ -export interface GroundingAttribution { - segment: Segment; - confidenceScore?: number; - web?: WebAttribution; - retrievedContext?: RetrievedContextAttribution; +export interface SearchEntrypoint { + /** + * HTML/CSS snippet that can be embedded in a web page. The snippet is designed to avoid undesired + * interaction with the rest of the page's CSS. + * + * To ensure proper rendering and prevent CSS conflicts, it is recommended + * to encapsulate this `renderedContent` within a shadow DOM when embedding it + * into a webpage. See {@link https://developer.mozilla.org/en-US/docs/Web/API/Web_components/Using_shadow_DOM | MDN: Using shadow DOM}. + * + * @example + * ```javascript + * const container = document.createElement('div'); + * document.body.appendChild(container); + * container.attachShadow({ mode: 'open' }).innerHTML = renderedContent; + * ``` + */ + renderedContent?: string; +} + +/** + * Represents a chunk of retrieved data that supports a claim in the model's response. + * This is part of the grounding information provided when grounding is enabled. + * + * Important: If using Grounding with Google Search, you are required to comply with the + * {@link https://cloud.google.com/terms/service-terms | Service Specific Terms} for "Grounding with Google Search". + * + * @public + */ +export interface GroundingChunk { + /** + * Contains details if the grounding chunk is from a web source. + */ + web?: WebGroundingChunk; +} + +/** + * A grounding chunk from the web. + * + * Important: If using Grounding with Google Search, you are required to comply with the + * {@link https://cloud.google.com/terms/service-terms | Service Specific Terms} for "Grounding with Google Search". + * + * @public + */ +export interface WebGroundingChunk { + /** + * The URI of the retrieved web page. + */ + uri?: string; + /** + * The title of the retrieved web page. + */ + title?: string; + /** + * The domain of the original URI from which the content was retrieved (e.g., `example.com`). + * + * This property is only supported in the Vertex AI Gemini API ({@link VertexAIBackend}). + * When using the Gemini Developer API ({@link GoogleAIBackend}), this property will be undefined. + */ + domain?: string; } /** + * Provides information about how a specific segment of the model's response + * is supported by the retrieved grounding chunks. + * + * Important: If using Grounding with Google Search, you are required to comply with the + * {@link https://cloud.google.com/terms/service-terms | Service Specific Terms} for "Grounding with Google Search". + * + * @public + */ +export interface GroundingSupport { + /** + * Specifies the segment of the model's response content that this grounding support pertains to. + */ + segment?: Segment; + /** + * A list of indices that refer to specific {@link GroundingChunk} objects within the + * {@link GroundingMetadata.groundingChunks} array. These referenced chunks + * are the sources that support the claim made in the associated `segment` of the response. + * For example, an array `[1, 3, 4]` means that `groundingChunks[1]`, `groundingChunks[3]`, + * and `groundingChunks[4]` are the retrieved content supporting this part of the response. + */ + groundingChunkIndices?: number[]; +} + +/** + * Represents a specific segment within a {@link Content} object, often used to + * pinpoint the exact location of text or data that grounding information refers to. + * * @public */ export interface Segment { + /** + * The zero-based index of the {@link Part} object within the `parts` array + * of its parent {@link Content} object. This identifies which part of the + * content the segment belongs to. + */ partIndex: number; + /** + * The zero-based start index of the segment within the specified `Part`, + * measured in UTF-8 bytes. This offset is inclusive, starting from 0 at the + * beginning of the part's content (e.g., `Part.text`). + */ startIndex: number; + /** + * The zero-based end index of the segment within the specified `Part`, + * measured in UTF-8 bytes. This offset is exclusive, meaning the character + * at this index is not included in the segment. + */ endIndex: number; + /** + * The text corresponding to the segment from the response. + */ + text: string; +} + +/** + * @deprecated + * @public + */ +export interface GroundingAttribution { + segment: Segment; + confidenceScore?: number; + web?: WebAttribution; + retrievedContext?: RetrievedContextAttribution; } /**