diff --git a/output/openapi/elasticsearch-serverless-openapi.json b/output/openapi/elasticsearch-serverless-openapi.json index 5d1bf30103..3146ee0c56 100644 --- a/output/openapi/elasticsearch-serverless-openapi.json +++ b/output/openapi/elasticsearch-serverless-openapi.json @@ -28540,6 +28540,9 @@ } ] }, + "retriever": { + "$ref": "#/components/schemas/_types:RetrieverContainer" + }, "script_fields": { "description": "Retrieve a script evaluation (based on different fields) for each hit.", "type": "object", @@ -59560,6 +59563,142 @@ "_types:RankBase": { "type": "object" }, + "_types:RetrieverContainer": { + "type": "object", + "properties": { + "standard": { + "$ref": "#/components/schemas/_types:StandardRetriever" + }, + "knn": { + "$ref": "#/components/schemas/_types:KnnRetriever" + }, + "rrf": { + "$ref": "#/components/schemas/_types:RRFRetriever" + } + }, + "minProperties": 1, + "maxProperties": 1 + }, + "_types:StandardRetriever": { + "allOf": [ + { + "$ref": "#/components/schemas/_types:RetrieverBase" + }, + { + "type": "object", + "properties": { + "query": { + "$ref": "#/components/schemas/_types.query_dsl:QueryContainer" + }, + "search_after": { + "$ref": "#/components/schemas/_types:SortResults" + }, + "terminate_after": { + "description": "Maximum number of documents to collect for each shard.", + "type": "number" + }, + "sort": { + "$ref": "#/components/schemas/_types:Sort" + }, + "min_score": { + "description": "Minimum _score for matching documents. Documents with a lower _score are not included in the top documents.", + "type": "number" + }, + "collapse": { + "$ref": "#/components/schemas/_global.search._types:FieldCollapse" + } + } + } + ] + }, + "_types:RetrieverBase": { + "type": "object", + "properties": { + "filter": { + "description": "Query to filter the documents that can match.", + "oneOf": [ + { + "$ref": "#/components/schemas/_types.query_dsl:QueryContainer" + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/_types.query_dsl:QueryContainer" + } + } + ] + } + } + }, + "_types:KnnRetriever": { + "allOf": [ + { + "$ref": "#/components/schemas/_types:RetrieverBase" + }, + { + "type": "object", + "properties": { + "field": { + "description": "The name of the vector field to search against.", + "type": "string" + }, + "query_vector": { + "$ref": "#/components/schemas/_types:QueryVector" + }, + "query_vector_builder": { + "$ref": "#/components/schemas/_types:QueryVectorBuilder" + }, + "k": { + "description": "Number of nearest neighbors to return as top hits.", + "type": "number" + }, + "num_candidates": { + "description": "Number of nearest neighbor candidates to consider per shard.", + "type": "number" + }, + "similarity": { + "description": "The minimum similarity required for a document to be considered a match.", + "type": "number" + } + }, + "required": [ + "field", + "k", + "num_candidates" + ] + } + ] + }, + "_types:RRFRetriever": { + "allOf": [ + { + "$ref": "#/components/schemas/_types:RetrieverBase" + }, + { + "type": "object", + "properties": { + "retrievers": { + "description": "A list of child retrievers to specify which sets of returned top documents will have the RRF formula applied to them.", + "type": "array", + "items": { + "$ref": "#/components/schemas/_types:RetrieverContainer" + } + }, + "rank_constant": { + "description": "This value determines how much influence documents in individual result sets per query have over the final ranked result set.", + "type": "number" + }, + "window_size": { + "description": "This value determines the size of the individual result sets per query.", + "type": "number" + } + }, + "required": [ + "retrievers" + ] + } + ] + }, "search_application._types:SearchApplication": { "type": "object", "properties": { diff --git a/output/schema/schema.json b/output/schema/schema.json index 85285f2676..71401c8c64 100644 --- a/output/schema/schema.json +++ b/output/schema/schema.json @@ -27270,7 +27270,7 @@ "type": { "kind": "instance_of", "type": { - "name": "long", + "name": "integer", "namespace": "_types" } } @@ -27282,7 +27282,7 @@ "type": { "kind": "instance_of", "type": { - "name": "long", + "name": "integer", "namespace": "_types" } } @@ -32344,6 +32344,25 @@ "kind": "union_of" } }, + { + "availability": { + "serverless": {}, + "stack": { + "since": "8.14.0" + } + }, + "description": "A retriever is a specification to describe top documents returned from a search. A retriever replaces other elements of the search API that also return top documents such as query and knn.", + "name": "retriever", + "required": false, + "since": "8.14.0", + "type": { + "kind": "instance_of", + "type": { + "name": "RetrieverContainer", + "namespace": "_types" + } + } + }, { "description": "Retrieve a script evaluation (based on different fields) for each hit.", "name": "script_fields", @@ -33182,7 +33201,7 @@ } } ], - "specLocation": "_global/search/SearchRequest.ts#L53-L513" + "specLocation": "_global/search/SearchRequest.ts#L54-L520" }, { "body": { @@ -44873,7 +44892,7 @@ "type": { "kind": "instance_of", "type": { - "name": "long", + "name": "integer", "namespace": "_types" } } @@ -44920,6 +44939,94 @@ ], "specLocation": "_types/Knn.ts#L54-L67" }, + { + "inherits": { + "type": { + "name": "RetrieverBase", + "namespace": "_types" + } + }, + "kind": "interface", + "name": { + "name": "KnnRetriever", + "namespace": "_types" + }, + "properties": [ + { + "description": "The name of the vector field to search against.", + "name": "field", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + }, + { + "description": "Query vector. Must have the same number of dimensions as the vector field you are searching against. You must provide a query_vector_builder or query_vector, but not both.", + "name": "query_vector", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "QueryVector", + "namespace": "_types" + } + } + }, + { + "description": "Defines a model to build a query vector.", + "name": "query_vector_builder", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "QueryVectorBuilder", + "namespace": "_types" + } + } + }, + { + "description": "Number of nearest neighbors to return as top hits.", + "name": "k", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "integer", + "namespace": "_types" + } + } + }, + { + "description": "Number of nearest neighbor candidates to consider per shard.", + "name": "num_candidates", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "integer", + "namespace": "_types" + } + } + }, + { + "description": "The minimum similarity required for a document to be considered a match.", + "name": "similarity", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "float", + "namespace": "_types" + } + } + } + ], + "specLocation": "_types/Retriever.ts#L58-L71" + }, { "kind": "interface", "name": { @@ -44970,7 +45077,7 @@ "type": { "kind": "instance_of", "type": { - "name": "long", + "name": "integer", "namespace": "_types" } } @@ -44982,7 +45089,7 @@ "type": { "kind": "instance_of", "type": { - "name": "long", + "name": "integer", "namespace": "_types" } } @@ -46401,6 +46508,61 @@ "kind": "container" } }, + { + "inherits": { + "type": { + "name": "RetrieverBase", + "namespace": "_types" + } + }, + "kind": "interface", + "name": { + "name": "RRFRetriever", + "namespace": "_types" + }, + "properties": [ + { + "description": "A list of child retrievers to specify which sets of returned top documents will have the RRF formula applied to them.", + "name": "retrievers", + "required": true, + "type": { + "kind": "array_of", + "value": { + "kind": "instance_of", + "type": { + "name": "RetrieverContainer", + "namespace": "_types" + } + } + } + }, + { + "description": "This value determines how much influence documents in individual result sets per query have over the final ranked result set.", + "name": "rank_constant", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "integer", + "namespace": "_types" + } + } + }, + { + "description": "This value determines the size of the individual result sets per query.", + "name": "window_size", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "integer", + "namespace": "_types" + } + } + } + ], + "specLocation": "_types/Retriever.ts#L73-L80" + }, { "kind": "interface", "name": { @@ -46791,6 +46953,92 @@ ], "specLocation": "_types/Retries.ts#L22-L25" }, + { + "kind": "interface", + "name": { + "name": "RetrieverBase", + "namespace": "_types" + }, + "properties": [ + { + "description": "Query to filter the documents that can match.", + "name": "filter", + "required": false, + "type": { + "items": [ + { + "kind": "instance_of", + "type": { + "name": "QueryContainer", + "namespace": "_types.query_dsl" + } + }, + { + "kind": "array_of", + "value": { + "kind": "instance_of", + "type": { + "name": "QueryContainer", + "namespace": "_types.query_dsl" + } + } + } + ], + "kind": "union_of" + } + } + ], + "specLocation": "_types/Retriever.ts#L38-L41" + }, + { + "kind": "interface", + "name": { + "name": "RetrieverContainer", + "namespace": "_types" + }, + "properties": [ + { + "description": "A retriever that replaces the functionality of a traditional query.", + "name": "standard", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "StandardRetriever", + "namespace": "_types" + } + } + }, + { + "description": "A retriever that replaces the functionality of a knn search.", + "name": "knn", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "KnnRetriever", + "namespace": "_types" + } + } + }, + { + "description": "A retriever that produces top documents from reciprocal rank fusion (RRF).", + "name": "rrf", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "RRFRetriever", + "namespace": "_types" + } + } + } + ], + "specLocation": "_types/Retriever.ts#L26-L36", + "variants": { + "kind": "container" + } + }, { "kind": "type_alias", "name": { @@ -48328,6 +48576,94 @@ } } }, + { + "inherits": { + "type": { + "name": "RetrieverBase", + "namespace": "_types" + } + }, + "kind": "interface", + "name": { + "name": "StandardRetriever", + "namespace": "_types" + }, + "properties": [ + { + "description": "Defines a query to retrieve a set of top documents.", + "name": "query", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "QueryContainer", + "namespace": "_types.query_dsl" + } + } + }, + { + "description": "Defines a search after object parameter used for pagination.", + "name": "search_after", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "SortResults", + "namespace": "_types" + } + } + }, + { + "description": "Maximum number of documents to collect for each shard.", + "name": "terminate_after", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "integer", + "namespace": "_types" + } + } + }, + { + "description": "A sort object that that specifies the order of matching documents.", + "name": "sort", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "Sort", + "namespace": "_types" + } + } + }, + { + "description": "Minimum _score for matching documents. Documents with a lower _score are not included in the top documents.", + "name": "min_score", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "float", + "namespace": "_types" + } + } + }, + { + "description": "Collapses the top documents by a specified key into a single top document per key.", + "name": "collapse", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "FieldCollapse", + "namespace": "_global.search._types" + } + } + } + ], + "specLocation": "_types/Retriever.ts#L43-L56" + }, { "kind": "interface", "name": { diff --git a/output/typescript/types.ts b/output/typescript/types.ts index af1bacc5cf..e1407d293b 100644 --- a/output/typescript/types.ts +++ b/output/typescript/types.ts @@ -671,8 +671,8 @@ export interface KnnSearchResponse { export interface KnnSearchQuery { field: Field query_vector: QueryVector - k: long - num_candidates: long + k: integer + num_candidates: integer } export interface MgetMultiGetError { @@ -1206,6 +1206,7 @@ export interface SearchRequest extends RequestBase { profile?: boolean query?: QueryDslQueryContainer rescore?: SearchRescore | SearchRescore[] + retriever?: RetrieverContainer script_fields?: Record search_after?: SortResults size?: integer @@ -2361,17 +2362,26 @@ export interface KnnQuery extends QueryDslQueryBase { field: Field query_vector?: QueryVector query_vector_builder?: QueryVectorBuilder - num_candidates?: long + num_candidates?: integer filter?: QueryDslQueryContainer | QueryDslQueryContainer[] similarity?: float } +export interface KnnRetriever extends RetrieverBase { + field: string + query_vector?: QueryVector + query_vector_builder?: QueryVectorBuilder + k: integer + num_candidates: integer + similarity?: float +} + export interface KnnSearch { field: Field query_vector?: QueryVector query_vector_builder?: QueryVectorBuilder - k?: long - num_candidates?: long + k?: integer + num_candidates?: integer boost?: float filter?: QueryDslQueryContainer | QueryDslQueryContainer[] similarity?: float @@ -2510,6 +2520,12 @@ export interface QueryVectorBuilder { text_embedding?: TextEmbedding } +export interface RRFRetriever extends RetrieverBase { + retrievers: RetrieverContainer[] + rank_constant?: integer + window_size?: integer +} + export interface RankBase { [key: string]: never } @@ -2560,6 +2576,16 @@ export interface Retries { search: long } +export interface RetrieverBase { + filter?: QueryDslQueryContainer | QueryDslQueryContainer[] +} + +export interface RetrieverContainer { + standard?: StandardRetriever + knn?: KnnRetriever + rrf?: RRFRetriever +} + export type Routing = string export interface RrfRank { @@ -2715,6 +2741,15 @@ export type SortOrder = 'asc' | 'desc' export type SortResults = FieldValue[] +export interface StandardRetriever extends RetrieverBase { + query?: QueryDslQueryContainer + search_after?: SortResults + terminate_after?: integer + sort?: Sort + min_score?: float + collapse?: SearchFieldCollapse +} + export interface StoreStats { size?: ByteSize size_in_bytes: long diff --git a/specification/_global/knn_search/_types/Knn.ts b/specification/_global/knn_search/_types/Knn.ts index a23483e51e..568847e3e6 100644 --- a/specification/_global/knn_search/_types/Knn.ts +++ b/specification/_global/knn_search/_types/Knn.ts @@ -18,7 +18,7 @@ */ import { Field } from '@_types/common' -import { long, float } from '@_types/Numeric' +import { integer } from '@_types/Numeric' import { QueryVector } from '@_types/Knn' export interface Query { @@ -27,7 +27,7 @@ export interface Query { /** The query vector */ query_vector: QueryVector /** The final number of nearest neighbors to return as top hits */ - k: long + k: integer /** The number of nearest neighbor candidates to consider per shard */ - num_candidates: long + num_candidates: integer } diff --git a/specification/_global/search/SearchRequest.ts b/specification/_global/search/SearchRequest.ts index a82b77bb3b..225b109e20 100644 --- a/specification/_global/search/SearchRequest.ts +++ b/specification/_global/search/SearchRequest.ts @@ -48,6 +48,7 @@ import { Operator } from '@_types/query_dsl/Operator' import { Sort, SortResults } from '@_types/sort' import { KnnSearch } from '@_types/Knn' import { RankContainer } from '@_types/Rank' +import { RetrieverContainer } from '@_types/Retriever' import { UserDefinedValue } from '@spec_utils/UserDefinedValue' /** @@ -416,6 +417,12 @@ export interface Request extends RequestBase { * Can be used to improve precision by reordering just the top (for example 100 - 500) documents returned by the `query` and `post_filter` phases. */ rescore?: Rescore | Rescore[] + /** + * A retriever is a specification to describe top documents returned from a search. A retriever replaces other elements of the search API that also return top documents such as query and knn. + * @availability stack since=8.14.0 + * @availability serverless + */ + retriever?: RetrieverContainer /** * Retrieve a script evaluation (based on different fields) for each hit. */ diff --git a/specification/_types/Knn.ts b/specification/_types/Knn.ts index 20571d369a..461801bd31 100644 --- a/specification/_types/Knn.ts +++ b/specification/_types/Knn.ts @@ -18,7 +18,7 @@ */ import { Field } from '@_types/common' -import { long, float } from '@_types/Numeric' +import { float, integer } from '@_types/Numeric' import { QueryBase, QueryContainer } from './query_dsl/abstractions' import { InnerHits } from '@global/search/_types/hits' @@ -35,9 +35,9 @@ export interface KnnSearch { /** The query vector builder. You must provide a query_vector_builder or query_vector, but not both. */ query_vector_builder?: QueryVectorBuilder /** The final number of nearest neighbors to return as top hits */ - k?: long + k?: integer /** The number of nearest neighbor candidates to consider per shard */ - num_candidates?: long + num_candidates?: integer /** Boost value to apply to kNN scores */ boost?: float /** Filters for the kNN search query */ @@ -59,7 +59,7 @@ export interface KnnQuery extends QueryBase { /** The query vector builder. You must provide a query_vector_builder or query_vector, but not both. */ query_vector_builder?: QueryVectorBuilder /** The number of nearest neighbor candidates to consider per shard */ - num_candidates?: long + num_candidates?: integer /** Filters for the kNN search query */ filter?: QueryContainer | QueryContainer[] /** The minimum similarity for a vector to be considered a match */ diff --git a/specification/_types/Retriever.ts b/specification/_types/Retriever.ts new file mode 100644 index 0000000000..bd179180a0 --- /dev/null +++ b/specification/_types/Retriever.ts @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { float, integer } from '@_types/Numeric' +import { QueryContainer } from './query_dsl/abstractions' +import { QueryVector, QueryVectorBuilder } from '@_types/Knn' +import { Sort, SortResults } from '@_types/sort' +import { FieldCollapse } from '@global/search/_types/FieldCollapse' + +/** + * @variants container + */ +export class RetrieverContainer { + /** A retriever that replaces the functionality of a traditional query. */ + standard?: StandardRetriever + /** A retriever that replaces the functionality of a knn search. */ + knn?: KnnRetriever + /** A retriever that produces top documents from reciprocal rank fusion (RRF). */ + rrf?: RRFRetriever +} + +export class RetrieverBase { + /** Query to filter the documents that can match. */ + filter?: QueryContainer | QueryContainer[] +} + +export class StandardRetriever extends RetrieverBase { + /** Defines a query to retrieve a set of top documents. */ + query?: QueryContainer + /** Defines a search after object parameter used for pagination. */ + search_after?: SortResults + /** Maximum number of documents to collect for each shard. */ + terminate_after?: integer + /** A sort object that that specifies the order of matching documents. */ + sort?: Sort + /** Minimum _score for matching documents. Documents with a lower _score are not included in the top documents. */ + min_score?: float + /** Collapses the top documents by a specified key into a single top document per key. */ + collapse?: FieldCollapse +} + +export class KnnRetriever extends RetrieverBase { + /** The name of the vector field to search against. */ + field: string + /** Query vector. Must have the same number of dimensions as the vector field you are searching against. You must provide a query_vector_builder or query_vector, but not both. */ + query_vector?: QueryVector + /** Defines a model to build a query vector. */ + query_vector_builder?: QueryVectorBuilder + /** Number of nearest neighbors to return as top hits. */ + k: integer + /** Number of nearest neighbor candidates to consider per shard. */ + num_candidates: integer + /** The minimum similarity required for a document to be considered a match. */ + similarity?: float +} + +export class RRFRetriever extends RetrieverBase { + /** A list of child retrievers to specify which sets of returned top documents will have the RRF formula applied to them. */ + retrievers: RetrieverContainer[] + /** This value determines how much influence documents in individual result sets per query have over the final ranked result set. */ + rank_constant?: integer + /** This value determines the size of the individual result sets per query. */ + window_size?: integer +}