Merge branch 'main' of github.com:elastic/elasticsearch into custom-inference-service

Author: jonathan-buttner

.buildkite/scripts/third-party-test-credentials.sh

@@ -7,6 +7,7 @@ set -euo pipefail
 # The second/lowercase export is what the tests expect/require
 
 if [[ "${USE_3RD_PARTY_AZURE_CREDENTIALS:-}" == "true" ]]; then
+  # These credentials expire periodically and must be manually renewed - the process is in the onboarding/process docs.
   json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_test_creds)
 
   AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id)
@@ -19,6 +20,7 @@ if [[ "${USE_3RD_PARTY_AZURE_CREDENTIALS:-}" == "true" ]]; then
 fi
 
 if [[ "${USE_3RD_PARTY_AZURE_SAS_CREDENTIALS:-}" == "true" ]]; then
+  # These credentials expire periodically and must be manually renewed - the process is in the onboarding/process docs.
   json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_sas_test_creds)
 
   AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id)

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesAggregatorBenchmark.java

@@ -21,10 +21,13 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.data.OrdinalBytesRefVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.AggregationOperator;
 import org.elasticsearch.compute.operator.DriverContext;
@@ -282,11 +285,18 @@ private static Block dataBlock(int groups, String dataType) {
         int blockLength = blockLength(groups);
         return switch (dataType) {
             case BYTES_REF -> {
-                try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(blockLength)) {
+                try (
+                    BytesRefVector.Builder dict = blockFactory.newBytesRefVectorBuilder(blockLength);
+                    IntVector.Builder ords = blockFactory.newIntVectorBuilder(blockLength)
+                ) {
+                    final int dictLength = Math.min(blockLength, KEYWORDS.length);
+                    for (int i = 0; i < dictLength; i++) {
+                        dict.appendBytesRef(KEYWORDS[i]);
+                    }
                     for (int i = 0; i < blockLength; i++) {
-                        builder.appendBytesRef(KEYWORDS[i % KEYWORDS.length]);
+                        ords.appendInt(i % dictLength);
                     }
-                    yield builder.build();
+                    yield new OrdinalBytesRefVector(ords.build(), dict.build()).asBlock();
                 }
             }
             case INT -> {

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java

@@ -17,7 +17,7 @@
 import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.simdvec.internal.vectorization.ES91OSQVectorsScorer;
+import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
 import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;

docs/changelog/127355.yaml

@@ -0,0 +1,5 @@
+pr: 127355
+summary: '`text ==` and `text !=` pushdown'
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/127629.yaml

@@ -0,0 +1,5 @@
+pr: 127629
+summary: ES|QL SAMPLE aggregation function
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/127752.yaml

@@ -0,0 +1,6 @@
+pr: 127752
+summary: Downsampling does not consider passthrough fields as dimensions
+area: Downsampling
+type: bug
+issues:
+ - 125156

docs/changelog/127817.yaml

@@ -0,0 +1,5 @@
+pr: 127817
+summary: Replace auto-read with proper flow-control in HTTP pipeline
+area: Network
+type: enhancement
+issues: []

docs/changelog/127824.yaml

@@ -0,0 +1,6 @@
+pr: 127824
+summary: Skip the validation when retrieving the index mode during reindexing a time
+  series data stream
+area: TSDB
+type: bug
+issues: []

docs/changelog/127849.yaml

@@ -0,0 +1,5 @@
+pr: 127849
+summary: Optimize ordinal inputs in Values aggregation
+area: "ES|QL"
+type: enhancement
+issues: []

docs/changelog/127856.yaml

@@ -0,0 +1,5 @@
+pr: 127856
+summary: Fix services API Google Vertex AI Rerank location field requirement
+area: Machine Learning
+type: bug
+issues: []

docs/reference/elasticsearch/mapping-reference/keyword.md

@@ -232,6 +232,42 @@ Will become:
 }
 ```
 
+If `null_value` is configured, `null` values are replaced with the `null_value` in synthetic source:
+
+$$$synthetic-source-keyword-example-null-value$$$
+
+```console
+PUT idx
+{
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties": {
+      "kwd": { "type": "keyword", "null_value": "NA" }
+    }
+  }
+}
+PUT idx/_doc/1
+{
+  "kwd": ["foo", null, "bar"]
+}
+```
+
+Will become:
+
+```console-result
+{
+  "kwd": ["NA", "bar", "foo"]
+}
+```
+
 
 ## Constant keyword field type [constant-keyword-field-type]
 

docs/reference/elasticsearch/mapping-reference/text.md

@@ -104,11 +104,17 @@ Synthetic `_source` is Generally Available only for TSDB indices (indices that h
 ::::
 
 
-`text` fields support [synthetic `_source`](/reference/elasticsearch/mapping-reference/mapping-source-field.md#synthetic-source) if they have a [`keyword`](/reference/elasticsearch/mapping-reference/keyword.md#keyword-synthetic-source) sub-field that supports synthetic `_source` or if the `text` field sets `store` to `true`. Either way, it may not have [`copy_to`](/reference/elasticsearch/mapping-reference/copy-to.md).
+`text` fields can use a [`keyword`](/reference/elasticsearch/mapping-reference/keyword.md#keyword-synthetic-source) sub-field to support [synthetic `_source`](/reference/elasticsearch/mapping-reference/mapping-source-field.md#synthetic-source) without storing values of the text field itself.
 
-If using a sub-`keyword` field, then the values are sorted in the same way as a `keyword` field’s values are sorted. By default, that means sorted with duplicates removed. So:
+In this case, the synthetic source of the `text` field will have the same [modifications](/reference/elasticsearch/mapping-reference/mapping-source-field.md#synthetic-source) as a `keyword` field.
 
-$$$synthetic-source-text-example-default$$$
+These modifications can impact usage of `text` fields:
+* Reordering text fields can have an effect on [phrase](/reference/query-languages/query-dsl/query-dsl-match-query-phrase.md) and [span](/reference/query-languages/query-dsl/span-queries.md) queries. See the discussion about [`position_increment_gap`](/reference/elasticsearch/mapping-reference/position-increment-gap.md) for more details. You can avoid this by making sure the `slop` parameter on the phrase queries is lower than the `position_increment_gap`. This is the default.
+* Handling of `null` values is different. `text` fields ignore `null` values, but `keyword` fields support replacing `null` values with a value specified in the `null_value` parameter. This replacement is represented in synthetic source.
+
+For example:
+
+$$$synthetic-source-text-example-multi-field$$$
 
 ```console
 PUT idx
@@ -127,8 +133,9 @@ PUT idx
       "text": {
         "type": "text",
         "fields": {
-          "raw": {
-            "type": "keyword"
+          "kwd": {
+            "type": "keyword",
+            "null_value": "NA"
           }
         }
       }
@@ -138,6 +145,7 @@ PUT idx
 PUT idx/_doc/1
 {
   "text": [
+    null,
     "the quick brown fox",
     "the quick brown fox",
     "jumped over the lazy dog"
@@ -150,18 +158,15 @@ Will become:
 ```console-result
 {
   "text": [
+    "NA",
     "jumped over the lazy dog",
     "the quick brown fox"
   ]
 }
 ```
 
-::::{note}
-Reordering text fields can have an effect on [phrase](/reference/query-languages/query-dsl/query-dsl-match-query-phrase.md) and [span](/reference/query-languages/query-dsl/span-queries.md) queries. See the discussion about [`position_increment_gap`](/reference/elasticsearch/mapping-reference/position-increment-gap.md) for more detail. You can avoid this by making sure the `slop` parameter on the phrase queries is lower than the `position_increment_gap`. This is the default.
-::::
 
-
-If the `text` field sets `store` to true then order and duplicates are preserved.
+If the `text` field sets `store` to `true` then the sub-field is not used and no modifications are applied. For example:
 
 $$$synthetic-source-text-example-stored$$$
 
@@ -179,7 +184,15 @@ PUT idx
   },
   "mappings": {
     "properties": {
-      "text": { "type": "text", "store": true }
+      "text": {
+        "type": "text",
+        "store": true,
+        "fields": {
+          "raw": {
+            "type": "keyword"
+          }
+        }
+      }
     }
   }
 }