Merge branch 'main' of https://github.com/elastic/elasticsearch into esql-text-embedding-function

Author: afoucret

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java

@@ -92,7 +92,8 @@ public class ValuesSourceReaderBenchmark {
         "double",
         "keyword",
         "stored_keyword",
-        "3_stored_keywords" };
+        "3_stored_keywords",
+        "keyword_mv" };
 
     private static final int BLOCK_LENGTH = 16 * 1024;
     private static final int INDEX_SIZE = 10 * BLOCK_LENGTH;
@@ -332,7 +333,7 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
     @Param({ "in_order", "shuffled" })
     public String layout;
 
-    @Param({ "long", "keyword", "stored_keyword" })
+    @Param({ "long", "keyword", "stored_keyword", "keyword_mv" })
     public String name;
 
     private Directory directory;
@@ -398,6 +399,22 @@ public void benchmark() {
                         }
                     }
                 }
+                case "keyword_mv" -> {
+                    BytesRef scratch = new BytesRef();
+                    BytesRefBlock values = op.getOutput().<BytesRefBlock>getBlock(1);
+                    for (int p = 0; p < values.getPositionCount(); p++) {
+                        int count = values.getValueCount(p);
+                        if (count > 0) {
+                            int first = values.getFirstValueIndex(p);
+                            for (int i = 0; i < count; i++) {
+                                BytesRef r = values.getBytesRef(first + i, scratch);
+                                r.offset++;
+                                r.length--;
+                                sum += Integer.parseInt(r.utf8ToString());
+                            }
+                        }
+                    }
+                }
             }
         }
         long expected = 0;
@@ -407,6 +424,16 @@ public void benchmark() {
                     expected += i % 1000;
                 }
                 break;
+            case "keyword_mv":
+                for (int i = 0; i < INDEX_SIZE; i++) {
+                    int v1 = i % 1000;
+                    expected += v1;
+                    int v2 = i % 500;
+                    if (v1 != v2) {
+                        expected += v2;
+                    }
+                }
+                break;
             case "3_stored_keywords":
                 for (int i = 0; i < INDEX_SIZE; i++) {
                     expected += 3 * (i % 1000);
@@ -461,7 +488,9 @@ private void setupIndex() throws IOException {
                         new StoredField("double", (double) i),
                         new KeywordFieldMapper.KeywordField("keyword_1", new BytesRef(c + i % 1000), keywordFieldType),
                         new KeywordFieldMapper.KeywordField("keyword_2", new BytesRef(c + i % 1000), keywordFieldType),
-                        new KeywordFieldMapper.KeywordField("keyword_3", new BytesRef(c + i % 1000), keywordFieldType)
+                        new KeywordFieldMapper.KeywordField("keyword_3", new BytesRef(c + i % 1000), keywordFieldType),
+                        new KeywordFieldMapper.KeywordField("keyword_mv", new BytesRef(c + i % 1000), keywordFieldType),
+                        new KeywordFieldMapper.KeywordField("keyword_mv", new BytesRef(c + i % 500), keywordFieldType)
                     )
                 );
                 if (i % COMMIT_INTERVAL == 0) {

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesAggregatorBenchmark.java

@@ -95,8 +95,7 @@ static void selfTest() {
         try {
             for (String groups : ValuesAggregatorBenchmark.class.getField("groups").getAnnotationsByType(Param.class)[0].value()) {
                 for (String dataType : ValuesAggregatorBenchmark.class.getField("dataType").getAnnotationsByType(Param.class)[0].value()) {
-                    run(Integer.parseInt(groups), dataType, 10, 0);
-                    run(Integer.parseInt(groups), dataType, 10, 1);
+                    run(Integer.parseInt(groups), dataType, 10);
                 }
             }
         } catch (NoSuchFieldException e) {
@@ -114,10 +113,7 @@ static void selfTest() {
     @Param({ BYTES_REF, INT, LONG })
     public String dataType;
 
-    @Param({ "0", "1" })
-    public int numOrdinalMerges;
-
-    private static Operator operator(DriverContext driverContext, int groups, String dataType, int numOrdinalMerges) {
+    private static Operator operator(DriverContext driverContext, int groups, String dataType) {
         if (groups == 1) {
             return new AggregationOperator(
                 List.of(supplier(dataType).aggregatorFactory(AggregatorMode.SINGLE, List.of(0)).apply(driverContext)),
@@ -132,20 +128,8 @@ private static Operator operator(DriverContext driverContext, int groups, String
         ) {
             @Override
             public Page getOutput() {
-                mergeOrdinal();
                 return super.getOutput();
             }
-
-            // simulate OrdinalsGroupingOperator
-            void mergeOrdinal() {
-                var merged = supplier(dataType).groupingAggregatorFactory(AggregatorMode.SINGLE, List.of(1)).apply(driverContext);
-                for (int i = 0; i < numOrdinalMerges; i++) {
-                    for (int p = 0; p < groups; p++) {
-                        merged.addIntermediateRow(p, aggregators.getFirst(), p);
-                    }
-                }
-                aggregators.set(0, merged);
-            }
         };
     }
 
@@ -352,12 +336,12 @@ private static Block groupingBlock(int groups) {
 
     @Benchmark
     public void run() {
-        run(groups, dataType, OP_COUNT, numOrdinalMerges);
+        run(groups, dataType, OP_COUNT);
     }
 
-    private static void run(int groups, String dataType, int opCount, int numOrdinalMerges) {
+    private static void run(int groups, String dataType, int opCount) {
         DriverContext driverContext = driverContext();
-        try (Operator operator = operator(driverContext, groups, dataType, numOrdinalMerges)) {
+        try (Operator operator = operator(driverContext, groups, dataType)) {
             Page page = page(groups, dataType);
             for (int i = 0; i < opCount; i++) {
                 operator.addInput(page.shallowCopy());

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java

@@ -17,7 +17,7 @@ jna               = 5.12.1
 netty             = 4.1.118.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
-awsv2sdk            = 2.30.38
+awsv2sdk            = 2.31.78
 reactive_streams    = 1.0.4
 
 antlr4            = 4.13.1

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java

@@ -0,0 +1,5 @@
+pr: 121914
+summary: Support Fields API in conditional ingest processors
+area: Infra/Core
+type: feature
+issues: []

build-tools-internal/version.properties

@@ -0,0 +1,6 @@
+pr: 128639
+summary: Substitue `date_trunc` with `round_to` when the pre-calculated rounding points
+  are available
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/121914.yaml

@@ -0,0 +1,5 @@
+pr: 129929
+summary: Add support for RLIKE (LIST) with pushdown
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/128639.yaml

@@ -0,0 +1,6 @@
+pr: 131050
+summary: Upgrade AWS Java SDK to 2.31.78
+area: "Snapshot/Restore"
+type: upgrade
+issues: []
+

docs/changelog/129929.yaml

@@ -0,0 +1,5 @@
+pr: 131061
+summary: Speed up reading multivalued keywords
+area: ES|QL
+type: enhancement
+issues: []